Logging, closing, modifying

Author: Mehakmeet Singh

hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsOutputStream.java

@@ -36,10 +36,10 @@
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
 
-import org.apache.hadoop.fs.FileSystem.Statistics;
 import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AbfsRestOperationException;
 import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AzureBlobFileSystemException;
 import org.apache.hadoop.io.ElasticByteBufferPool;
+import org.apache.hadoop.fs.FileSystem.Statistics;
 import org.apache.hadoop.fs.FSExceptionMessages;
 import org.apache.hadoop.fs.StreamCapabilities;
 import org.apache.hadoop.fs.Syncable;
@@ -51,7 +51,6 @@
  */
 public class AbfsOutputStream extends OutputStream implements Syncable, StreamCapabilities {
   private final AbfsClient client;
-  private final Statistics statistics;
   private final String path;
   private long position;
   private boolean closed;
@@ -82,6 +81,8 @@ public class AbfsOutputStream extends OutputStream implements Syncable, StreamCa
   private final ElasticByteBufferPool byteBufferPool
           = new ElasticByteBufferPool();
 
+  private final Statistics statistics;
+
   public AbfsOutputStream(
       final AbfsClient client,
       final Statistics statistics,

hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsTestWithTimeout.java

@@ -110,12 +110,15 @@ protected boolean validateContent(AzureBlobFileSystem fs, Path path,
 
       while (valueOfContentAtPos != -1 && pos < lenOfOriginalByteArray) {
         if (originalByteArray[pos] != valueOfContentAtPos) {
+          assertEquals("Mismatch in content validation at position {}", pos,
+              originalByteArray[pos], valueOfContentAtPos);
           return false;
         }
         valueOfContentAtPos = (byte) in.read();
         pos++;
       }
       if (valueOfContentAtPos != -1) {
+        assertEquals("Expected end of file", -1, valueOfContentAtPos);
         return false;
       }
       return true;

hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsStreamStatistics.java

@@ -18,9 +18,11 @@
 
 package org.apache.hadoop.fs.azurebfs;
 
-import org.junit.Assert;
 import org.junit.Test;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FileSystem;
@@ -35,6 +37,9 @@ public class ITestAbfsStreamStatistics extends AbstractAbfsIntegrationTest {
   public ITestAbfsStreamStatistics() throws Exception {
   }
 
+  private static final Logger LOG =
+      LoggerFactory.getLogger(ITestAbfsStreamStatistics.class);
+
   private static int LARGE_NUMBER_OF_OPS = 1000000;
 
   /***
@@ -71,20 +76,29 @@ public void testAbfsStreamOps() throws Exception {
       //Test for a single write operation
       assertReadWriteOps("write", 1, statistics.getWriteOps());
 
+      //Flushing output stream to see content to read
+      outForOneOperation.hflush();
       inForOneOperation = fs.open(smallOperationsFile);
-      inForOneOperation.read(testReadWriteOps.getBytes(), 0,
+      statistics.reset();
+      int result = inForOneOperation.read(testReadWriteOps.getBytes(), 0,
           testReadWriteOps.getBytes().length);
 
-      //Test for a single read operation
-      assertReadWriteOps("read", 1, statistics.getReadOps());
+      LOG.info("Result of Read operation : {}", result);
+      /*
+      Testing if 2 read_ops value is coming after reading full content from a
+      file (3 if anything to read from Buffer too).
+      Reason: read() call gives read_ops=1,
+      reading from AbfsClient(http GET) gives read_ops=2.
+       */
+      assertReadWriteOps("read", 2, statistics.getReadOps());
 
     } finally {
-      IOUtils.cleanupWithLogger(null, inForOneOperation,
+      IOUtils.cleanupWithLogger(LOG, inForOneOperation,
           outForOneOperation);
     }
 
     //Validating if content is being written in the smallOperationsFile
-    Assert.assertEquals("Mismatch in content validation", true,
+    assertTrue("Mismatch in content validation",
         validateContent(fs, smallOperationsFile,
             testReadWriteOps.getBytes()));
 
@@ -101,6 +115,8 @@ public void testAbfsStreamOps() throws Exception {
         //Creating the String for content Validation
         largeOperationsValidationString.append(testReadWriteOps);
       }
+      LOG.info("Number of bytes of Large data written: {}",
+          largeOperationsValidationString.toString().getBytes().length);
 
       //Test for 1000000 write operations
       assertReadWriteOps("write", largeValue, statistics.getWriteOps());
@@ -116,12 +132,11 @@ public void testAbfsStreamOps() throws Exception {
       assertReadWriteOps("read", largeValue, statistics.getReadOps());
 
     } finally {
-      IOUtils.cleanupWithLogger(null, inForLargeOperations,
+      IOUtils.cleanupWithLogger(LOG, inForLargeOperations,
           outForLargeOperations);
     }
-
     //Validating if content is being written in largeOperationsFile
-    Assert.assertTrue("Mismatch in content validation",
+    assertTrue("Mismatch in content validation",
         validateContent(fs, largeOperationsFile,
             largeOperationsValidationString.toString().getBytes()));
 
@@ -137,8 +152,7 @@ public void testAbfsStreamOps() throws Exception {
 
   private void assertReadWriteOps(String operation, long expectedValue,
       long actualValue) {
-    Assert
-        .assertEquals("Mismatch in " + operation + " operations", expectedValue,
-            actualValue);
+    assertEquals("Mismatch in " + operation + " operations", expectedValue,
+        actualValue);
   }
 }

hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemOauth.java

@@ -22,7 +22,6 @@
 import java.io.InputStream;
 import java.util.Map;
 
-import org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys;
 import org.junit.Assume;
 import org.junit.Test;
 
@@ -31,6 +30,7 @@
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys;
 import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AbfsRestOperationException;
 import org.apache.hadoop.fs.azurebfs.contracts.services.AzureServiceErrorCode;
 import org.apache.hadoop.fs.azurebfs.services.AuthType;
@@ -146,6 +146,8 @@ public void testBlobDataReader() throws Exception {
       abfsStore.openFileForWrite(EXISTED_FILE_PATH, fs.getFsStatistics(), true);
     } catch (AbfsRestOperationException e) {
       assertEquals(AzureServiceErrorCode.AUTHORIZATION_PERMISSION_MISS_MATCH, e.getErrorCode());
+    } finally {
+      abfsStore.close();
     }
 
   }