From bfb4afab8d49bbc9730691ae3ca5cde5d8b5963d Mon Sep 17 00:00:00 2001
From: Mehakmeet Singh <mehakmeet.singh@cloudera.com>
Date: Thu, 27 Feb 2020 17:50:04 +0530
Subject: [PATCH 1/9] CDPD-8485 Adding file system counters

- Write_ops
- Read_ops
- Bytes_written (already updated)
- Bytes_Read (already updated)

Change-Id: I77349fdd158babd66df665713201fa9c8606f191
---
 .../fs/azurebfs/AzureBlobFileSystem.java      |  4 +-
 .../fs/azurebfs/AzureBlobFileSystemStore.java |  6 +-
 .../fs/azurebfs/services/AbfsInputStream.java | 10 +++
 .../azurebfs/services/AbfsOutputStream.java   | 12 +++
 .../azurebfs/AbstractAbfsTestWithTimeout.java | 48 +++++++++++
 .../fs/azurebfs/ITestAbfsInputStream.java     | 81 +++++++++++++++++++
 .../ITestAzureBlobFileSystemCreate.java       | 56 +++++++++++++
 .../ITestAzureBlobFileSystemOauth.java        |  2 +-
 8 files changed, 214 insertions(+), 5 deletions(-)
 create mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsInputStream.java

diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java
index 8eda2f3730400..4ddc2e396c4de 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java
@@ -188,7 +188,7 @@ public FSDataOutputStream create(final Path f, final FsPermission permission, fi
     Path qualifiedPath = makeQualified(f);
 
     try {
-      OutputStream outputStream = abfsStore.createFile(qualifiedPath, overwrite,
+      OutputStream outputStream = abfsStore.createFile(qualifiedPath, statistics, overwrite,
           permission == null ? FsPermission.getFileDefault() : permission, FsPermission.getUMask(getConf()));
       return new FSDataOutputStream(outputStream, statistics);
     } catch(AzureBlobFileSystemException ex) {
@@ -250,7 +250,7 @@ public FSDataOutputStream append(final Path f, final int bufferSize, final Progr
     Path qualifiedPath = makeQualified(f);
 
     try {
-      OutputStream outputStream = abfsStore.openFileForWrite(qualifiedPath, false);
+      OutputStream outputStream = abfsStore.openFileForWrite(qualifiedPath, statistics, false);
       return new FSDataOutputStream(outputStream, statistics);
     } catch(AzureBlobFileSystemException ex) {
       checkException(f, ex);
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
index 9268cf8af8c9c..69ef940ade296 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
@@ -412,7 +412,7 @@ public void deleteFilesystem() throws AzureBlobFileSystemException {
     }
   }
 
-  public OutputStream createFile(final Path path, final boolean overwrite, final FsPermission permission,
+  public OutputStream createFile(final Path path, final FileSystem.Statistics statistics, final boolean overwrite, final FsPermission permission,
                                  final FsPermission umask) throws AzureBlobFileSystemException {
     try (AbfsPerfInfo perfInfo = startTracking("createFile", "createPath")) {
       boolean isNamespaceEnabled = getIsNamespaceEnabled();
@@ -436,6 +436,7 @@ public OutputStream createFile(final Path path, final boolean overwrite, final F
 
       return new AbfsOutputStream(
           client,
+          statistics,
           AbfsHttpConstants.FORWARD_SLASH + getRelativePath(path),
           0,
           abfsConfiguration.getWriteBufferSize(),
@@ -496,7 +497,7 @@ public AbfsInputStream openFileForRead(final Path path, final FileSystem.Statist
     }
   }
 
-  public OutputStream openFileForWrite(final Path path, final boolean overwrite) throws
+  public OutputStream openFileForWrite(final Path path, final FileSystem.Statistics statistics, final boolean overwrite) throws
           AzureBlobFileSystemException {
     try (AbfsPerfInfo perfInfo = startTracking("openFileForWrite", "getPathStatus")) {
       LOG.debug("openFileForWrite filesystem: {} path: {} overwrite: {}",
@@ -529,6 +530,7 @@ public OutputStream openFileForWrite(final Path path, final boolean overwrite) t
 
       return new AbfsOutputStream(
           client,
+          statistics,
           AbfsHttpConstants.FORWARD_SLASH + getRelativePath(path),
           offset,
           abfsConfiguration.getWriteBufferSize(),
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java
index 8dc3b8f07079d..77c92ca338e32 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java
@@ -101,6 +101,7 @@ public synchronized int read(final byte[] b, final int off, final int len) throw
     int currentLen = len;
     int lastReadBytes;
     int totalReadBytes = 0;
+    incrementReadOps();
     do {
       lastReadBytes = readOneBlock(b, currentOff, currentLen);
       if (lastReadBytes > 0) {
@@ -201,6 +202,7 @@ private int readInternal(final long position, final byte[] b, final int offset,
       // try reading from buffers first
       receivedBytes = ReadBufferManager.getBufferManager().getBlock(this, position, length, b);
       if (receivedBytes > 0) {
+        incrementReadOps();
         return receivedBytes;
       }
 
@@ -236,6 +238,7 @@ int readRemote(long position, byte[] b, int offset, int length) throws IOExcepti
     try (AbfsPerfInfo perfInfo = new AbfsPerfInfo(tracker, "readRemote", "read")) {
       op = client.read(path, position, b, offset, length, tolerateOobAppends ? "*" : eTag);
       perfInfo.registerResult(op.getResult()).registerSuccess(true);
+      incrementReadOps();
     } catch (AzureBlobFileSystemException ex) {
       if (ex instanceof AbfsRestOperationException) {
         AbfsRestOperationException ere = (AbfsRestOperationException) ex;
@@ -252,6 +255,13 @@ int readRemote(long position, byte[] b, int offset, int length) throws IOExcepti
     return (int) bytesRead;
   }
 
+  /**
+   * Increment Read Operations
+   */
+  public void incrementReadOps() {
+    statistics.incrementReadOps(1);
+  }
+
   /**
    * Seek to given position in stream.
    * @param n position to seek to
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsOutputStream.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsOutputStream.java
index 57e42e07b2907..27ec3260b0743 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsOutputStream.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsOutputStream.java
@@ -36,6 +36,7 @@
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
 
+import org.apache.hadoop.fs.FileSystem.Statistics;
 import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AbfsRestOperationException;
 import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AzureBlobFileSystemException;
 import org.apache.hadoop.io.ElasticByteBufferPool;
@@ -50,6 +51,7 @@
  */
 public class AbfsOutputStream extends OutputStream implements Syncable, StreamCapabilities {
   private final AbfsClient client;
+  private final Statistics statistics;
   private final String path;
   private long position;
   private boolean closed;
@@ -82,6 +84,7 @@ public class AbfsOutputStream extends OutputStream implements Syncable, StreamCa
 
   public AbfsOutputStream(
       final AbfsClient client,
+      final Statistics statistics,
       final String path,
       final long position,
       final int bufferSize,
@@ -90,6 +93,7 @@ public AbfsOutputStream(
       final boolean supportAppendWithFlush,
       final boolean appendBlob) {
     this.client = client;
+    this.statistics = statistics;
     this.path = path;
     this.position = position;
     this.closed = false;
@@ -187,6 +191,14 @@ public synchronized void write(final byte[] data, final int off, final int lengt
 
       writableBytes = bufferSize - bufferIndex;
     }
+    incrementWriteOps();
+  }
+
+  /**
+   * Increment Write Operations.
+   */
+  public void incrementWriteOps() {
+    statistics.incrementWriteOps(1);
   }
 
   /**
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsTestWithTimeout.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsTestWithTimeout.java
index fee90abeabc9e..ea27e88565059 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsTestWithTimeout.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsTestWithTimeout.java
@@ -17,12 +17,19 @@
  */
 package org.apache.hadoop.fs.azurebfs;
 
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.Path;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Rule;
 import org.junit.rules.TestName;
 import org.junit.rules.Timeout;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Arrays;
 
 import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.TEST_TIMEOUT;
 
@@ -31,6 +38,9 @@
  * This class does not attempt to bind to Azure.
  */
 public class AbstractAbfsTestWithTimeout extends Assert {
+  private static final Logger LOG =
+      LoggerFactory.getLogger(AbstractAbfsTestWithTimeout.class);
+
   /**
    * The name of the current method.
    */
@@ -67,4 +77,42 @@ public void nameThread() {
   protected int getTestTimeoutMillis() {
     return TEST_TIMEOUT;
   }
+
+  /**
+   * Describe a test in the logs.
+   *
+   * @param text text to print
+   * @param args arguments to format in the printing
+   */
+  protected void describe(String text, Object... args) {
+    LOG.info("\n\n{}: {}\n",
+        methodName.getMethodName(),
+        String.format(text, args));
+  }
+
+  /**
+   * Validate Contents written on a file in Abfs
+   *
+   * @param fs AzureBlobFileSystem
+   * @param path Path of the file
+   * @param originalByteArray original byte array
+   * @return
+   * @throws IOException
+   */
+  protected boolean validateContent(AzureBlobFileSystem fs, Path path,
+      byte[] originalByteArray)
+      throws IOException {
+    FSDataInputStream in = fs.open(path);
+    byte[] contentByteArray = new byte[originalByteArray.length];
+    int seekPos = 0;
+    while (in.read() != -1) {
+      in.seek(seekPos);
+      contentByteArray[seekPos] = (byte) (in.read());
+      seekPos++;
+    }
+
+    return Arrays.equals(contentByteArray, originalByteArray);
+
+  }
+
 }
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsInputStream.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsInputStream.java
new file mode 100644
index 0000000000000..d0cf3ef863c63
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsInputStream.java
@@ -0,0 +1,81 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.azurebfs.services.AbfsInputStream;
+
+/**
+ * Test Abfs Input Stream.
+ */
+
+public class ITestAbfsInputStream extends AbstractAbfsIntegrationTest {
+  public ITestAbfsInputStream() throws Exception {
+  }
+
+  /***
+   * {@link AbfsInputStream#incrementReadOps()}
+   *
+   * @throws Exception
+   */
+  @Test
+  public void testAbfsInputStreamReadOps() throws Exception {
+    describe("Test to see correct population of Read operations in Abfs");
+
+    final AzureBlobFileSystem fs = getFileSystem();
+    Path smallFile = new Path("testOneReadCall");
+    Path largeFile = new Path("testLargeReadCalls");
+    FileSystem.Statistics statistics = fs.getFsStatistics();
+    String testReadOps = "test this";
+    statistics.reset();
+
+    //Test for zero read operation
+    Assert.assertEquals(0, statistics.getReadOps());
+
+    FSDataOutputStream outForOneCall = fs.create(smallFile);
+    statistics.reset();
+    outForOneCall.write(testReadOps.getBytes());
+    FSDataInputStream inForOneCall = fs.open(smallFile);
+    inForOneCall.read(testReadOps.getBytes(), 0, testReadOps.getBytes().length);
+
+    //Test for one read operation
+    Assert.assertEquals(1, statistics.getReadOps());
+
+    FSDataOutputStream outForLargeCalls = fs.create(largeFile);
+    statistics.reset();
+    outForLargeCalls.write(testReadOps.getBytes());
+    FSDataInputStream inForLargeCalls = fs.open(largeFile);
+
+    for (int i = 0; i < 1000; i++)
+      inForLargeCalls
+          .read(testReadOps.getBytes(), 0, testReadOps.getBytes().length);
+
+    //Test for thousand read operations
+    Assert.assertEquals(1000, statistics.getReadOps());
+    statistics.reset();
+
+  }
+
+}
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCreate.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCreate.java
index 94368a4f36955..cf99c3023d061 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCreate.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCreate.java
@@ -23,6 +23,9 @@
 import java.io.IOException;
 import java.util.EnumSet;
 
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.azurebfs.services.AbfsOutputStream;
+import org.junit.Assert;
 import org.junit.Test;
 
 import org.apache.hadoop.fs.CreateFlag;
@@ -59,6 +62,59 @@ public void testEnsureFileCreatedImmediately() throws Exception {
     assertIsFile(fs, TEST_FILE_PATH);
   }
 
+  /**
+   * {@link AbfsOutputStream#incrementWriteOps()}
+   *
+   * @throws Exception
+   */
+  @Test
+  public void testWriteOpsMetric() throws Exception {
+    describe("Test to see correct population of write operations in Abfs");
+    final AzureBlobFileSystem fs = getFileSystem();
+    Path smallFile = new Path("testOneCall");
+    Path largeFile = new Path("testLargeCalls");
+    String testWriteOps = "test";
+    FileSystem.Statistics statistics = fs.getFsStatistics();
+    statistics.reset();
+
+    //Test for zero write operation
+    Assert.assertEquals(0, statistics.getWriteOps());
+
+    FSDataOutputStream outForOneCall = fs.create(smallFile);
+    statistics.reset();
+    //Writing "test" 1 time
+    outForOneCall
+        .write(testWriteOps.getBytes(), 0, testWriteOps.getBytes().length);
+
+    //Test for one write operation
+    Assert.assertEquals(1, statistics.getWriteOps());
+
+    outForOneCall.close();
+    //validating Content of file
+    Assert.assertEquals(true, validateContent(fs, smallFile,
+        testWriteOps.getBytes()));
+
+    String largeFileValidationString = "";
+    FSDataOutputStream outForLargeCalls = fs.create(largeFile);
+    statistics.reset();
+    //Writing "test" 1000 times
+    for (int i = 0; i < 1000; i++) {
+      outForLargeCalls.write(testWriteOps.getBytes(), 0,
+          testWriteOps.getBytes().length);
+
+      //Creating Validation string of "test" 1000 times
+      largeFileValidationString += testWriteOps;
+    }
+
+    //Test for thousand write operations
+    Assert.assertEquals(1000, statistics.getWriteOps());
+
+    outForLargeCalls.close();
+    //Validating if actually "test" is being written thousand times in largeFile
+    Assert.assertEquals(true, validateContent(fs, largeFile,
+        largeFileValidationString.getBytes()));
+  }
+
   @Test
   @SuppressWarnings("deprecation")
   public void testCreateNonRecursive() throws Exception {
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemOauth.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemOauth.java
index 533f47125654e..39a75958e784d 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemOauth.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemOauth.java
@@ -143,7 +143,7 @@ public void testBlobDataReader() throws Exception {
 
     // TEST WRITE FILE
     try {
-      abfsStore.openFileForWrite(EXISTED_FILE_PATH, true);
+      abfsStore.openFileForWrite(EXISTED_FILE_PATH, fs.getFsStatistics(),true);
     } catch (AbfsRestOperationException e) {
       assertEquals(AzureServiceErrorCode.AUTHORIZATION_PERMISSION_MISS_MATCH, e.getErrorCode());
     }

From ff01bcf889eccb566e069386065769a77a96b01e Mon Sep 17 00:00:00 2001
From: Mehakmeet Singh <mehakmeet.singh@cloudera.com>
Date: Wed, 11 Mar 2020 17:43:38 +0530
Subject: [PATCH 2/9] Fixing Review Comments

---
 .../fs/azurebfs/AzureBlobFileSystemStore.java |   6 +-
 .../fs/azurebfs/services/AbfsInputStream.java |   2 +-
 .../azurebfs/AbstractAbfsTestWithTimeout.java |  34 +++---
 .../fs/azurebfs/ITestAbfsInputStream.java     |  81 -------------
 .../azurebfs/ITestAbfsStreamStatistics.java   | 109 ++++++++++++++++++
 .../ITestAzureBlobFileSystemCreate.java       |  56 ---------
 .../ITestAzureBlobFileSystemOauth.java        |   2 +-
 7 files changed, 134 insertions(+), 156 deletions(-)
 delete mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsInputStream.java
 create mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsStreamStatistics.java

diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
index 69ef940ade296..73fcf2f67584b 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java
@@ -412,8 +412,10 @@ public void deleteFilesystem() throws AzureBlobFileSystemException {
     }
   }
 
-  public OutputStream createFile(final Path path, final FileSystem.Statistics statistics, final boolean overwrite, final FsPermission permission,
-                                 final FsPermission umask) throws AzureBlobFileSystemException {
+  public OutputStream createFile(final Path path,
+      final FileSystem.Statistics statistics,
+      final boolean overwrite, final FsPermission permission,
+      final FsPermission umask) throws AzureBlobFileSystemException {
     try (AbfsPerfInfo perfInfo = startTracking("createFile", "createPath")) {
       boolean isNamespaceEnabled = getIsNamespaceEnabled();
       LOG.debug("createFile filesystem: {} path: {} overwrite: {} permission: {} umask: {} isNamespaceEnabled: {}",
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java
index 77c92ca338e32..ba662e96b9470 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java
@@ -256,7 +256,7 @@ int readRemote(long position, byte[] b, int offset, int length) throws IOExcepti
   }
 
   /**
-   * Increment Read Operations
+   * Increment Read Operations.
    */
   public void incrementReadOps() {
     statistics.incrementReadOps(1);
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsTestWithTimeout.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsTestWithTimeout.java
index ea27e88565059..8ddaca1440b05 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsTestWithTimeout.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsTestWithTimeout.java
@@ -17,8 +17,8 @@
  */
 package org.apache.hadoop.fs.azurebfs;
 
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.Path;
+import java.io.IOException;
+
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.BeforeClass;
@@ -28,8 +28,8 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.io.IOException;
-import java.util.Arrays;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.Path;
 
 import static org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys.TEST_TIMEOUT;
 
@@ -91,10 +91,10 @@ protected void describe(String text, Object... args) {
   }
 
   /**
-   * Validate Contents written on a file in Abfs
+   * Validate Contents written on a file in Abfs.
    *
-   * @param fs AzureBlobFileSystem
-   * @param path Path of the file
+   * @param fs                AzureBlobFileSystem
+   * @param path              Path of the file
    * @param originalByteArray original byte array
    * @return
    * @throws IOException
@@ -103,16 +103,20 @@ protected boolean validateContent(AzureBlobFileSystem fs, Path path,
       byte[] originalByteArray)
       throws IOException {
     FSDataInputStream in = fs.open(path);
-    byte[] contentByteArray = new byte[originalByteArray.length];
-    int seekPos = 0;
-    while (in.read() != -1) {
-      in.seek(seekPos);
-      contentByteArray[seekPos] = (byte) (in.read());
-      seekPos++;
-    }
 
-    return Arrays.equals(contentByteArray, originalByteArray);
+    int pos = 0;
+    int lenOfOriginalByteArray = originalByteArray.length;
+    byte valueOfContentAtPos = (byte) in.read();
 
+    while (valueOfContentAtPos != -1 && pos < lenOfOriginalByteArray) {
+      if (originalByteArray[pos] != valueOfContentAtPos)
+        return false;
+      valueOfContentAtPos = (byte) in.read();
+      pos++;
+    }
+    if (valueOfContentAtPos != -1)
+      return false;
+    return true;
   }
 
 }
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsInputStream.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsInputStream.java
deleted file mode 100644
index d0cf3ef863c63..0000000000000
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsInputStream.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azurebfs;
-
-import org.junit.Assert;
-import org.junit.Test;
-
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.azurebfs.services.AbfsInputStream;
-
-/**
- * Test Abfs Input Stream.
- */
-
-public class ITestAbfsInputStream extends AbstractAbfsIntegrationTest {
-  public ITestAbfsInputStream() throws Exception {
-  }
-
-  /***
-   * {@link AbfsInputStream#incrementReadOps()}
-   *
-   * @throws Exception
-   */
-  @Test
-  public void testAbfsInputStreamReadOps() throws Exception {
-    describe("Test to see correct population of Read operations in Abfs");
-
-    final AzureBlobFileSystem fs = getFileSystem();
-    Path smallFile = new Path("testOneReadCall");
-    Path largeFile = new Path("testLargeReadCalls");
-    FileSystem.Statistics statistics = fs.getFsStatistics();
-    String testReadOps = "test this";
-    statistics.reset();
-
-    //Test for zero read operation
-    Assert.assertEquals(0, statistics.getReadOps());
-
-    FSDataOutputStream outForOneCall = fs.create(smallFile);
-    statistics.reset();
-    outForOneCall.write(testReadOps.getBytes());
-    FSDataInputStream inForOneCall = fs.open(smallFile);
-    inForOneCall.read(testReadOps.getBytes(), 0, testReadOps.getBytes().length);
-
-    //Test for one read operation
-    Assert.assertEquals(1, statistics.getReadOps());
-
-    FSDataOutputStream outForLargeCalls = fs.create(largeFile);
-    statistics.reset();
-    outForLargeCalls.write(testReadOps.getBytes());
-    FSDataInputStream inForLargeCalls = fs.open(largeFile);
-
-    for (int i = 0; i < 1000; i++)
-      inForLargeCalls
-          .read(testReadOps.getBytes(), 0, testReadOps.getBytes().length);
-
-    //Test for thousand read operations
-    Assert.assertEquals(1000, statistics.getReadOps());
-    statistics.reset();
-
-  }
-
-}
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsStreamStatistics.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsStreamStatistics.java
new file mode 100644
index 0000000000000..a4370f0420567
--- /dev/null
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsStreamStatistics.java
@@ -0,0 +1,109 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.azurebfs;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.azurebfs.services.AbfsInputStream;
+
+/**
+ * Test Abfs Stream.
+ */
+
+public class ITestAbfsStreamStatistics extends AbstractAbfsIntegrationTest {
+  public ITestAbfsStreamStatistics() throws Exception {
+  }
+
+  /***
+   * {@link AbfsInputStream#incrementReadOps()}.
+   *
+   * @throws Exception
+   */
+  @Test
+  public void testAbfsStreamOps() throws Exception {
+    describe("Test to see correct population of read and write operations in "
+        + "Abfs");
+
+    final AzureBlobFileSystem fs = getFileSystem();
+    Path smallOperaionsFile = new Path("testOneReadWriteOps");
+    Path largeOperationsFile = new Path("testLargeReadWriteOps");
+    FileSystem.Statistics statistics = fs.getFsStatistics();
+    String testReadWriteOps = "test this";
+    statistics.reset();
+
+    //Test for zero read and write operation
+    Assert.assertEquals("Zero read operations", 0, statistics.getReadOps());
+    Assert.assertEquals("Zero write operations", 0, statistics.getWriteOps());
+
+    FSDataOutputStream outForOneOperation = fs.create(smallOperaionsFile);
+    statistics.reset();
+    outForOneOperation.write(testReadWriteOps.getBytes());
+    FSDataInputStream inForOneCall = fs.open(smallOperaionsFile);
+    inForOneCall.read(testReadWriteOps.getBytes(), 0,
+        testReadWriteOps.getBytes().length);
+
+    //Test for one read and write operation
+    Assert.assertEquals("one read operation is performed", 1,
+        statistics.getReadOps());
+    Assert.assertEquals("one write operation is performed", 1,
+        statistics.getWriteOps());
+
+    outForOneOperation.close();
+    //validating Content of file
+    Assert.assertEquals("one operation Content validation", true,
+        validateContent(fs, smallOperaionsFile,
+            testReadWriteOps.getBytes()));
+
+    FSDataOutputStream outForLargeOperations = fs.create(largeOperationsFile);
+    statistics.reset();
+
+    StringBuilder largeOperationsValidationString = new StringBuilder();
+    for (int i = 0; i < 1000000; i++) {
+      outForLargeOperations.write(testReadWriteOps.getBytes());
+
+      //Creating the String for content Validation
+      largeOperationsValidationString.append(testReadWriteOps);
+    }
+
+    FSDataInputStream inForLargeCalls = fs.open(largeOperationsFile);
+
+    for (int i = 0; i < 1000000; i++)
+      inForLargeCalls
+          .read(testReadWriteOps.getBytes(), 0,
+              testReadWriteOps.getBytes().length);
+
+    //Test for one million read and write operations
+    Assert.assertEquals("Large read operations", 1000000,
+        statistics.getReadOps());
+    Assert.assertEquals("Large write operations", 1000000,
+        statistics.getWriteOps());
+
+    outForLargeOperations.close();
+    //Validating if actually "test" is being written million times in largeOperationsFile
+    Assert.assertEquals("Large File content validation", true,
+        validateContent(fs, largeOperationsFile,
+            largeOperationsValidationString.toString().getBytes()));
+
+  }
+}
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCreate.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCreate.java
index cf99c3023d061..94368a4f36955 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCreate.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCreate.java
@@ -23,9 +23,6 @@
 import java.io.IOException;
 import java.util.EnumSet;
 
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.azurebfs.services.AbfsOutputStream;
-import org.junit.Assert;
 import org.junit.Test;
 
 import org.apache.hadoop.fs.CreateFlag;
@@ -62,59 +59,6 @@ public void testEnsureFileCreatedImmediately() throws Exception {
     assertIsFile(fs, TEST_FILE_PATH);
   }
 
-  /**
-   * {@link AbfsOutputStream#incrementWriteOps()}
-   *
-   * @throws Exception
-   */
-  @Test
-  public void testWriteOpsMetric() throws Exception {
-    describe("Test to see correct population of write operations in Abfs");
-    final AzureBlobFileSystem fs = getFileSystem();
-    Path smallFile = new Path("testOneCall");
-    Path largeFile = new Path("testLargeCalls");
-    String testWriteOps = "test";
-    FileSystem.Statistics statistics = fs.getFsStatistics();
-    statistics.reset();
-
-    //Test for zero write operation
-    Assert.assertEquals(0, statistics.getWriteOps());
-
-    FSDataOutputStream outForOneCall = fs.create(smallFile);
-    statistics.reset();
-    //Writing "test" 1 time
-    outForOneCall
-        .write(testWriteOps.getBytes(), 0, testWriteOps.getBytes().length);
-
-    //Test for one write operation
-    Assert.assertEquals(1, statistics.getWriteOps());
-
-    outForOneCall.close();
-    //validating Content of file
-    Assert.assertEquals(true, validateContent(fs, smallFile,
-        testWriteOps.getBytes()));
-
-    String largeFileValidationString = "";
-    FSDataOutputStream outForLargeCalls = fs.create(largeFile);
-    statistics.reset();
-    //Writing "test" 1000 times
-    for (int i = 0; i < 1000; i++) {
-      outForLargeCalls.write(testWriteOps.getBytes(), 0,
-          testWriteOps.getBytes().length);
-
-      //Creating Validation string of "test" 1000 times
-      largeFileValidationString += testWriteOps;
-    }
-
-    //Test for thousand write operations
-    Assert.assertEquals(1000, statistics.getWriteOps());
-
-    outForLargeCalls.close();
-    //Validating if actually "test" is being written thousand times in largeFile
-    Assert.assertEquals(true, validateContent(fs, largeFile,
-        largeFileValidationString.getBytes()));
-  }
-
   @Test
   @SuppressWarnings("deprecation")
   public void testCreateNonRecursive() throws Exception {
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemOauth.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemOauth.java
index 39a75958e784d..5016609676d70 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemOauth.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemOauth.java
@@ -143,7 +143,7 @@ public void testBlobDataReader() throws Exception {
 
     // TEST WRITE FILE
     try {
-      abfsStore.openFileForWrite(EXISTED_FILE_PATH, fs.getFsStatistics(),true);
+      abfsStore.openFileForWrite(EXISTED_FILE_PATH, fs.getFsStatistics(), true);
     } catch (AbfsRestOperationException e) {
       assertEquals(AzureServiceErrorCode.AUTHORIZATION_PERMISSION_MISS_MATCH, e.getErrorCode());
     }

From 3e59ca51ef546aa303a7c50d092cffaa17ff664d Mon Sep 17 00:00:00 2001
From: Mehakmeet Singh <mehakmeet.singh@cloudera.com>
Date: Thu, 12 Mar 2020 22:50:41 +0530
Subject: [PATCH 3/9] Meaningful assert messages

---
 .../azurebfs/ITestAbfsStreamStatistics.java   | 20 ++++++++++---------
 1 file changed, 11 insertions(+), 9 deletions(-)

diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsStreamStatistics.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsStreamStatistics.java
index a4370f0420567..f80e708376344 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsStreamStatistics.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsStreamStatistics.java
@@ -53,8 +53,10 @@ public void testAbfsStreamOps() throws Exception {
     statistics.reset();
 
     //Test for zero read and write operation
-    Assert.assertEquals("Zero read operations", 0, statistics.getReadOps());
-    Assert.assertEquals("Zero write operations", 0, statistics.getWriteOps());
+    Assert.assertEquals("Mismatch in read operations", 0,
+        statistics.getReadOps());
+    Assert.assertEquals("Mismatch in write operations", 0,
+        statistics.getWriteOps());
 
     FSDataOutputStream outForOneOperation = fs.create(smallOperaionsFile);
     statistics.reset();
@@ -64,14 +66,14 @@ public void testAbfsStreamOps() throws Exception {
         testReadWriteOps.getBytes().length);
 
     //Test for one read and write operation
-    Assert.assertEquals("one read operation is performed", 1,
+    Assert.assertEquals("Mismatch in read operations", 1,
         statistics.getReadOps());
-    Assert.assertEquals("one write operation is performed", 1,
+    Assert.assertEquals("Mismatch in write operations", 1,
         statistics.getWriteOps());
 
     outForOneOperation.close();
-    //validating Content of file
-    Assert.assertEquals("one operation Content validation", true,
+    //Validating if Content is being written in the smallFile
+    Assert.assertEquals("Mismatch in content validation", true,
         validateContent(fs, smallOperaionsFile,
             testReadWriteOps.getBytes()));
 
@@ -94,14 +96,14 @@ public void testAbfsStreamOps() throws Exception {
               testReadWriteOps.getBytes().length);
 
     //Test for one million read and write operations
-    Assert.assertEquals("Large read operations", 1000000,
+    Assert.assertEquals("Mismatch in read operations", 1000000,
         statistics.getReadOps());
-    Assert.assertEquals("Large write operations", 1000000,
+    Assert.assertEquals("Mismatch in write operations", 1000000,
         statistics.getWriteOps());
 
     outForLargeOperations.close();
     //Validating if actually "test" is being written million times in largeOperationsFile
-    Assert.assertEquals("Large File content validation", true,
+    Assert.assertEquals("Mismatch in content validation", true,
         validateContent(fs, largeOperationsFile,
             largeOperationsValidationString.toString().getBytes()));
 

From ecc4c80fda62d497a4ff4df8de1515d30a255652 Mon Sep 17 00:00:00 2001
From: Mehakmeet Singh <mehakmeet.singh@cloudera.com>
Date: Thu, 12 Mar 2020 23:21:00 +0530
Subject: [PATCH 4/9] minor nits in test

---
 .../fs/azurebfs/ITestAbfsStreamStatistics.java       | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsStreamStatistics.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsStreamStatistics.java
index f80e708376344..c88fe143d7b99 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsStreamStatistics.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsStreamStatistics.java
@@ -46,7 +46,7 @@ public void testAbfsStreamOps() throws Exception {
         + "Abfs");
 
     final AzureBlobFileSystem fs = getFileSystem();
-    Path smallOperaionsFile = new Path("testOneReadWriteOps");
+    Path smallOperationsFile = new Path("testOneReadWriteOps");
     Path largeOperationsFile = new Path("testLargeReadWriteOps");
     FileSystem.Statistics statistics = fs.getFsStatistics();
     String testReadWriteOps = "test this";
@@ -58,10 +58,10 @@ public void testAbfsStreamOps() throws Exception {
     Assert.assertEquals("Mismatch in write operations", 0,
         statistics.getWriteOps());
 
-    FSDataOutputStream outForOneOperation = fs.create(smallOperaionsFile);
+    FSDataOutputStream outForOneOperation = fs.create(smallOperationsFile);
     statistics.reset();
     outForOneOperation.write(testReadWriteOps.getBytes());
-    FSDataInputStream inForOneCall = fs.open(smallOperaionsFile);
+    FSDataInputStream inForOneCall = fs.open(smallOperationsFile);
     inForOneCall.read(testReadWriteOps.getBytes(), 0,
         testReadWriteOps.getBytes().length);
 
@@ -72,9 +72,9 @@ public void testAbfsStreamOps() throws Exception {
         statistics.getWriteOps());
 
     outForOneOperation.close();
-    //Validating if Content is being written in the smallFile
+    //Validating if content is being written in the smallOperationsFile
     Assert.assertEquals("Mismatch in content validation", true,
-        validateContent(fs, smallOperaionsFile,
+        validateContent(fs, smallOperationsFile,
             testReadWriteOps.getBytes()));
 
     FSDataOutputStream outForLargeOperations = fs.create(largeOperationsFile);
@@ -102,7 +102,7 @@ public void testAbfsStreamOps() throws Exception {
         statistics.getWriteOps());
 
     outForLargeOperations.close();
-    //Validating if actually "test" is being written million times in largeOperationsFile
+    //Validating if content is being written in largeOperationsFile
     Assert.assertEquals("Mismatch in content validation", true,
         validateContent(fs, largeOperationsFile,
             largeOperationsValidationString.toString().getBytes()));

From 536d1e11cc1d01084d510380774d045e717cee09 Mon Sep 17 00:00:00 2001
From: Mehakmeet Singh <mehakmeet.singh@cloudera.com>
Date: Thu, 12 Mar 2020 23:21:00 +0530
Subject: [PATCH 5/9] minor nits in test

---
 .../fs/azurebfs/services/AbfsInputStream.java |  6 +-
 .../azurebfs/services/AbfsOutputStream.java   |  6 +-
 .../azurebfs/AbstractAbfsTestWithTimeout.java |  8 +--
 .../azurebfs/ITestAbfsStreamStatistics.java   | 69 ++++++++++++-------
 4 files changed, 57 insertions(+), 32 deletions(-)

diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java
index ba662e96b9470..0c06014a40832 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java
@@ -258,8 +258,10 @@ int readRemote(long position, byte[] b, int offset, int length) throws IOExcepti
   /**
    * Increment Read Operations.
    */
-  public void incrementReadOps() {
-    statistics.incrementReadOps(1);
+  private void incrementReadOps() {
+    if (statistics != null) {
+      statistics.incrementReadOps(1);
+    }
   }
 
   /**
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsOutputStream.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsOutputStream.java
index 27ec3260b0743..8fb76890596e7 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsOutputStream.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsOutputStream.java
@@ -197,8 +197,10 @@ public synchronized void write(final byte[] data, final int off, final int lengt
   /**
    * Increment Write Operations.
    */
-  public void incrementWriteOps() {
-    statistics.incrementWriteOps(1);
+  private void incrementWriteOps() {
+    if (statistics != null) {
+      statistics.incrementWriteOps(1);
+    }
   }
 
   /**
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsTestWithTimeout.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsTestWithTimeout.java
index 8ddaca1440b05..854065103b182 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsTestWithTimeout.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsTestWithTimeout.java
@@ -6,9 +6,9 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -96,7 +96,7 @@ protected void describe(String text, Object... args) {
    * @param fs                AzureBlobFileSystem
    * @param path              Path of the file
    * @param originalByteArray original byte array
-   * @return
+   * @return if content is validated true else, false
    * @throws IOException
    */
   protected boolean validateContent(AzureBlobFileSystem fs, Path path,
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsStreamStatistics.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsStreamStatistics.java
index c88fe143d7b99..bb9cbd17d9ece 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsStreamStatistics.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsStreamStatistics.java
@@ -25,7 +25,6 @@
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.azurebfs.services.AbfsInputStream;
 
 /**
  * Test Abfs Stream.
@@ -36,7 +35,8 @@ public ITestAbfsStreamStatistics() throws Exception {
   }
 
   /***
-   * {@link AbfsInputStream#incrementReadOps()}.
+   * Testing {@code incrementReadOps()} in class {@code AbfsInputStream} and
+   * {@code incrementWriteOps()} in class {@code AbfsOutputStream}.
    *
    * @throws Exception
    */
@@ -77,35 +77,56 @@ public void testAbfsStreamOps() throws Exception {
         validateContent(fs, smallOperationsFile,
             testReadWriteOps.getBytes()));
 
-    FSDataOutputStream outForLargeOperations = fs.create(largeOperationsFile);
-    statistics.reset();
-
+    FSDataOutputStream outForLargeOperations = null;
+    FSDataInputStream inForLargeOperations = null;
     StringBuilder largeOperationsValidationString = new StringBuilder();
-    for (int i = 0; i < 1000000; i++) {
-      outForLargeOperations.write(testReadWriteOps.getBytes());
-
-      //Creating the String for content Validation
-      largeOperationsValidationString.append(testReadWriteOps);
+    try {
+      outForLargeOperations = fs.create(largeOperationsFile);
+      statistics.reset();
+      int largeValue = 1000000;
+      for (int i = 0; i < largeValue; i++) {
+        outForLargeOperations.write(testReadWriteOps.getBytes());
+
+        //Creating the String for content Validation
+        largeOperationsValidationString.append(testReadWriteOps);
+      }
+
+      inForLargeOperations = fs.open(largeOperationsFile);
+      for (int i = 0; i < largeValue; i++)
+        inForLargeOperations
+            .read(testReadWriteOps.getBytes(), 0,
+                testReadWriteOps.getBytes().length);
+
+      //Test for one million read and write operations
+      assertReadWriteOps(largeValue, statistics);
+    } finally {
+      if (inForLargeOperations != null) {
+        inForLargeOperations.close();
+      }
+      if (outForLargeOperations != null) {
+        outForLargeOperations.close();
+      }
     }
 
-    FSDataInputStream inForLargeCalls = fs.open(largeOperationsFile);
+    //Validating if content is being written in largeOperationsFile
+    Assert.assertTrue("Mismatch in content validation",
+        validateContent(fs, largeOperationsFile,
+            largeOperationsValidationString.toString().getBytes()));
 
-    for (int i = 0; i < 1000000; i++)
-      inForLargeCalls
-          .read(testReadWriteOps.getBytes(), 0,
-              testReadWriteOps.getBytes().length);
+  }
 
-    //Test for one million read and write operations
-    Assert.assertEquals("Mismatch in read operations", 1000000,
+  /**
+   * Method for Read and Write Ops Assertion.
+   *
+   * @param expectedReadWriteOps Expected Value
+   * @param statistics           fs stats to get Actual Values
+   */
+  private void assertReadWriteOps(long expectedReadWriteOps,
+      FileSystem.Statistics statistics) {
+    Assert.assertEquals("Mismatch in read operations", expectedReadWriteOps,
         statistics.getReadOps());
-    Assert.assertEquals("Mismatch in write operations", 1000000,
+    Assert.assertEquals("Mismatch in write operations", expectedReadWriteOps,
         statistics.getWriteOps());
 
-    outForLargeOperations.close();
-    //Validating if content is being written in largeOperationsFile
-    Assert.assertEquals("Mismatch in content validation", true,
-        validateContent(fs, largeOperationsFile,
-            largeOperationsValidationString.toString().getBytes()));
-
   }
 }

From 0e33beaf5d7c3fb9524c2c4ad8c8343d62351a72 Mon Sep 17 00:00:00 2001
From: Mehakmeet Singh <mehakmeet.singh@cloudera.com>
Date: Wed, 18 Mar 2020 13:10:09 +0530
Subject: [PATCH 6/9] Fixing review comments

---
 .../azurebfs/AbstractAbfsTestWithTimeout.java |  6 +-
 .../azurebfs/ITestAbfsStreamStatistics.java   | 82 +++++++++++--------
 2 files changed, 52 insertions(+), 36 deletions(-)

diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsTestWithTimeout.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsTestWithTimeout.java
index 854065103b182..f6805fb88c4b5 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsTestWithTimeout.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsTestWithTimeout.java
@@ -6,9 +6,9 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsStreamStatistics.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsStreamStatistics.java
index bb9cbd17d9ece..4177de963aa56 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsStreamStatistics.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsStreamStatistics.java
@@ -6,9 +6,9 @@
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -52,26 +52,38 @@ public void testAbfsStreamOps() throws Exception {
     String testReadWriteOps = "test this";
     statistics.reset();
 
-    //Test for zero read and write operation
-    Assert.assertEquals("Mismatch in read operations", 0,
-        statistics.getReadOps());
-    Assert.assertEquals("Mismatch in write operations", 0,
-        statistics.getWriteOps());
+    //Test for zero write operation
+    assertReadWriteOps("write", 0, statistics.getWriteOps());
+
+    //Test for zero read operation
+    assertReadWriteOps("read", 0, statistics.getReadOps());
+
+    FSDataOutputStream outForOneOperation = null;
+    FSDataInputStream inForOneOperation = null;
+    try {
+      outForOneOperation = fs.create(smallOperationsFile);
+      statistics.reset();
+      outForOneOperation.write(testReadWriteOps.getBytes());
+
+      //Test for a single write operation
+      assertReadWriteOps("write", 1, statistics.getWriteOps());
+
+      inForOneOperation = fs.open(smallOperationsFile);
+      inForOneOperation.read(testReadWriteOps.getBytes(), 0,
+          testReadWriteOps.getBytes().length);
+
+      //Test for a single read operation
+      assertReadWriteOps("read", 1, statistics.getReadOps());
+
+    } finally {
+      if (inForOneOperation != null) {
+        inForOneOperation.close();
+      }
+      if (outForOneOperation != null) {
+        outForOneOperation.close();
+      }
+    }
 
-    FSDataOutputStream outForOneOperation = fs.create(smallOperationsFile);
-    statistics.reset();
-    outForOneOperation.write(testReadWriteOps.getBytes());
-    FSDataInputStream inForOneCall = fs.open(smallOperationsFile);
-    inForOneCall.read(testReadWriteOps.getBytes(), 0,
-        testReadWriteOps.getBytes().length);
-
-    //Test for one read and write operation
-    Assert.assertEquals("Mismatch in read operations", 1,
-        statistics.getReadOps());
-    Assert.assertEquals("Mismatch in write operations", 1,
-        statistics.getWriteOps());
-
-    outForOneOperation.close();
     //Validating if content is being written in the smallOperationsFile
     Assert.assertEquals("Mismatch in content validation", true,
         validateContent(fs, smallOperationsFile,
@@ -91,14 +103,18 @@ public void testAbfsStreamOps() throws Exception {
         largeOperationsValidationString.append(testReadWriteOps);
       }
 
+      //Test for 1000000 write operations
+      assertReadWriteOps("write", largeValue, statistics.getWriteOps());
+
       inForLargeOperations = fs.open(largeOperationsFile);
       for (int i = 0; i < largeValue; i++)
         inForLargeOperations
             .read(testReadWriteOps.getBytes(), 0,
                 testReadWriteOps.getBytes().length);
 
-      //Test for one million read and write operations
-      assertReadWriteOps(largeValue, statistics);
+      //Test for 1000000 read operations
+      assertReadWriteOps("read", largeValue, statistics.getReadOps());
+
     } finally {
       if (inForLargeOperations != null) {
         inForLargeOperations.close();
@@ -116,17 +132,17 @@ public void testAbfsStreamOps() throws Exception {
   }
 
   /**
-   * Method for Read and Write Ops Assertion.
+   * Generic method to assert both Read an write operations.
    *
-   * @param expectedReadWriteOps Expected Value
-   * @param statistics           fs stats to get Actual Values
+   * @param operation     what operation is being asserted
+   * @param expectedValue value which is expected
+   * @param actualValue   value which is actual
    */
-  private void assertReadWriteOps(long expectedReadWriteOps,
-      FileSystem.Statistics statistics) {
-    Assert.assertEquals("Mismatch in read operations", expectedReadWriteOps,
-        statistics.getReadOps());
-    Assert.assertEquals("Mismatch in write operations", expectedReadWriteOps,
-        statistics.getWriteOps());
 
+  private void assertReadWriteOps(String operation, long expectedValue,
+      long actualValue) {
+    Assert.assertEquals("Mismatch in " + operation + " operations",
+        expectedValue
+        , actualValue);
   }
 }

From 72f7181f03ba835ac16cace1d096c857f3967ab4 Mon Sep 17 00:00:00 2001
From: Mehakmeet Singh <mehakmeet.singh@cloudera.com>
Date: Wed, 18 Mar 2020 17:50:39 +0530
Subject: [PATCH 7/9] closing streams

---
 .../azurebfs/AbstractAbfsTestWithTimeout.java | 24 ++++++++-------
 .../azurebfs/ITestAbfsStreamStatistics.java   | 30 ++++++++-----------
 2 files changed, 27 insertions(+), 27 deletions(-)

diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsTestWithTimeout.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsTestWithTimeout.java
index f6805fb88c4b5..71efff97de10d 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsTestWithTimeout.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsTestWithTimeout.java
@@ -102,21 +102,25 @@ protected void describe(String text, Object... args) {
   protected boolean validateContent(AzureBlobFileSystem fs, Path path,
       byte[] originalByteArray)
       throws IOException {
-    FSDataInputStream in = fs.open(path);
-
     int pos = 0;
     int lenOfOriginalByteArray = originalByteArray.length;
-    byte valueOfContentAtPos = (byte) in.read();
 
-    while (valueOfContentAtPos != -1 && pos < lenOfOriginalByteArray) {
-      if (originalByteArray[pos] != valueOfContentAtPos)
+    try (FSDataInputStream in = fs.open(path)) {
+      byte valueOfContentAtPos = (byte) in.read();
+
+      while (valueOfContentAtPos != -1 && pos < lenOfOriginalByteArray) {
+        if (originalByteArray[pos] != valueOfContentAtPos) {
+          return false;
+        }
+        valueOfContentAtPos = (byte) in.read();
+        pos++;
+      }
+      if (valueOfContentAtPos != -1) {
         return false;
-      valueOfContentAtPos = (byte) in.read();
-      pos++;
+      }
+      return true;
     }
-    if (valueOfContentAtPos != -1)
-      return false;
-    return true;
+
   }
 
 }
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsStreamStatistics.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsStreamStatistics.java
index 4177de963aa56..d5314ed8ee0ef 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsStreamStatistics.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsStreamStatistics.java
@@ -25,6 +25,7 @@
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IOUtils;
 
 /**
  * Test Abfs Stream.
@@ -34,6 +35,8 @@ public class ITestAbfsStreamStatistics extends AbstractAbfsIntegrationTest {
   public ITestAbfsStreamStatistics() throws Exception {
   }
 
+  private static int LARGE_NUMBER_OF_OPS = 1000000;
+
   /***
    * Testing {@code incrementReadOps()} in class {@code AbfsInputStream} and
    * {@code incrementWriteOps()} in class {@code AbfsOutputStream}.
@@ -76,12 +79,8 @@ public void testAbfsStreamOps() throws Exception {
       assertReadWriteOps("read", 1, statistics.getReadOps());
 
     } finally {
-      if (inForOneOperation != null) {
-        inForOneOperation.close();
-      }
-      if (outForOneOperation != null) {
-        outForOneOperation.close();
-      }
+      IOUtils.cleanupWithLogger(null, inForOneOperation,
+          outForOneOperation);
     }
 
     //Validating if content is being written in the smallOperationsFile
@@ -95,7 +94,7 @@ public void testAbfsStreamOps() throws Exception {
     try {
       outForLargeOperations = fs.create(largeOperationsFile);
       statistics.reset();
-      int largeValue = 1000000;
+      int largeValue = LARGE_NUMBER_OF_OPS;
       for (int i = 0; i < largeValue; i++) {
         outForLargeOperations.write(testReadWriteOps.getBytes());
 
@@ -107,21 +106,18 @@ public void testAbfsStreamOps() throws Exception {
       assertReadWriteOps("write", largeValue, statistics.getWriteOps());
 
       inForLargeOperations = fs.open(largeOperationsFile);
-      for (int i = 0; i < largeValue; i++)
+      for (int i = 0; i < largeValue; i++) {
         inForLargeOperations
             .read(testReadWriteOps.getBytes(), 0,
                 testReadWriteOps.getBytes().length);
+      }
 
       //Test for 1000000 read operations
       assertReadWriteOps("read", largeValue, statistics.getReadOps());
 
     } finally {
-      if (inForLargeOperations != null) {
-        inForLargeOperations.close();
-      }
-      if (outForLargeOperations != null) {
-        outForLargeOperations.close();
-      }
+      IOUtils.cleanupWithLogger(null, inForLargeOperations,
+          outForLargeOperations);
     }
 
     //Validating if content is being written in largeOperationsFile
@@ -141,8 +137,8 @@ public void testAbfsStreamOps() throws Exception {
 
   private void assertReadWriteOps(String operation, long expectedValue,
       long actualValue) {
-    Assert.assertEquals("Mismatch in " + operation + " operations",
-        expectedValue
-        , actualValue);
+    Assert
+        .assertEquals("Mismatch in " + operation + " operations", expectedValue,
+            actualValue);
   }
 }

From 6b28cf2a041adae52f38dc080616566506a52120 Mon Sep 17 00:00:00 2001
From: Mehakmeet Singh <mehakmeet.singh@cloudera.com>
Date: Mon, 23 Mar 2020 08:55:38 +0530
Subject: [PATCH 8/9] Logging, closing, modifying

---
 .../azurebfs/services/AbfsOutputStream.java   |  5 ++-
 .../azurebfs/AbstractAbfsTestWithTimeout.java |  3 ++
 .../azurebfs/ITestAbfsStreamStatistics.java   | 38 +++++++++++++------
 .../ITestAzureBlobFileSystemOauth.java        |  4 +-
 4 files changed, 35 insertions(+), 15 deletions(-)

diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsOutputStream.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsOutputStream.java
index 8fb76890596e7..da9b7c6791598 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsOutputStream.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsOutputStream.java
@@ -36,10 +36,10 @@
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
 
-import org.apache.hadoop.fs.FileSystem.Statistics;
 import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AbfsRestOperationException;
 import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AzureBlobFileSystemException;
 import org.apache.hadoop.io.ElasticByteBufferPool;
+import org.apache.hadoop.fs.FileSystem.Statistics;
 import org.apache.hadoop.fs.FSExceptionMessages;
 import org.apache.hadoop.fs.StreamCapabilities;
 import org.apache.hadoop.fs.Syncable;
@@ -51,7 +51,6 @@
  */
 public class AbfsOutputStream extends OutputStream implements Syncable, StreamCapabilities {
   private final AbfsClient client;
-  private final Statistics statistics;
   private final String path;
   private long position;
   private boolean closed;
@@ -82,6 +81,8 @@ public class AbfsOutputStream extends OutputStream implements Syncable, StreamCa
   private final ElasticByteBufferPool byteBufferPool
           = new ElasticByteBufferPool();
 
+  private final Statistics statistics;
+
   public AbfsOutputStream(
       final AbfsClient client,
       final Statistics statistics,
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsTestWithTimeout.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsTestWithTimeout.java
index 71efff97de10d..0485422871ecc 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsTestWithTimeout.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/AbstractAbfsTestWithTimeout.java
@@ -110,12 +110,15 @@ protected boolean validateContent(AzureBlobFileSystem fs, Path path,
 
       while (valueOfContentAtPos != -1 && pos < lenOfOriginalByteArray) {
         if (originalByteArray[pos] != valueOfContentAtPos) {
+          assertEquals("Mismatch in content validation at position {}", pos,
+              originalByteArray[pos], valueOfContentAtPos);
           return false;
         }
         valueOfContentAtPos = (byte) in.read();
         pos++;
       }
       if (valueOfContentAtPos != -1) {
+        assertEquals("Expected end of file", -1, valueOfContentAtPos);
         return false;
       }
       return true;
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsStreamStatistics.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsStreamStatistics.java
index d5314ed8ee0ef..554400d26a3f2 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsStreamStatistics.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsStreamStatistics.java
@@ -18,9 +18,11 @@
 
 package org.apache.hadoop.fs.azurebfs;
 
-import org.junit.Assert;
 import org.junit.Test;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FileSystem;
@@ -35,6 +37,9 @@ public class ITestAbfsStreamStatistics extends AbstractAbfsIntegrationTest {
   public ITestAbfsStreamStatistics() throws Exception {
   }
 
+  private static final Logger LOG =
+      LoggerFactory.getLogger(ITestAbfsStreamStatistics.class);
+
   private static int LARGE_NUMBER_OF_OPS = 1000000;
 
   /***
@@ -71,20 +76,29 @@ public void testAbfsStreamOps() throws Exception {
       //Test for a single write operation
       assertReadWriteOps("write", 1, statistics.getWriteOps());
 
+      //Flushing output stream to see content to read
+      outForOneOperation.hflush();
       inForOneOperation = fs.open(smallOperationsFile);
-      inForOneOperation.read(testReadWriteOps.getBytes(), 0,
+      statistics.reset();
+      int result = inForOneOperation.read(testReadWriteOps.getBytes(), 0,
           testReadWriteOps.getBytes().length);
 
-      //Test for a single read operation
-      assertReadWriteOps("read", 1, statistics.getReadOps());
+      LOG.info("Result of Read operation : {}", result);
+      /*
+      Testing if 2 read_ops value is coming after reading full content from a
+      file (3 if anything to read from Buffer too).
+      Reason: read() call gives read_ops=1,
+      reading from AbfsClient(http GET) gives read_ops=2.
+       */
+      assertReadWriteOps("read", 2, statistics.getReadOps());
 
     } finally {
-      IOUtils.cleanupWithLogger(null, inForOneOperation,
+      IOUtils.cleanupWithLogger(LOG, inForOneOperation,
           outForOneOperation);
     }
 
     //Validating if content is being written in the smallOperationsFile
-    Assert.assertEquals("Mismatch in content validation", true,
+    assertTrue("Mismatch in content validation",
         validateContent(fs, smallOperationsFile,
             testReadWriteOps.getBytes()));
 
@@ -101,6 +115,8 @@ public void testAbfsStreamOps() throws Exception {
         //Creating the String for content Validation
         largeOperationsValidationString.append(testReadWriteOps);
       }
+      LOG.info("Number of bytes of Large data written: {}",
+          largeOperationsValidationString.toString().getBytes().length);
 
       //Test for 1000000 write operations
       assertReadWriteOps("write", largeValue, statistics.getWriteOps());
@@ -116,12 +132,11 @@ public void testAbfsStreamOps() throws Exception {
       assertReadWriteOps("read", largeValue, statistics.getReadOps());
 
     } finally {
-      IOUtils.cleanupWithLogger(null, inForLargeOperations,
+      IOUtils.cleanupWithLogger(LOG, inForLargeOperations,
           outForLargeOperations);
     }
-
     //Validating if content is being written in largeOperationsFile
-    Assert.assertTrue("Mismatch in content validation",
+    assertTrue("Mismatch in content validation",
         validateContent(fs, largeOperationsFile,
             largeOperationsValidationString.toString().getBytes()));
 
@@ -137,8 +152,7 @@ public void testAbfsStreamOps() throws Exception {
 
   private void assertReadWriteOps(String operation, long expectedValue,
       long actualValue) {
-    Assert
-        .assertEquals("Mismatch in " + operation + " operations", expectedValue,
-            actualValue);
+    assertEquals("Mismatch in " + operation + " operations", expectedValue,
+        actualValue);
   }
 }
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemOauth.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemOauth.java
index 5016609676d70..f3786fbff14cc 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemOauth.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemOauth.java
@@ -22,7 +22,6 @@
 import java.io.InputStream;
 import java.util.Map;
 
-import org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys;
 import org.junit.Assume;
 import org.junit.Test;
 
@@ -31,6 +30,7 @@
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys;
 import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AbfsRestOperationException;
 import org.apache.hadoop.fs.azurebfs.contracts.services.AzureServiceErrorCode;
 import org.apache.hadoop.fs.azurebfs.services.AuthType;
@@ -146,6 +146,8 @@ public void testBlobDataReader() throws Exception {
       abfsStore.openFileForWrite(EXISTED_FILE_PATH, fs.getFsStatistics(), true);
     } catch (AbfsRestOperationException e) {
       assertEquals(AzureServiceErrorCode.AUTHORIZATION_PERMISSION_MISS_MATCH, e.getErrorCode());
+    } finally {
+      abfsStore.close();
     }
 
   }

From cf4b7d459c9df3fb04aef5624496d53deb533d3d Mon Sep 17 00:00:00 2001
From: Mehakmeet Singh <mehakmeet.singh@cloudera.com>
Date: Mon, 23 Mar 2020 17:10:10 +0530
Subject: [PATCH 9/9] cleanup

---
 .../hadoop/fs/azurebfs/ITestAbfsStreamStatistics.java      | 1 -
 .../hadoop/fs/azurebfs/ITestAzureBlobFileSystemOauth.java  | 7 ++++++-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsStreamStatistics.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsStreamStatistics.java
index 554400d26a3f2..010b3aa5a59e8 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsStreamStatistics.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsStreamStatistics.java
@@ -19,7 +19,6 @@
 package org.apache.hadoop.fs.azurebfs;
 
 import org.junit.Test;
-
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemOauth.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemOauth.java
index f3786fbff14cc..e517f685784e7 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemOauth.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemOauth.java
@@ -24,6 +24,8 @@
 
 import org.junit.Assume;
 import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataInputStream;
@@ -34,6 +36,7 @@
 import org.apache.hadoop.fs.azurebfs.contracts.exceptions.AbfsRestOperationException;
 import org.apache.hadoop.fs.azurebfs.contracts.services.AzureServiceErrorCode;
 import org.apache.hadoop.fs.azurebfs.services.AuthType;
+import org.apache.hadoop.io.IOUtils;
 
 import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_OAUTH_CLIENT_ID;
 import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_OAUTH_CLIENT_SECRET;
@@ -52,6 +55,8 @@ public class ITestAzureBlobFileSystemOauth extends AbstractAbfsIntegrationTest{
   private static final Path FILE_PATH = new Path("/testFile");
   private static final Path EXISTED_FILE_PATH = new Path("/existedFile");
   private static final Path EXISTED_FOLDER_PATH = new Path("/existedFolder");
+  private static final Logger LOG =
+      LoggerFactory.getLogger(ITestAbfsStreamStatistics.class);
 
   public ITestAzureBlobFileSystemOauth() throws Exception {
     Assume.assumeTrue(this.getAuthType() == AuthType.OAuth);
@@ -147,7 +152,7 @@ public void testBlobDataReader() throws Exception {
     } catch (AbfsRestOperationException e) {
       assertEquals(AzureServiceErrorCode.AUTHORIZATION_PERMISSION_MISS_MATCH, e.getErrorCode());
     } finally {
-      abfsStore.close();
+      IOUtils.cleanupWithLogger(LOG, abfsStore);
     }
 
   }