From 814b536270a336b95e7661d73c1f5c2f7b007968 Mon Sep 17 00:00:00 2001
From: sdeka <sdeka@hortonworks.com>
Date: Wed, 21 Aug 2019 09:05:54 +0530
Subject: [PATCH 1/5] HDDS-1094. Performance test infrastructure : skip writing
 user data on Datanode. Contributed by Supratim Deka

---
 .../apache/hadoop/hdds/HddsConfigKeys.java    |   5 +
 .../container/keyvalue/KeyValueHandler.java   |   4 +-
 .../keyvalue/impl/ChunkManagerFactory.java    |  89 ++++++++++
 .../keyvalue/impl/ChunkManagerImpl.java       |  22 ++-
 .../keyvalue/impl/ChunkManagerNullImpl.java   | 160 ++++++++++++++++++
 .../ozone/freon/RandomKeyGenerator.java       |  10 +-
 ...taValidateWithNonPersistentContainers.java |  66 ++++++++
 7 files changed, 345 insertions(+), 11 deletions(-)
 create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerFactory.java
 create mode 100644 hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerNullImpl.java
 create mode 100644 hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestDataValidateWithNonPersistentContainers.java

diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
index 83e270b3bb309..4310aa23288ab 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
@@ -238,6 +238,11 @@ private HddsConfigKeys() {
   public static final String HDDS_SECURITY_CLIENT_SCM_CERTIFICATE_PROTOCOL_ACL =
       "hdds.security.client.scm.certificate.protocol.acl";
 
+  // Determines if the Container Chunk Manager will persist user data
+  public static final String HDDS_CONTAINER_PERSISTDATA =
+      "hdds.container.chunk.persistdata";
+  public static final boolean HDDS_CONTAINER_PERSISTDATA_DEFAULT = true;
+
   public static final String HDDS_DATANODE_HTTP_ENABLED_KEY =
       "hdds.datanode.http.enabled";
   public static final String HDDS_DATANODE_HTTP_BIND_HOST_KEY =
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
index a65b5be38de27..50e37064486f5 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java
@@ -71,7 +71,7 @@
 import org.apache.hadoop.ozone.container.keyvalue.helpers.BlockUtils;
 import org.apache.hadoop.ozone.container.keyvalue.helpers.KeyValueContainerUtil;
 import org.apache.hadoop.ozone.container.keyvalue.helpers.SmallFileUtils;
-import org.apache.hadoop.ozone.container.keyvalue.impl.ChunkManagerImpl;
+import org.apache.hadoop.ozone.container.keyvalue.impl.ChunkManagerFactory;
 import org.apache.hadoop.ozone.container.keyvalue.impl.BlockManagerImpl;
 import org.apache.hadoop.ozone.container.keyvalue.interfaces.ChunkManager;
 import org.apache.hadoop.ozone.container.keyvalue.interfaces.BlockManager;
@@ -114,7 +114,7 @@ public KeyValueHandler(Configuration config, StateContext context,
     doSyncWrite =
         conf.getBoolean(OzoneConfigKeys.DFS_CONTAINER_CHUNK_WRITE_SYNC_KEY,
             OzoneConfigKeys.DFS_CONTAINER_CHUNK_WRITE_SYNC_DEFAULT);
-    chunkManager = new ChunkManagerImpl(doSyncWrite);
+    chunkManager = ChunkManagerFactory.getChunkManager(config, doSyncWrite);
     volumeChoosingPolicy = ReflectionUtils.newInstance(conf.getClass(
         HDDS_DATANODE_VOLUME_CHOOSING_POLICY, RoundRobinVolumeChoosingPolicy
             .class, VolumeChoosingPolicy.class), conf);
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerFactory.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerFactory.java
new file mode 100644
index 0000000000000..5c7c72194f462
--- /dev/null
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerFactory.java
@@ -0,0 +1,89 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.container.keyvalue.impl;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.HddsConfigKeys;
+import org.apache.hadoop.ozone.container.keyvalue.interfaces.ChunkManager;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_CONTAINER_PERSISTDATA;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_CONTAINER_PERSISTDATA_DEFAULT;
+
+/**
+ * Select an appropriate ChunkManager implementation as per config setting.
+ * Ozone ChunkManager is a Singleton
+ */
+public final class ChunkManagerFactory {
+  static final Logger LOG = LoggerFactory.getLogger(ChunkManagerFactory.class);
+
+  private static ChunkManager instance = null;
+  private static boolean syncChunks = false;
+
+  private ChunkManagerFactory() {
+  }
+
+  public static ChunkManager getChunkManager(Configuration config,
+      boolean sync) {
+    if (instance == null) {
+      synchronized (ChunkManagerFactory.class) {
+        if (instance == null) {
+          instance = createChunkManager(config, sync);
+          syncChunks = sync;
+        }
+      }
+    }
+
+    Preconditions.checkArgument((syncChunks == sync),
+        "value of sync conflicts with previous invocation");
+    return instance;
+  }
+
+  private static ChunkManager createChunkManager(Configuration config,
+      boolean sync) {
+    ChunkManager manager = null;
+    boolean persist = config.getBoolean(HDDS_CONTAINER_PERSISTDATA,
+        HDDS_CONTAINER_PERSISTDATA_DEFAULT);
+
+    if (persist == false) {
+      boolean scrubber = config.getBoolean(
+          HddsConfigKeys.HDDS_CONTAINERSCRUB_ENABLED,
+          HddsConfigKeys.HDDS_CONTAINERSCRUB_ENABLED_DEFAULT);
+      if (scrubber) {
+        // Data Scrubber needs to be disabled for non-persistent chunks.
+        LOG.warn("Failed to set " + HDDS_CONTAINER_PERSISTDATA + " to false."
+            + " Please set " + HddsConfigKeys.HDDS_CONTAINERSCRUB_ENABLED
+            + " also to false to enable non-persistent containers.");
+        persist = true;
+      }
+    }
+
+    if (persist == true) {
+      manager = new ChunkManagerImpl(sync);
+    } else {
+      LOG.warn(HDDS_CONTAINER_PERSISTDATA
+          + " is set to false. All user data will be discarded.");
+      manager = new ChunkManagerNullImpl(sync);
+    }
+
+    return manager;
+  }
+}
\ No newline at end of file
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerImpl.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerImpl.java
index 1b7029882feb2..fc079374d4137 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerImpl.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerImpl.java
@@ -142,18 +142,12 @@ public void writeChunk(Container container, BlockID blockID, ChunkInfo info,
         // the same term and log index appended as the current transaction
         commitChunk(tmpChunkFile, chunkFile);
         // Increment container stats here, as we commit the data.
-        containerData.incrBytesUsed(info.getLen());
-        containerData.incrWriteCount();
-        containerData.incrWriteBytes(info.getLen());
+        updateContainerWriteStats(container, info, isOverwrite);
         break;
       case COMBINED:
         // directly write to the chunk file
         ChunkUtils.writeData(chunkFile, info, data, volumeIOStats, doSyncWrite);
-        if (!isOverwrite) {
-          containerData.incrBytesUsed(info.getLen());
-        }
-        containerData.incrWriteCount();
-        containerData.incrWriteBytes(info.getLen());
+        updateContainerWriteStats(container, info, isOverwrite);
         break;
       default:
         throw new IOException("Can not identify write operation.");
@@ -176,6 +170,18 @@ public void writeChunk(Container container, BlockID blockID, ChunkInfo info,
     }
   }
 
+  protected void updateContainerWriteStats(Container container, ChunkInfo info,
+      boolean isOverwrite) {
+    KeyValueContainerData containerData = (KeyValueContainerData) container
+        .getContainerData();
+
+    if (!isOverwrite) {
+      containerData.incrBytesUsed(info.getLen());
+    }
+    containerData.incrWriteCount();
+    containerData.incrWriteBytes(info.getLen());
+  }
+
   /**
    * reads the data defined by a chunk.
    *
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerNullImpl.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerNullImpl.java
new file mode 100644
index 0000000000000..e2d7472be9647
--- /dev/null
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerNullImpl.java
@@ -0,0 +1,160 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.container.keyvalue.impl;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.hdds.client.BlockID;
+import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
+import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo;
+import org.apache.hadoop.ozone.container.common.interfaces.Container;
+import org.apache.hadoop.ozone.container.common.transport.server.ratis.DispatcherContext;
+import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
+import org.apache.hadoop.ozone.container.common.volume.VolumeIOStats;
+import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
+import org.apache.hadoop.ozone.container.keyvalue.helpers.ChunkUtils;
+import org.apache.hadoop.util.Time;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.security.NoSuchAlgorithmException;
+import java.util.concurrent.ExecutionException;
+
+import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result.*;
+
+public class ChunkManagerNullImpl extends ChunkManagerImpl {
+  static final Logger LOG = LoggerFactory.getLogger(ChunkManagerNullImpl.class);
+
+  public ChunkManagerNullImpl(boolean sync) {
+    super(sync);
+  }
+
+  /**
+   * writes a given chunk.
+   *
+   * @param container - Container for the chunk
+   * @param blockID - ID of the block
+   * @param info - ChunkInfo
+   * @param data - data of the chunk
+   * @param dispatcherContext - dispatcherContextInfo
+   * @throws StorageContainerException
+   */
+  @Override
+  public void writeChunk(Container container, BlockID blockID, ChunkInfo info,
+      ByteBuffer data, DispatcherContext dispatcherContext)
+      throws StorageContainerException {
+    long writeTimeStart = Time.monotonicNow();
+
+    Preconditions.checkNotNull(dispatcherContext);
+    DispatcherContext.WriteChunkStage stage = dispatcherContext.getStage();
+
+    Logger log = LoggerFactory.getLogger(ChunkManagerImpl.class);
+
+    try {
+      KeyValueContainerData containerData = (KeyValueContainerData) container.getContainerData();
+      HddsVolume volume = containerData.getVolume();
+      VolumeIOStats volumeIOStats = volume.getVolumeIOStats();
+      int bufferSize;
+
+      switch (stage) {
+      case WRITE_DATA:
+        bufferSize = data.capacity();
+        if (bufferSize != info.getLen()) {
+          String err = String.format("data array does not match the length " +
+                  "specified. DataLen: %d Byte Array: %d",
+              info.getLen(), bufferSize);
+          log.error(err);
+          throw new StorageContainerException(err, INVALID_WRITE_SIZE);
+        }
+
+        // Increment volumeIO stats here.
+        volumeIOStats.incWriteTime(Time.monotonicNow() - writeTimeStart);
+        volumeIOStats.incWriteOpCount();
+        volumeIOStats.incWriteBytes(info.getLen());
+        break;
+      case COMMIT_DATA:
+        updateContainerWriteStats(container, info, false);
+        break;
+      case COMBINED:
+        updateContainerWriteStats(container, info, false);
+        break;
+      default:
+        throw new IOException("Can not identify write operation.");
+      }
+    } catch (IOException ex) {
+      LOG.error("write data failed. error: {}", ex);
+      throw new StorageContainerException("Internal error: ", ex,
+          CONTAINER_INTERNAL_ERROR);
+    }
+  }
+
+  /**
+   * return a zero-filled buffer.
+   *
+   * @param container - Container for the chunk
+   * @param blockID - ID of the block.
+   * @param info - ChunkInfo.
+   * @param dispatcherContext dispatcher context info.
+   * @return byte array
+   * TODO: Right now we do not support partial reads and writes of chunks.
+   * TODO: Explore if we need to do that for ozone.
+   */
+  @Override
+  public byte[] readChunk(Container container, BlockID blockID, ChunkInfo info,
+      DispatcherContext dispatcherContext) {
+
+    long readStartTime = Time.monotonicNow();
+
+    KeyValueContainerData containerData = (KeyValueContainerData) container
+        .getContainerData();
+    ByteBuffer data;
+    HddsVolume volume = containerData.getVolume();
+    VolumeIOStats volumeIOStats = volume.getVolumeIOStats();
+
+    data = ByteBuffer.allocate((int) info.getLen());
+
+    // Increment volumeIO stats here.
+    volumeIOStats.incReadTime(Time.monotonicNow() - readStartTime);
+    volumeIOStats.incReadOpCount();
+    volumeIOStats.incReadBytes(info.getLen());
+
+    return data.array();
+  }
+
+  /**
+   * Delete a given chunk - Do nothing except stats
+   *
+   * @param container - Container for the chunk
+   * @param blockID - ID of the block
+   * @param info - Chunk Info
+   */
+  @Override
+  public void deleteChunk(Container container, BlockID blockID,
+      ChunkInfo info) {
+    Preconditions.checkNotNull(blockID, "Block ID cannot be null.");
+    KeyValueContainerData containerData = (KeyValueContainerData) container.getContainerData();
+
+    if (info.getOffset() == 0) {
+      containerData.decrBytesUsed(info.getLen());
+    }
+  }
+}
\ No newline at end of file
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java
index 7cfd1fef81f3a..200df960ae293 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java
@@ -40,6 +40,7 @@
 import io.opentracing.Scope;
 import io.opentracing.util.GlobalTracer;
 import org.apache.commons.codec.digest.DigestUtils;
+import org.apache.hadoop.hdds.HddsConfigKeys;
 import org.apache.hadoop.hdds.cli.HddsVersionProvider;
 import org.apache.hadoop.hdds.client.OzoneQuota;
 import org.apache.hadoop.hdds.client.ReplicationFactor;
@@ -251,6 +252,12 @@ public void init(OzoneConfiguration configuration) throws IOException {
   @Override
   public Void call() throws Exception {
     if (ozoneConfiguration != null) {
+      if (ozoneConfiguration.getBoolean(HddsConfigKeys.HDDS_CONTAINER_PERSISTDATA,
+          HddsConfigKeys.HDDS_CONTAINER_PERSISTDATA_DEFAULT) == false) {
+        LOG.info("Override validateWrites to false, because "
+        + HddsConfigKeys.HDDS_CONTAINER_PERSISTDATA + " is set to false.");
+        validateWrites = false;
+      }
       init(ozoneConfiguration);
     } else {
       init(freon.createOzoneConfiguration());
@@ -282,6 +289,7 @@ public Void call() throws Exception {
     LOG.info("Number of Keys per Bucket: {}.", numOfKeys);
     LOG.info("Key size: {} bytes", keySize);
     LOG.info("Buffer size: {} bytes", bufferSize);
+    LOG.info("validateWrites : {}", validateWrites);
     for (int i = 0; i < numOfThreads; i++) {
       executor.submit(new ObjectCreator());
     }
@@ -548,7 +556,7 @@ long getSuccessfulValidationCount() {
    */
   @VisibleForTesting
   long getUnsuccessfulValidationCount() {
-    return writeValidationFailureCount;
+    return validateWrites ? writeValidationFailureCount : 0;
   }
 
   /**
diff --git a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestDataValidateWithNonPersistentContainers.java b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestDataValidateWithNonPersistentContainers.java
new file mode 100644
index 0000000000000..8bc939fc1f730
--- /dev/null
+++ b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestDataValidateWithNonPersistentContainers.java
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.freon;
+
+import org.apache.hadoop.hdds.HddsConfigKeys;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class TestDataValidateWithNonPersistentContainers extends TestDataValidate {
+  private static final Logger LOG =
+      LoggerFactory.getLogger(TestDataValidate.class);
+  /**
+   * Create a MiniDFSCluster for testing.
+   * <p>
+   * Ozone is made active by setting OZONE_ENABLED = true
+   *
+   */
+  @BeforeClass
+  public static void init() throws Exception {
+    OzoneConfiguration conf = new OzoneConfiguration();
+    conf.setBoolean(HddsConfigKeys.HDDS_CONTAINERSCRUB_ENABLED, false);
+    conf.setBoolean(HddsConfigKeys.HDDS_CONTAINER_PERSISTDATA, false);
+    conf.setBoolean(OzoneConfigKeys.OZONE_UNSAFEBYTEOPERATIONS_ENABLED,
+        false);
+    startCluster(conf);
+  }
+
+  /**
+   * Write validation is not supported for non-persistent containers.
+   * This test is a no-op.
+   */
+  @Test
+  @Override
+  public void validateWriteTest() throws Exception {
+    LOG.info("Skipping validateWriteTest for non-persistent containers.");
+  }
+
+  /**
+   * Shutdown MiniDFSCluster.
+   */
+  @AfterClass
+  public static void shutdown() {
+    shutdownCluster();
+  }
+}

From f31fa15f9a35423e7cc4be55175128e7b9c31808 Mon Sep 17 00:00:00 2001
From: sdeka <sdeka@hortonworks.com>
Date: Fri, 23 Aug 2019 13:40:44 +0530
Subject: [PATCH 2/5] addressed review comments.

---
 .../main/java/org/apache/hadoop/hdds/HddsConfigKeys.java | 3 ++-
 .../container/keyvalue/impl/ChunkManagerFactory.java     | 9 +++++----
 2 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
index 4310aa23288ab..9e757c1ee1781 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
@@ -238,7 +238,8 @@ private HddsConfigKeys() {
   public static final String HDDS_SECURITY_CLIENT_SCM_CERTIFICATE_PROTOCOL_ACL =
       "hdds.security.client.scm.certificate.protocol.acl";
 
-  // Determines if the Container Chunk Manager will persist user data
+  // Determines if the Container Chunk Manager will write user data to disk
+  // Set to false only for specific performance tests
   public static final String HDDS_CONTAINER_PERSISTDATA =
       "hdds.container.chunk.persistdata";
   public static final boolean HDDS_CONTAINER_PERSISTDATA_DEFAULT = true;
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerFactory.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerFactory.java
index 5c7c72194f462..4aed3ec9c28c8 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerFactory.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerFactory.java
@@ -35,7 +35,7 @@
 public final class ChunkManagerFactory {
   static final Logger LOG = LoggerFactory.getLogger(ChunkManagerFactory.class);
 
-  private static ChunkManager instance = null;
+  private static volatile ChunkManager instance = null;
   private static boolean syncChunks = false;
 
   private ChunkManagerFactory() {
@@ -63,7 +63,7 @@ private static ChunkManager createChunkManager(Configuration config,
     boolean persist = config.getBoolean(HDDS_CONTAINER_PERSISTDATA,
         HDDS_CONTAINER_PERSISTDATA_DEFAULT);
 
-    if (persist == false) {
+    if (!persist) {
       boolean scrubber = config.getBoolean(
           HddsConfigKeys.HDDS_CONTAINERSCRUB_ENABLED,
           HddsConfigKeys.HDDS_CONTAINERSCRUB_ENABLED_DEFAULT);
@@ -76,11 +76,12 @@ private static ChunkManager createChunkManager(Configuration config,
       }
     }
 
-    if (persist == true) {
+    if (persist) {
       manager = new ChunkManagerImpl(sync);
     } else {
       LOG.warn(HDDS_CONTAINER_PERSISTDATA
-          + " is set to false. All user data will be discarded.");
+          + " is set to false. This should be used only for testing."
+          + " All user data will be discarded.");
       manager = new ChunkManagerNullImpl(sync);
     }
 

From 308bc9327e13f18626e3ee6399b2a2add1ca17f0 Mon Sep 17 00:00:00 2001
From: sdeka <sdeka@hortonworks.com>
Date: Fri, 23 Aug 2019 14:40:23 +0530
Subject: [PATCH 3/5] fixed checkstyle issues

---
 ...llImpl.java => ChunkManagerDummyImpl.java} | 27 ++++++++++---------
 .../keyvalue/impl/ChunkManagerFactory.java    |  2 +-
 .../ozone/freon/RandomKeyGenerator.java       |  7 ++---
 ... TestDataValidateWithDummyContainers.java} |  3 ++-
 4 files changed, 21 insertions(+), 18 deletions(-)
 rename hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/{ChunkManagerNullImpl.java => ChunkManagerDummyImpl.java} (88%)
 rename hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/{TestDataValidateWithNonPersistentContainers.java => TestDataValidateWithDummyContainers.java} (96%)

diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerNullImpl.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerDummyImpl.java
similarity index 88%
rename from hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerNullImpl.java
rename to hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerDummyImpl.java
index e2d7472be9647..2b6d3163d406a 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerNullImpl.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerDummyImpl.java
@@ -19,7 +19,6 @@
 package org.apache.hadoop.ozone.container.keyvalue.impl;
 
 import com.google.common.base.Preconditions;
-import org.apache.hadoop.fs.FileUtil;
 import org.apache.hadoop.hdds.client.BlockID;
 import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
 import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo;
@@ -28,23 +27,23 @@
 import org.apache.hadoop.ozone.container.common.volume.HddsVolume;
 import org.apache.hadoop.ozone.container.common.volume.VolumeIOStats;
 import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
-import org.apache.hadoop.ozone.container.keyvalue.helpers.ChunkUtils;
 import org.apache.hadoop.util.Time;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.io.File;
 import java.io.IOException;
 import java.nio.ByteBuffer;
-import java.security.NoSuchAlgorithmException;
-import java.util.concurrent.ExecutionException;
 
 import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result.*;
 
-public class ChunkManagerNullImpl extends ChunkManagerImpl {
-  static final Logger LOG = LoggerFactory.getLogger(ChunkManagerNullImpl.class);
+/**
+ * Implementation of ChunkManager built for running performance tests.
+ * Chunks are not written to disk, Reads are returned with zero-filled buffers
+ */
+public class ChunkManagerDummyImpl extends ChunkManagerImpl {
+  static final Logger LOG = LoggerFactory.getLogger(ChunkManagerDummyImpl.class);
 
-  public ChunkManagerNullImpl(boolean sync) {
+  public ChunkManagerDummyImpl(boolean sync) {
     super(sync);
   }
 
@@ -70,7 +69,8 @@ public void writeChunk(Container container, BlockID blockID, ChunkInfo info,
     Logger log = LoggerFactory.getLogger(ChunkManagerImpl.class);
 
     try {
-      KeyValueContainerData containerData = (KeyValueContainerData) container.getContainerData();
+      KeyValueContainerData containerData =
+          (KeyValueContainerData) container.getContainerData();
       HddsVolume volume = containerData.getVolume();
       VolumeIOStats volumeIOStats = volume.getVolumeIOStats();
       int bufferSize;
@@ -79,8 +79,8 @@ public void writeChunk(Container container, BlockID blockID, ChunkInfo info,
       case WRITE_DATA:
         bufferSize = data.capacity();
         if (bufferSize != info.getLen()) {
-          String err = String.format("data array does not match the length " +
-                  "specified. DataLen: %d Byte Array: %d",
+          String err = String.format("data array does not match the length "
+                  + "specified. DataLen: %d Byte Array: %d",
               info.getLen(), bufferSize);
           log.error(err);
           throw new StorageContainerException(err, INVALID_WRITE_SIZE);
@@ -141,7 +141,7 @@ public byte[] readChunk(Container container, BlockID blockID, ChunkInfo info,
   }
 
   /**
-   * Delete a given chunk - Do nothing except stats
+   * Delete a given chunk - Do nothing except stats.
    *
    * @param container - Container for the chunk
    * @param blockID - ID of the block
@@ -151,7 +151,8 @@ public byte[] readChunk(Container container, BlockID blockID, ChunkInfo info,
   public void deleteChunk(Container container, BlockID blockID,
       ChunkInfo info) {
     Preconditions.checkNotNull(blockID, "Block ID cannot be null.");
-    KeyValueContainerData containerData = (KeyValueContainerData) container.getContainerData();
+    KeyValueContainerData containerData =
+        (KeyValueContainerData) container.getContainerData();
 
     if (info.getOffset() == 0) {
       containerData.decrBytesUsed(info.getLen());
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerFactory.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerFactory.java
index 4aed3ec9c28c8..8e2687b65bdfb 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerFactory.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerFactory.java
@@ -82,7 +82,7 @@ private static ChunkManager createChunkManager(Configuration config,
       LOG.warn(HDDS_CONTAINER_PERSISTDATA
           + " is set to false. This should be used only for testing."
           + " All user data will be discarded.");
-      manager = new ChunkManagerNullImpl(sync);
+      manager = new ChunkManagerDummyImpl(sync);
     }
 
     return manager;
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java
index 200df960ae293..2bf79c5c78248 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java
@@ -252,10 +252,11 @@ public void init(OzoneConfiguration configuration) throws IOException {
   @Override
   public Void call() throws Exception {
     if (ozoneConfiguration != null) {
-      if (ozoneConfiguration.getBoolean(HddsConfigKeys.HDDS_CONTAINER_PERSISTDATA,
-          HddsConfigKeys.HDDS_CONTAINER_PERSISTDATA_DEFAULT) == false) {
+      if (!ozoneConfiguration.getBoolean(
+          HddsConfigKeys.HDDS_CONTAINER_PERSISTDATA,
+          HddsConfigKeys.HDDS_CONTAINER_PERSISTDATA_DEFAULT)) {
         LOG.info("Override validateWrites to false, because "
-        + HddsConfigKeys.HDDS_CONTAINER_PERSISTDATA + " is set to false.");
+            + HddsConfigKeys.HDDS_CONTAINER_PERSISTDATA + " is set to false.");
         validateWrites = false;
       }
       init(ozoneConfiguration);
diff --git a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestDataValidateWithNonPersistentContainers.java b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestDataValidateWithDummyContainers.java
similarity index 96%
rename from hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestDataValidateWithNonPersistentContainers.java
rename to hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestDataValidateWithDummyContainers.java
index 8bc939fc1f730..3f03292f02690 100644
--- a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestDataValidateWithNonPersistentContainers.java
+++ b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestDataValidateWithDummyContainers.java
@@ -27,7 +27,8 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-public class TestDataValidateWithNonPersistentContainers extends TestDataValidate {
+public class TestDataValidateWithDummyContainers
+    extends TestDataValidate {
   private static final Logger LOG =
       LoggerFactory.getLogger(TestDataValidate.class);
   /**

From 83a5cdf699b4e893163e524668abb5c7938a049a Mon Sep 17 00:00:00 2001
From: sdeka <sdeka@hortonworks.com>
Date: Fri, 23 Aug 2019 14:50:09 +0530
Subject: [PATCH 4/5] fixed leftover checkstyle issue

---
 .../ozone/freon/TestDataValidateWithDummyContainers.java      | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestDataValidateWithDummyContainers.java b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestDataValidateWithDummyContainers.java
index 3f03292f02690..d8c98c26ebcdb 100644
--- a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestDataValidateWithDummyContainers.java
+++ b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestDataValidateWithDummyContainers.java
@@ -27,6 +27,10 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+/**
+ * Tests Freon with MiniOzoneCluster and ChunkManagerDummyImpl.
+ * Data validation is disabled in RandomKeyGenerator.
+ */
 public class TestDataValidateWithDummyContainers
     extends TestDataValidate {
   private static final Logger LOG =

From 4c684691b9d083499f801799a56a76a2e3342115 Mon Sep 17 00:00:00 2001
From: sdeka <sdeka@hortonworks.com>
Date: Wed, 28 Aug 2019 09:19:40 +0530
Subject: [PATCH 5/5] checkstyle issues.

---
 .../ozone/container/keyvalue/impl/ChunkManagerDummyImpl.java   | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerDummyImpl.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerDummyImpl.java
index 2b6d3163d406a..9d63c166034f4 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerDummyImpl.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/ChunkManagerDummyImpl.java
@@ -41,7 +41,8 @@
  * Chunks are not written to disk, Reads are returned with zero-filled buffers
  */
 public class ChunkManagerDummyImpl extends ChunkManagerImpl {
-  static final Logger LOG = LoggerFactory.getLogger(ChunkManagerDummyImpl.class);
+  static final Logger LOG = LoggerFactory.getLogger(
+      ChunkManagerDummyImpl.class);
 
   public ChunkManagerDummyImpl(boolean sync) {
     super(sync);