add max count.

Author: hfutatzhanghb

hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java

@@ -75,6 +75,8 @@
 
 import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_EC_WRITE_ALLOW_END_BLOCKGROUP_INADVANCE;
 import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_EC_WRITE_ALLOW_END_BLOCKGROUP_INADVANCE_DEFAULT;
+import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_EC_WRITE_MAX_END_BLOCKGROUP_INADVANCE_COUNT;
+import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DFS_CLIENT_EC_WRITE_MAX_END_BLOCKGROUP_INADVANCE_COUNT_DEFAULT;
 import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.Write.ECRedundancy.DFS_CLIENT_EC_WRITE_FAILED_BLOCKS_TOLERATED;
 import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.Write.ECRedundancy.DFS_CLIENT_EC_WRITE_FAILED_BLOCKS_TOLERATED_DEFAILT;
 
@@ -289,6 +291,8 @@ private void flipDataBuffers() {
   private int blockGroupIndex;
   private long datanodeRestartTimeout;
   private final int failedBlocksTolerated;
+  private final int maxEndBlockGroupInAdvanceCount;
+  private int curEndBlockGroupInAdvanceCount;
   private final boolean allowEndBlockGroupInAdvance;
   private boolean endBlockGroupInAdvance;
 
@@ -342,6 +346,9 @@ private void flipDataBuffers() {
     allowEndBlockGroupInAdvance = dfsClient.getConfiguration().getBoolean(
         DFS_CLIENT_EC_WRITE_ALLOW_END_BLOCKGROUP_INADVANCE,
         DFS_CLIENT_EC_WRITE_ALLOW_END_BLOCKGROUP_INADVANCE_DEFAULT);
+    maxEndBlockGroupInAdvanceCount = dfsClient.getConfiguration().getInt(
+        DFS_CLIENT_EC_WRITE_MAX_END_BLOCKGROUP_INADVANCE_COUNT,
+        DFS_CLIENT_EC_WRITE_MAX_END_BLOCKGROUP_INADVANCE_COUNT_DEFAULT);
   }
 
   /** Construct a new output stream for appending to a file. */
@@ -577,12 +584,14 @@ private boolean shouldEndBlockGroup() {
   }
 
   private boolean shouldEndBlockGroupInAdvance() {
-    if (!allowEndBlockGroupInAdvance) {
+    if (!allowEndBlockGroupInAdvance ||
+        curEndBlockGroupInAdvanceCount > maxEndBlockGroupInAdvanceCount) {
       return false;
     }
     if (DFSClientFaultInjector.get().mockEndBlockGroupInAdvance()) {
       LOG.info("Block group {} ends in advance.", currentBlockGroup);
       this.endBlockGroupInAdvance = true;
+      curEndBlockGroupInAdvanceCount++;
       return true;
     }
 
@@ -594,6 +603,7 @@ private boolean shouldEndBlockGroupInAdvance() {
     if (overFailedStreamer && stripeFull) {
       LOG.info("Block group {} ends in advance.", currentBlockGroup);
       this.endBlockGroupInAdvance = true;
+      curEndBlockGroupInAdvanceCount++;
       return true;
     }
     return false;

hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java

@@ -300,6 +300,10 @@ public interface HdfsClientConfigKeys {
       "dfs.client.ec.write.allow.end.blockgroup.inadvance";
   boolean DFS_CLIENT_EC_WRITE_ALLOW_END_BLOCKGROUP_INADVANCE_DEFAULT = false;
 
+  String DFS_CLIENT_EC_WRITE_MAX_END_BLOCKGROUP_INADVANCE_COUNT =
+      "dfs.client.ec.write.max.end.blockgroup.inadvance.count";
+  int DFS_CLIENT_EC_WRITE_MAX_END_BLOCKGROUP_INADVANCE_COUNT_DEFAULT = 10;
+
   /**
    * These are deprecated config keys to client code.
    */

hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml

@@ -6708,4 +6708,12 @@
       Whether allow client ends non-full block group in advance or not.
     </description>
   </property>
+  <property>
+    <name>dfs.client.ec.write.max.end.blockgroup.inadvance.count</name>
+    <value>10</value>
+    <description>
+      Max count of ending block group in advance allowed.
+      This prevents the creation of large amounts of metadata in NameNode.
+    </description>
+  </property>
 </configuration>