Skip to content

Commit c3134ab

Browse files
authored
HDFS-15814. Make some parameters configurable for DataNodeDiskMetrics (#2676)
1 parent f9a073c commit c3134ab

File tree

4 files changed

+46
-6
lines changed

4 files changed

+46
-6
lines changed

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -676,6 +676,14 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
676676
"dfs.datanode.slowpeer.low.threshold.ms";
677677
public static final long DFS_DATANODE_SLOWPEER_LOW_THRESHOLD_MS_DEFAULT =
678678
5L;
679+
public static final String DFS_DATANODE_MIN_OUTLIER_DETECTION_DISKS_KEY =
680+
"dfs.datanode.min.outlier.detection.disks";
681+
public static final long DFS_DATANODE_MIN_OUTLIER_DETECTION_DISKS_DEFAULT =
682+
5L;
683+
public static final String DFS_DATANODE_SLOWDISK_LOW_THRESHOLD_MS_KEY =
684+
"dfs.datanode.slowdisk.low.threshold.ms";
685+
public static final long DFS_DATANODE_SLOWDISK_LOW_THRESHOLD_MS_DEFAULT =
686+
20L;
679687
public static final String DFS_DATANODE_HOST_NAME_KEY =
680688
HdfsClientConfigKeys.DeprecatedKeys.DFS_DATANODE_HOST_NAME_KEY;
681689
public static final String DFS_NAMENODE_CHECKPOINT_DIR_KEY =

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1489,7 +1489,7 @@ void startDataNode(List<StorageLocation> dataDirectories,
14891489

14901490
if (dnConf.diskStatsEnabled) {
14911491
diskMetrics = new DataNodeDiskMetrics(this,
1492-
dnConf.outliersReportIntervalMs);
1492+
dnConf.outliersReportIntervalMs, getConf());
14931493
}
14941494
}
14951495

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeDiskMetrics.java

Lines changed: 21 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@
1717
*/
1818
package org.apache.hadoop.hdfs.server.datanode.metrics;
1919

20+
import org.apache.hadoop.conf.Configuration;
21+
import org.apache.hadoop.hdfs.DFSConfigKeys;
2022
import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
2123
import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableMap;
2224
import org.apache.hadoop.thirdparty.com.google.common.collect.Maps;
@@ -48,8 +50,6 @@ public class DataNodeDiskMetrics {
4850
DataNodeDiskMetrics.class);
4951

5052
private DataNode dn;
51-
private final long MIN_OUTLIER_DETECTION_DISKS = 5;
52-
private final long SLOW_DISK_LOW_THRESHOLD_MS = 20;
5353
private final long detectionInterval;
5454
private volatile boolean shouldRun;
5555
private OutlierDetector slowDiskDetector;
@@ -61,11 +61,27 @@ public class DataNodeDiskMetrics {
6161
// code, status should not be overridden by daemon thread.
6262
private boolean overrideStatus = true;
6363

64-
public DataNodeDiskMetrics(DataNode dn, long diskOutlierDetectionIntervalMs) {
64+
/**
65+
* Minimum number of disks to run outlier detection.
66+
*/
67+
private final long minOutlierDetectionDisks;
68+
/**
69+
* Threshold in milliseconds below which a disk is definitely not slow.
70+
*/
71+
private final long lowThresholdMs;
72+
73+
public DataNodeDiskMetrics(DataNode dn, long diskOutlierDetectionIntervalMs,
74+
Configuration conf) {
6575
this.dn = dn;
6676
this.detectionInterval = diskOutlierDetectionIntervalMs;
67-
slowDiskDetector = new OutlierDetector(MIN_OUTLIER_DETECTION_DISKS,
68-
SLOW_DISK_LOW_THRESHOLD_MS);
77+
minOutlierDetectionDisks =
78+
conf.getLong(DFSConfigKeys.DFS_DATANODE_MIN_OUTLIER_DETECTION_DISKS_KEY,
79+
DFSConfigKeys.DFS_DATANODE_MIN_OUTLIER_DETECTION_DISKS_DEFAULT);
80+
lowThresholdMs =
81+
conf.getLong(DFSConfigKeys.DFS_DATANODE_SLOWDISK_LOW_THRESHOLD_MS_KEY,
82+
DFSConfigKeys.DFS_DATANODE_SLOWDISK_LOW_THRESHOLD_MS_DEFAULT);
83+
slowDiskDetector =
84+
new OutlierDetector(minOutlierDetectionDisks, lowThresholdMs);
6985
shouldRun = true;
7086
startDiskOutlierDetectionThread();
7187
}

hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2370,6 +2370,22 @@
23702370
</description>
23712371
</property>
23722372

2373+
<property>
2374+
<name>dfs.datanode.min.outlier.detection.disks</name>
2375+
<value>5</value>
2376+
<description>
2377+
Minimum number of disks to run outlier detection.
2378+
</description>
2379+
</property>
2380+
2381+
<property>
2382+
<name>dfs.datanode.slowdisk.low.threshold.ms</name>
2383+
<value>20</value>
2384+
<description>
2385+
Threshold in milliseconds below which a disk is definitely not slow.
2386+
</description>
2387+
</property>
2388+
23732389
<property>
23742390
<name>hadoop.user.group.metrics.percentiles.intervals</name>
23752391
<value></value>

0 commit comments

Comments
 (0)