From a41affd11e122eabd6587b92d704fe3c10f315d4 Mon Sep 17 00:00:00 2001 From: Gera Shegalov Date: Wed, 22 Jun 2016 00:20:35 -0700 Subject: [PATCH 1/2] MAPREDUCE-6721.001 --- .../task/reduce/MergeManagerImpl.java | 2 +- .../src/main/resources/mapred-default.xml | 3 ++- .../task/reduce/TestMergeManager.java | 18 ++++++++++++++++++ 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java index c99a330478ab7..1673ff80a7c28 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java @@ -178,7 +178,7 @@ public MergeManagerImpl(TaskAttemptID reduceId, JobConf jobConf, final float singleShuffleMemoryLimitPercent = jobConf.getFloat(MRJobConfig.SHUFFLE_MEMORY_LIMIT_PERCENT, DEFAULT_SHUFFLE_MEMORY_LIMIT_PERCENT); - if (singleShuffleMemoryLimitPercent <= 0.0f + if (singleShuffleMemoryLimitPercent < 0.0f || singleShuffleMemoryLimitPercent > 1.0f) { throw new IllegalArgumentException("Invalid value for " + MRJobConfig.SHUFFLE_MEMORY_LIMIT_PERCENT + ": " diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml index d973bd42e618b..ebc43aacf0eaf 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml @@ -447,7 +447,8 @@ mapreduce.reduce.shuffle.memory.limit.percent 0.25 Expert: Maximum percentage of the in-memory limit that a - single shuffle can consume + single shuffle can consume. Range of valid values is [0.0, 1.0]. If the value + is 0.0 map outputs are shuffled directly to disk. diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestMergeManager.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestMergeManager.java index ef860afaf85b5..019e55d30640f 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestMergeManager.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestMergeManager.java @@ -41,6 +41,7 @@ import org.apache.hadoop.mapred.MROutputFiles; import org.apache.hadoop.mapred.MapOutputFile; import org.apache.hadoop.mapreduce.MRJobConfig; +import org.apache.hadoop.mapreduce.TaskAttemptID; import org.apache.hadoop.mapreduce.task.reduce.MergeManagerImpl.CompressAwarePath; import org.junit.Assert; import org.junit.Test; @@ -289,4 +290,21 @@ null, conf, mock(LocalFileSystem.class), null, null, null, null, null, assertTrue("Large in-memory reduce area unusable: " + maxInMemReduce, maxInMemReduce > Integer.MAX_VALUE); } + + @Test + public void testZeroShuffleMemoryLimitPercent() throws Exception { + final JobConf jobConf = new JobConf(); + jobConf.setFloat(MRJobConfig.SHUFFLE_MEMORY_LIMIT_PERCENT, 0.0f); + final MergeManagerImpl mgr = + new MergeManagerImpl(null, jobConf, mock(LocalFileSystem.class), + null, null, null, null, null, null, null, null, null, null, + new MROutputFiles()); + final long mapOutputSize = 10; + final int fetcher = 1; + final MapOutput mapOutput = mgr.reserve( + TaskAttemptID.forName("attempt_0_1_m_1_1"), + mapOutputSize, fetcher); + assertEquals("Tiny map outputs should be shuffled to disk", "DISK", + mapOutput.getDescription()); + } } From 53b7e19c935f542b033bd39ba2e82fd53ef37e57 Mon Sep 17 00:00:00 2001 From: Gera Shegalov Date: Wed, 22 Jun 2016 14:24:22 -0700 Subject: [PATCH 2/2] MAPREDUCE-6721.002: addressing javac warnings --- .../hadoop/mapreduce/task/reduce/TestMergeManager.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestMergeManager.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestMergeManager.java index 019e55d30640f..1c0d25b09a3a1 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestMergeManager.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestMergeManager.java @@ -295,13 +295,13 @@ null, conf, mock(LocalFileSystem.class), null, null, null, null, null, public void testZeroShuffleMemoryLimitPercent() throws Exception { final JobConf jobConf = new JobConf(); jobConf.setFloat(MRJobConfig.SHUFFLE_MEMORY_LIMIT_PERCENT, 0.0f); - final MergeManagerImpl mgr = - new MergeManagerImpl(null, jobConf, mock(LocalFileSystem.class), + final MergeManager mgr = + new MergeManagerImpl<>(null, jobConf, mock(LocalFileSystem.class), null, null, null, null, null, null, null, null, null, null, new MROutputFiles()); final long mapOutputSize = 10; final int fetcher = 1; - final MapOutput mapOutput = mgr.reserve( + final MapOutput mapOutput = mgr.reserve( TaskAttemptID.forName("attempt_0_1_m_1_1"), mapOutputSize, fetcher); assertEquals("Tiny map outputs should be shuffled to disk", "DISK",