From 6c77ec99abb66171e28f9e455260b972023d80d8 Mon Sep 17 00:00:00 2001 From: Ahmed Hussein Date: Fri, 12 Feb 2021 18:00:32 +0000 Subject: [PATCH 1/3] HADOOP-16810. Increase entropy on precommit Linux VMs --- dev-support/bin/hadoop.sh | 2 ++ start-build-env.sh | 22 ++++++++++++++++------ 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/dev-support/bin/hadoop.sh b/dev-support/bin/hadoop.sh index 3343014aae8bb..c92942b86160b 100755 --- a/dev-support/bin/hadoop.sh +++ b/dev-support/bin/hadoop.sh @@ -35,6 +35,8 @@ function personality_globals JIRA_ISSUE_RE='^(HADOOP|YARN|MAPREDUCE|HDFS)-[0-9]+$' #shellcheck disable=SC2034 GITHUB_REPO_DEFAULT="apache/hadoop" + # mount urandom to increase entropy + DOCKER_EXTRAARGS=("-v" "/dev/urandom:/dev/random") HADOOP_HOMEBREW_DIR=${HADOOP_HOMEBREW_DIR:-$(brew --prefix 2>/dev/null)} if [[ -z "${HADOOP_HOMEBREW_DIR}" ]]; then diff --git a/start-build-env.sh b/start-build-env.sh index 2ee44de41aa6e..1d3c94a325ff4 100755 --- a/start-build-env.sh +++ b/start-build-env.sh @@ -87,10 +87,20 @@ DOCKER_INTERACTIVE_RUN=${DOCKER_INTERACTIVE_RUN-"-i -t"} # within the container and use the result on your normal # system. And this also is a significant speedup in subsequent # builds because the dependencies are downloaded only once. -docker run --rm=true $DOCKER_INTERACTIVE_RUN \ - -v "${PWD}:${DOCKER_HOME_DIR}/hadoop${V_OPTS:-}" \ - -w "${DOCKER_HOME_DIR}/hadoop" \ - -v "${HOME}/.m2:${DOCKER_HOME_DIR}/.m2${V_OPTS:-}" \ - -v "${HOME}/.gnupg:${DOCKER_HOME_DIR}/.gnupg${V_OPTS:-}" \ - -u "${USER_ID}" \ +dockerargs=(--rm=true) +dockerargs+=($DOCKER_INTERACTIVE_RUN) +# use urandom to increase entropy +dockerargs+=(-v "/dev/urandom:/dev/random${V_OPTS:-}") +# mount current directory +dockerargs+=(-v "${PWD}:${DOCKER_HOME_DIR}/hadoop${V_OPTS:-}") +# mount maven directory +dockerargs+=(-v "${HOME}/.m2:${DOCKER_HOME_DIR}/.m2${V_OPTS:-}") +# mount gnu +dockerargs+=(-v "${HOME}/.gnupg:${DOCKER_HOME_DIR}/.gnupg${V_OPTS:-}") +# set work directory +dockerargs+=(-w "${DOCKER_HOME_DIR}/hadoop") +# set user +dockerargs+=(-u "${USER_ID}") + +docker run "${dockerargs[@]}" \ "hadoop-build-${USER_ID}" "$@" From 1b2770afaa5a4128190f6463218d8f2ded30d6e8 Mon Sep 17 00:00:00 2001 From: Ahmed Hussein Date: Thu, 18 Feb 2021 10:42:37 -0600 Subject: [PATCH 2/3] HADOOP-16810. check intermediate encryption works --- .../mapred/TestMRIntermediateDataEncryption.java | 15 +-------------- start-build-env.sh | 4 +++- 2 files changed, 4 insertions(+), 15 deletions(-) diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRIntermediateDataEncryption.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRIntermediateDataEncryption.java index fa8dacf6dd507..ce4a6998360c8 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRIntermediateDataEncryption.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRIntermediateDataEncryption.java @@ -58,12 +58,7 @@ public class TestMRIntermediateDataEncryption { private static final Logger LOG = LoggerFactory.getLogger(TestMRIntermediateDataEncryption.class); - /** - * Use urandom to avoid the YarnChild process from hanging on low entropy - * systems. - */ - private static final String JVM_SECURITY_EGD_OPT = - "-Djava.security.egd=file:/dev/./urandom"; + // Where MR job's input will reside. private static final Path INPUT_DIR = new Path("/test/input"); // Where output goes. @@ -115,14 +110,6 @@ public static void setupClass() throws Exception { Configuration conf = new Configuration(); conf.setBoolean(MRJobConfig.MR_ENCRYPTED_INTERMEDIATE_DATA, true); - // Set the jvm arguments. - conf.set(MRJobConfig.MR_AM_ADMIN_COMMAND_OPTS, - JVM_SECURITY_EGD_OPT); - final String childJVMOpts = JVM_SECURITY_EGD_OPT - + " " + conf.get("mapred.child.java.opts", " "); - conf.set("mapred.child.java.opts", childJVMOpts); - - // Start the mini-MR and mini-DFS clusters. dfsCluster = new MiniDFSCluster.Builder(conf) .numDataNodes(NUM_NODES).build(); diff --git a/start-build-env.sh b/start-build-env.sh index 1d3c94a325ff4..6cc2318e70843 100755 --- a/start-build-env.sh +++ b/start-build-env.sh @@ -87,7 +87,7 @@ DOCKER_INTERACTIVE_RUN=${DOCKER_INTERACTIVE_RUN-"-i -t"} # within the container and use the result on your normal # system. And this also is a significant speedup in subsequent # builds because the dependencies are downloaded only once. -dockerargs=(--rm=true) +dockerargs=("--rm=true") dockerargs+=($DOCKER_INTERACTIVE_RUN) # use urandom to increase entropy dockerargs+=(-v "/dev/urandom:/dev/random${V_OPTS:-}") @@ -102,5 +102,7 @@ dockerargs+=(-w "${DOCKER_HOME_DIR}/hadoop") # set user dockerargs+=(-u "${USER_ID}") +set -x + docker run "${dockerargs[@]}" \ "hadoop-build-${USER_ID}" "$@" From 2c8a4171b6b518814c1836fb76458e47fb4d3429 Mon Sep 17 00:00:00 2001 From: Ahmed Hussein Date: Fri, 19 Feb 2021 19:57:38 -0600 Subject: [PATCH 3/3] HADOOP-16810. remove double quotes --- start-build-env.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/start-build-env.sh b/start-build-env.sh index 6cc2318e70843..1d3c94a325ff4 100755 --- a/start-build-env.sh +++ b/start-build-env.sh @@ -87,7 +87,7 @@ DOCKER_INTERACTIVE_RUN=${DOCKER_INTERACTIVE_RUN-"-i -t"} # within the container and use the result on your normal # system. And this also is a significant speedup in subsequent # builds because the dependencies are downloaded only once. -dockerargs=("--rm=true") +dockerargs=(--rm=true) dockerargs+=($DOCKER_INTERACTIVE_RUN) # use urandom to increase entropy dockerargs+=(-v "/dev/urandom:/dev/random${V_OPTS:-}") @@ -102,7 +102,5 @@ dockerargs+=(-w "${DOCKER_HOME_DIR}/hadoop") # set user dockerargs+=(-u "${USER_ID}") -set -x - docker run "${dockerargs[@]}" \ "hadoop-build-${USER_ID}" "$@"