From 2fb22b1f8895e99a4ed73af44c5ea24da5108b81 Mon Sep 17 00:00:00 2001 From: Takanobu Asanuma Date: Mon, 28 Oct 2019 16:51:54 +0900 Subject: [PATCH 1/4] HDFS-14907. [Dynamometer] DataNode can't find junit jar when using Hadoop-3 binary --- .../src/main/resources/start-component.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/main/resources/start-component.sh b/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/main/resources/start-component.sh index e1974cab2e6c8..f0f9234b414ff 100644 --- a/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/main/resources/start-component.sh +++ b/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/main/resources/start-component.sh @@ -94,6 +94,12 @@ function hdfs_original { extraClasspathDir="$(pwd)/additionalClasspath/" mkdir -p "${extraClasspathDir}" +junitClassPath="$(find $hadoopHome -name "junit*.jar" | head -n 1)" +if [[ -z "$junitClassPath" ]]; then + echo "Can't find junit jar file." + exit 1 +fi + # Change environment variables for the Hadoop process export HADOOP_HOME="$hadoopHome" export HADOOP_PREFIX="$hadoopHome" @@ -107,7 +113,7 @@ export HADOOP_CONF_DIR="${confDir}" export YARN_CONF_DIR="${confDir}" export HADOOP_LOG_DIR="${logDir}" export HADOOP_PID_DIR="${pidDir}" -HADOOP_CLASSPATH="$(pwd)/dependencies/*:$extraClasspathDir" +HADOOP_CLASSPATH="$(pwd)/dependencies/*:$extraClasspathDir:$junitClassPath" export HADOOP_CLASSPATH echo "Environment variables are set as:" echo "(note that this doesn't include changes made by hadoop-env.sh)" From 21a295b299b7aba6c4de9725cd10c213d7c6700d Mon Sep 17 00:00:00 2001 From: Takanobu Asanuma Date: Mon, 28 Oct 2019 19:14:34 +0900 Subject: [PATCH 2/4] fix a shellcheck issue --- .../src/main/resources/start-component.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/main/resources/start-component.sh b/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/main/resources/start-component.sh index f0f9234b414ff..ef7f5f3c51fd6 100644 --- a/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/main/resources/start-component.sh +++ b/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/main/resources/start-component.sh @@ -94,7 +94,7 @@ function hdfs_original { extraClasspathDir="$(pwd)/additionalClasspath/" mkdir -p "${extraClasspathDir}" -junitClassPath="$(find $hadoopHome -name "junit*.jar" | head -n 1)" +junitClassPath="$(find ${hadoopHome} -name "junit*.jar" | head -n 1)" if [[ -z "$junitClassPath" ]]; then echo "Can't find junit jar file." exit 1 From e625662b9140cf806fa465b0ccb900c305f12d52 Mon Sep 17 00:00:00 2001 From: Takanobu Asanuma Date: Mon, 28 Oct 2019 20:45:50 +0900 Subject: [PATCH 3/4] fix a shellcheck issue again --- .../src/main/resources/start-component.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/main/resources/start-component.sh b/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/main/resources/start-component.sh index ef7f5f3c51fd6..d5bb9fa2bb39a 100644 --- a/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/main/resources/start-component.sh +++ b/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/main/resources/start-component.sh @@ -94,7 +94,7 @@ function hdfs_original { extraClasspathDir="$(pwd)/additionalClasspath/" mkdir -p "${extraClasspathDir}" -junitClassPath="$(find ${hadoopHome} -name "junit*.jar" | head -n 1)" +junitClassPath="$(find "${hadoopHome}" -name "junit*.jar" | head -n 1)" if [[ -z "$junitClassPath" ]]; then echo "Can't find junit jar file." exit 1 From 995a87961d90850b214d334ab906a33861732f72 Mon Sep 17 00:00:00 2001 From: Takanobu Asanuma Date: Tue, 29 Oct 2019 10:06:33 +0900 Subject: [PATCH 4/4] add more comments --- .../src/main/resources/start-component.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/main/resources/start-component.sh b/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/main/resources/start-component.sh index d5bb9fa2bb39a..3a07b72f18298 100644 --- a/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/main/resources/start-component.sh +++ b/hadoop-tools/hadoop-dynamometer/hadoop-dynamometer-infra/src/main/resources/start-component.sh @@ -94,9 +94,10 @@ function hdfs_original { extraClasspathDir="$(pwd)/additionalClasspath/" mkdir -p "${extraClasspathDir}" +# DataNodes need junit jar to run SimulatedDataNodes junitClassPath="$(find "${hadoopHome}" -name "junit*.jar" | head -n 1)" if [[ -z "$junitClassPath" ]]; then - echo "Can't find junit jar file." + echo "Can't find junit jar file in ${hadoopHome}." exit 1 fi