From 8967079fbbea6ee487a86dc54ad4a6760be77353 Mon Sep 17 00:00:00 2001 From: taccart Date: Wed, 14 Oct 2020 18:11:46 +0200 Subject: [PATCH 1/5] HADOOP-16649 - hadoop_add_param : replace the regexp test by an iterative equality test on each value. --- .../src/main/bin/hadoop-functions.sh | 22 ++++++++++++++----- .../src/test/scripts/hadoop_add_param.bats | 19 +++++++++++++++- 2 files changed, 35 insertions(+), 6 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh index 4be554aef6c25..f16c988aaabc0 100755 --- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh +++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh @@ -1126,6 +1126,8 @@ function hadoop_validate_classname ## @param envvar ## @param checkstring ## @param appendstring + + function hadoop_add_param { # @@ -1136,19 +1138,29 @@ function hadoop_add_param # # doing it this way allows us to support all sorts of # different syntaxes, just so long as they are space - # delimited + # delimited. # - if [[ ! ${!1} =~ $2 ]] ; then - #shellcheck disable=SC2140 + # testing with string regexp fails (see HADOOP-16649) : we' ll test equality on each string instead. + + accepted=true + for s in $(echo "${!1}" | tr ' ' '\n'); do + if [ $2 = $s ] ; then + accepted=false + hadoop_debug " $s already contains $2" + fi + done + + if [ $accepted == true ]; then eval "$1"="'${!1} $3'" if [[ ${!1:0:1} = ' ' ]]; then #shellcheck disable=SC2140 eval "$1"="'${!1# }'" fi - hadoop_debug "$1 accepted $3" + hadoop_debug "$1 accepted $2 - $3" else - hadoop_debug "$1 declined $3" + hadoop_debug "skipped! " fi + } ## @description Register the given `shellprofile` to the Hadoop diff --git a/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_add_param.bats b/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_add_param.bats index 5d65db00ef5fa..5a591aac3a9aa 100644 --- a/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_add_param.bats +++ b/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_add_param.bats @@ -46,4 +46,21 @@ load hadoop-functions_test_helper echo ">${testvar}<" [ "${testvar}" = "foo bar baz" ] -} \ No newline at end of file +} + + +@test "hadoop_add_param (HADOOP-16649 a)" { + hadoop_add_param testvar hadoop-azure-datalake hadoop-azure-datalake + hadoop_add_param testvar hadoop-azure hadoop-azure + + echo ">${testvar}<" + [ "${testvar}" = "hadoop-azure-datalake hadoop-azure" ] +} +@test "hadoop_add_param (HADOOP-16649 b)" { + hadoop_add_param testvar hadoop-azure hadoop-azure + hadoop_add_param testvar hadoop-azure-datalake hadoop-azure-datalake + + echo ">${testvar}<" + [ "${testvar}" = "hadoop-azure hadoop-azure-datalake" ] +} + From d32656f35dbac289a73878d200a60ce787b62fb6 Mon Sep 17 00:00:00 2001 From: taccart Date: Tue, 20 Oct 2020 09:56:47 +0200 Subject: [PATCH 2/5] update code and test for add_param <-key> --- .../src/main/bin/hadoop-functions.sh | 31 ++++++++++++++----- .../src/test/scripts/hadoop_add_param.bats | 10 ++++++ 2 files changed, 33 insertions(+), 8 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh index f16c988aaabc0..1c5378dcdca53 100755 --- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh +++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh @@ -1135,28 +1135,43 @@ function hadoop_add_param # $1 is what we are adding to # $2 is the name of what we want to add (key) # $3 is the key+value of what we're adding - # + #hadoop_finalize_hadoop_heap # doing it this way allows us to support all sorts of # different syntaxes, just so long as they are space # delimited. # # testing with string regexp fails (see HADOOP-16649) : we' ll test equality on each string instead. + # reject if either key is already present in form of "-key[=]*" or value is already present accepted=true - for s in $(echo "${!1}" | tr ' ' '\n'); do - if [ $2 = $s ] ; then - accepted=false - hadoop_debug " $s already contains $2" - fi - done + if [[ ${3:0:1} == "-" ]]; then + # first char of value is a - (ex: -Xmx1024), we search for presence of "key (ex: -Xmx) + for s in $(echo "${!1}" | tr ' ' '\n'); do + key="${s%=*}" + if [[ $key =~ -$2 ]] ; then { + accepted=false + hadoop_debug "${!1} already contains key $2 ($key) of value $3" + } + fi + done + else + for s in $(echo "${!1}" | tr ' ' '\n'); do + # first char of value is NOT a - : we search for presence of value (ex: hadoop-azure) + if [[ "$3" = "$s" ]] ; then + accepted=false + hadoop_debug "${!1} already contains value $3" + fi + done + fi if [ $accepted == true ]; then + hadoop_debug "'${!1}' accepted new key '$2', value '$3'" eval "$1"="'${!1} $3'" if [[ ${!1:0:1} = ' ' ]]; then #shellcheck disable=SC2140 eval "$1"="'${!1# }'" fi - hadoop_debug "$1 accepted $2 - $3" + hadoop_debug "new string : '${!1}'" else hadoop_debug "skipped! " fi diff --git a/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_add_param.bats b/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_add_param.bats index 5a591aac3a9aa..e86bf0c76b7d4 100644 --- a/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_add_param.bats +++ b/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_add_param.bats @@ -56,6 +56,8 @@ load hadoop-functions_test_helper echo ">${testvar}<" [ "${testvar}" = "hadoop-azure-datalake hadoop-azure" ] } + + @test "hadoop_add_param (HADOOP-16649 b)" { hadoop_add_param testvar hadoop-azure hadoop-azure hadoop_add_param testvar hadoop-azure-datalake hadoop-azure-datalake @@ -64,3 +66,11 @@ load hadoop-functions_test_helper [ "${testvar}" = "hadoop-azure hadoop-azure-datalake" ] } +@test "hadoop_add_param (HADOOP-16649 c )" { + hadoop_add_param testvar Xmx -Xmx2048 + hadoop_add_param testvar Xmx -Xmx128 + hadoop_add_param testvar Xms -Xms32 + echo ">${testvar}<" + [ "${testvar}" = "-Xmx2048 -Xms32" ] +} + From 4fb084f997cbbbafcb8841f7c694f280f62dcc3b Mon Sep 17 00:00:00 2001 From: taccart Date: Tue, 20 Oct 2020 11:36:37 +0200 Subject: [PATCH 3/5] Ugly echo added to address shellcheck's SC2140 --- .../hadoop-common/src/main/bin/hadoop-functions.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh index 1c5378dcdca53..06f9cb6a803bf 100755 --- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh +++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh @@ -1166,7 +1166,7 @@ function hadoop_add_param fi if [ $accepted == true ]; then hadoop_debug "'${!1}' accepted new key '$2', value '$3'" - eval "$1"="'${!1} $3'" + eval "$1"="$(echo \"${!1} $3\")" if [[ ${!1:0:1} = ' ' ]]; then #shellcheck disable=SC2140 eval "$1"="'${!1# }'" From 6f3a6855560f02d4ed160cc75d28c8c20ff2e8fe Mon Sep 17 00:00:00 2001 From: taccart Date: Wed, 21 Oct 2020 08:46:25 +0200 Subject: [PATCH 4/5] Adress the ShellCheck SC2086 SC2116 --- .../hadoop-common/src/main/bin/hadoop-functions.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh index 06f9cb6a803bf..fd294cfda87b6 100755 --- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh +++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh @@ -1166,10 +1166,11 @@ function hadoop_add_param fi if [ $accepted == true ]; then hadoop_debug "'${!1}' accepted new key '$2', value '$3'" - eval "$1"="$(echo \"${!1} $3\")" + eval "$1"="\"${!1} $3\"" if [[ ${!1:0:1} = ' ' ]]; then #shellcheck disable=SC2140 - eval "$1"="'${!1# }'" + eval "$1"="\"${!1# }\"" + fi hadoop_debug "new string : '${!1}'" else From e595876ebae392e28357903cbe53af666b99d904 Mon Sep 17 00:00:00 2001 From: taccart Date: Wed, 21 Oct 2020 10:57:56 +0200 Subject: [PATCH 5/5] #shellcheck disable=SC2140 --- .../hadoop-common/src/main/bin/hadoop-functions.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh index fd294cfda87b6..e71a2a24f9863 100755 --- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh +++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh @@ -1166,6 +1166,7 @@ function hadoop_add_param fi if [ $accepted == true ]; then hadoop_debug "'${!1}' accepted new key '$2', value '$3'" + #shellcheck disable=SC2140 eval "$1"="\"${!1} $3\"" if [[ ${!1:0:1} = ' ' ]]; then #shellcheck disable=SC2140