From 225e56e4f62951f5d6244b20f46e9163959efca4 Mon Sep 17 00:00:00 2001 From: James Baiera Date: Fri, 18 Oct 2019 17:02:54 -0400 Subject: [PATCH 1/2] Fix CI failures due to hadoop distro --- .../org/elasticsearch/hadoop/gradle/BuildPlugin.groovy | 5 ++--- licenses/hadoop-client-2.2.0.jar.sha1 | 1 - licenses/hadoop-client-2.7.6.jar.sha1 | 1 + licenses/hadoop-common-2.2.0.jar.sha1 | 1 - licenses/hadoop-common-2.7.6.jar.sha1 | 1 + licenses/hadoop-mapreduce-client-core-2.2.0.jar.sha1 | 1 - licenses/hadoop-mapreduce-client-core-2.7.6.jar.sha1 | 1 + mr/licenses/hadoop-client-2.2.0.jar.sha1 | 1 - mr/licenses/hadoop-client-2.7.6.jar.sha1 | 1 + mr/licenses/hadoop-common-2.2.0.jar.sha1 | 1 - mr/licenses/hadoop-common-2.7.6.jar.sha1 | 1 + mr/licenses/hadoop-mapreduce-client-core-2.2.0.jar.sha1 | 1 - mr/licenses/hadoop-mapreduce-client-core-2.7.6.jar.sha1 | 1 + 13 files changed, 8 insertions(+), 9 deletions(-) delete mode 100644 licenses/hadoop-client-2.2.0.jar.sha1 create mode 100644 licenses/hadoop-client-2.7.6.jar.sha1 delete mode 100644 licenses/hadoop-common-2.2.0.jar.sha1 create mode 100644 licenses/hadoop-common-2.7.6.jar.sha1 delete mode 100644 licenses/hadoop-mapreduce-client-core-2.2.0.jar.sha1 create mode 100644 licenses/hadoop-mapreduce-client-core-2.7.6.jar.sha1 delete mode 100644 mr/licenses/hadoop-client-2.2.0.jar.sha1 create mode 100644 mr/licenses/hadoop-client-2.7.6.jar.sha1 delete mode 100644 mr/licenses/hadoop-common-2.2.0.jar.sha1 create mode 100644 mr/licenses/hadoop-common-2.7.6.jar.sha1 delete mode 100644 mr/licenses/hadoop-mapreduce-client-core-2.2.0.jar.sha1 create mode 100644 mr/licenses/hadoop-mapreduce-client-core-2.7.6.jar.sha1 diff --git a/buildSrc/src/main/groovy/org/elasticsearch/hadoop/gradle/BuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/hadoop/gradle/BuildPlugin.groovy index 86285c122..62ba9311f 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/hadoop/gradle/BuildPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/hadoop/gradle/BuildPlugin.groovy @@ -161,7 +161,7 @@ class BuildPlugin implements Plugin { // Hadoop versions project.rootProject.ext.hadoopClient = [] - project.rootProject.ext.hadoopDistro = project.hasProperty("distro") ? project.getProperty("distro") : "hadoopStable" + project.rootProject.ext.hadoopDistro = project.hasProperty("distro") ? project.getProperty("distro") : "hadoopYarn" switch (project.rootProject.ext.hadoopDistro) { // Hadoop YARN/2.0.x @@ -171,8 +171,7 @@ class BuildPlugin implements Plugin { project.rootProject.ext.hadoopClient = ["org.apache.hadoop:hadoop-client:$version"] println "Using Apache Hadoop on YARN [$version]" break - - default: + case "hadoopStable": String version = project.hadoop22Version project.rootProject.ext.hadoopVersion = version project.rootProject.ext.hadoopClient = ["org.apache.hadoop:hadoop-client:$version"] diff --git a/licenses/hadoop-client-2.2.0.jar.sha1 b/licenses/hadoop-client-2.2.0.jar.sha1 deleted file mode 100644 index a256d5fa7..000000000 --- a/licenses/hadoop-client-2.2.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -8bc8c12f1991d1145e40011c8a8935798948246e \ No newline at end of file diff --git a/licenses/hadoop-client-2.7.6.jar.sha1 b/licenses/hadoop-client-2.7.6.jar.sha1 new file mode 100644 index 000000000..ca970cbee --- /dev/null +++ b/licenses/hadoop-client-2.7.6.jar.sha1 @@ -0,0 +1 @@ +1e81abd78494a0f41f36178152c20e2deb3e9e8d \ No newline at end of file diff --git a/licenses/hadoop-common-2.2.0.jar.sha1 b/licenses/hadoop-common-2.2.0.jar.sha1 deleted file mode 100644 index 9267fbc3b..000000000 --- a/licenses/hadoop-common-2.2.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -a840350e4a1f5483f949ec5e72cfa2c6b2b3cf94 \ No newline at end of file diff --git a/licenses/hadoop-common-2.7.6.jar.sha1 b/licenses/hadoop-common-2.7.6.jar.sha1 new file mode 100644 index 000000000..79e8c537b --- /dev/null +++ b/licenses/hadoop-common-2.7.6.jar.sha1 @@ -0,0 +1 @@ +b3af0a73d6d3226cd46d4d29c6272a046b90a9f1 \ No newline at end of file diff --git a/licenses/hadoop-mapreduce-client-core-2.2.0.jar.sha1 b/licenses/hadoop-mapreduce-client-core-2.2.0.jar.sha1 deleted file mode 100644 index 66b7e7139..000000000 --- a/licenses/hadoop-mapreduce-client-core-2.2.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -4be274d45f35543d3c4dd8e2bfed2cebc56696c7 \ No newline at end of file diff --git a/licenses/hadoop-mapreduce-client-core-2.7.6.jar.sha1 b/licenses/hadoop-mapreduce-client-core-2.7.6.jar.sha1 new file mode 100644 index 000000000..c8291a1d5 --- /dev/null +++ b/licenses/hadoop-mapreduce-client-core-2.7.6.jar.sha1 @@ -0,0 +1 @@ +fb3b92f837656dbbd2ad5d96f02f09af67c59a17 \ No newline at end of file diff --git a/mr/licenses/hadoop-client-2.2.0.jar.sha1 b/mr/licenses/hadoop-client-2.2.0.jar.sha1 deleted file mode 100644 index a256d5fa7..000000000 --- a/mr/licenses/hadoop-client-2.2.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -8bc8c12f1991d1145e40011c8a8935798948246e \ No newline at end of file diff --git a/mr/licenses/hadoop-client-2.7.6.jar.sha1 b/mr/licenses/hadoop-client-2.7.6.jar.sha1 new file mode 100644 index 000000000..ca970cbee --- /dev/null +++ b/mr/licenses/hadoop-client-2.7.6.jar.sha1 @@ -0,0 +1 @@ +1e81abd78494a0f41f36178152c20e2deb3e9e8d \ No newline at end of file diff --git a/mr/licenses/hadoop-common-2.2.0.jar.sha1 b/mr/licenses/hadoop-common-2.2.0.jar.sha1 deleted file mode 100644 index 9267fbc3b..000000000 --- a/mr/licenses/hadoop-common-2.2.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -a840350e4a1f5483f949ec5e72cfa2c6b2b3cf94 \ No newline at end of file diff --git a/mr/licenses/hadoop-common-2.7.6.jar.sha1 b/mr/licenses/hadoop-common-2.7.6.jar.sha1 new file mode 100644 index 000000000..79e8c537b --- /dev/null +++ b/mr/licenses/hadoop-common-2.7.6.jar.sha1 @@ -0,0 +1 @@ +b3af0a73d6d3226cd46d4d29c6272a046b90a9f1 \ No newline at end of file diff --git a/mr/licenses/hadoop-mapreduce-client-core-2.2.0.jar.sha1 b/mr/licenses/hadoop-mapreduce-client-core-2.2.0.jar.sha1 deleted file mode 100644 index 66b7e7139..000000000 --- a/mr/licenses/hadoop-mapreduce-client-core-2.2.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -4be274d45f35543d3c4dd8e2bfed2cebc56696c7 \ No newline at end of file diff --git a/mr/licenses/hadoop-mapreduce-client-core-2.7.6.jar.sha1 b/mr/licenses/hadoop-mapreduce-client-core-2.7.6.jar.sha1 new file mode 100644 index 000000000..c8291a1d5 --- /dev/null +++ b/mr/licenses/hadoop-mapreduce-client-core-2.7.6.jar.sha1 @@ -0,0 +1 @@ +fb3b92f837656dbbd2ad5d96f02f09af67c59a17 \ No newline at end of file From 28f45e2377d7755bb1562b8f3040fb547851a7f5 Mon Sep 17 00:00:00 2001 From: James Baiera Date: Thu, 24 Oct 2019 16:00:46 -0400 Subject: [PATCH 2/2] Pull out common code from switch block. Add default case. --- .../hadoop/gradle/BuildPlugin.groovy | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/buildSrc/src/main/groovy/org/elasticsearch/hadoop/gradle/BuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/hadoop/gradle/BuildPlugin.groovy index 62ba9311f..9f5442fba 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/hadoop/gradle/BuildPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/hadoop/gradle/BuildPlugin.groovy @@ -160,23 +160,21 @@ class BuildPlugin implements Plugin { println "Using Gradle [${project.gradle.gradleVersion}]" // Hadoop versions - project.rootProject.ext.hadoopClient = [] project.rootProject.ext.hadoopDistro = project.hasProperty("distro") ? project.getProperty("distro") : "hadoopYarn" - switch (project.rootProject.ext.hadoopDistro) { // Hadoop YARN/2.0.x case "hadoopYarn": - String version = project.hadoop2Version - project.rootProject.ext.hadoopVersion = version - project.rootProject.ext.hadoopClient = ["org.apache.hadoop:hadoop-client:$version"] - println "Using Apache Hadoop on YARN [$version]" + project.rootProject.ext.hadoopVersion = project.hadoop2Version + println "Using Apache Hadoop on YARN [$project.hadoop2Version]" break case "hadoopStable": - String version = project.hadoop22Version - project.rootProject.ext.hadoopVersion = version - project.rootProject.ext.hadoopClient = ["org.apache.hadoop:hadoop-client:$version"] - println "Using Apache Hadoop [$version]" + project.rootProject.ext.hadoopVersion = project.hadoop22Version + println "Using Apache Hadoop [$project.hadoop22Version]" + break + default: + throw new GradleException("Invalid [hadoopDistro] setting: [$project.rootProject.ext.hadoopDistro]") } + project.rootProject.ext.hadoopClient = ["org.apache.hadoop:hadoop-client:$project.rootProject.ext.hadoopVersion"] } project.ext.eshadoopVersion = project.rootProject.ext.eshadoopVersion project.ext.elasticsearchVersion = project.rootProject.ext.elasticsearchVersion