From 417bb1ce18f458ae8324150f3ca95476ed8a04f4 Mon Sep 17 00:00:00 2001 From: Vivek Ratnavel Subramanian Date: Tue, 5 Mar 2019 17:38:49 -0800 Subject: [PATCH 1/3] HDDS-1226. ozone-filesystem jar missing in hadoop classpath --- hadoop-ozone/dist/dev-support/bin/dist-layout-stitching | 3 +++ 1 file changed, 3 insertions(+) diff --git a/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching b/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching index dc2819b6f8346..377540afb3ba1 100755 --- a/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching +++ b/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching @@ -110,6 +110,9 @@ run cp "${ROOT}/hadoop-ozone/common/src/main/bin/stop-ozone.sh" "sbin/" run mkdir -p "./share/hadoop/ozoneplugin" run cp "${ROOT}/hadoop-ozone/objectstore-service/target/hadoop-ozone-objectstore-service-${HDDS_VERSION}-plugin.jar" "./share/hadoop/ozoneplugin/hadoop-ozone-datanode-plugin-${HDDS_VERSION}.jar" +#shaded ozone filesystem service +run mkdir -p "./share/hadoop/ozonefs" +run cp "${ROOT}/hadoop-ozone/ozonefs-lib/target/hadoop-ozone-filesystem-lib-${HDDS_VERSION}.jar" "./share/hadoop/ozonefs/hadoop-ozone-filesystem-lib-${HDDS_VERSION}.jar" # Optional documentation, could be missing cp -r "${ROOT}/hadoop-hdds/docs/target/classes/docs" ./ From 211a5ddc31cfd68bcf502b095bb0198d00c7a54f Mon Sep 17 00:00:00 2001 From: Vivek Ratnavel Subramanian Date: Wed, 6 Mar 2019 12:59:19 -0800 Subject: [PATCH 2/3] refactor ozonefs-lib --- hadoop-hdds/docs/content/OzoneFS.md | 4 ++-- hadoop-ozone/dist/dev-support/bin/dist-layout-stitching | 4 ---- hadoop-ozone/dist/pom.xml | 2 +- .../dist/src/main/compose/ozonefs/docker-compose.yaml | 2 +- hadoop-ozone/{ozonefs-lib => ozonefs-lib-current}/pom.xml | 2 +- hadoop-ozone/pom.xml | 4 ++-- 6 files changed, 7 insertions(+), 11 deletions(-) rename hadoop-ozone/{ozonefs-lib => ozonefs-lib-current}/pom.xml (98%) diff --git a/hadoop-hdds/docs/content/OzoneFS.md b/hadoop-hdds/docs/content/OzoneFS.md index b7f8a742297ec..84884341ecc5b 100644 --- a/hadoop-hdds/docs/content/OzoneFS.md +++ b/hadoop-hdds/docs/content/OzoneFS.md @@ -56,7 +56,7 @@ This will make this bucket to be the default file system for HDFS dfs commands a You also need to add the ozone-filesystem.jar file to the classpath: {{< highlight bash >}} -export HADOOP_CLASSPATH=/opt/ozone/share/ozonefs/lib/hadoop-ozone-filesystem-lib-.*.jar:$HADOOP_CLASSPATH +export HADOOP_CLASSPATH=/opt/ozone/share/ozonefs/lib/hadoop-ozone-filesystem-lib-current.*.jar:$HADOOP_CLASSPATH {{< /highlight >}} @@ -82,7 +82,7 @@ Please note that any keys created/deleted in the bucket using methods apart from There are two ozonefs files which includes all the dependencies: - * share/ozone/lib/hadoop-ozone-filesystem-lib-VERSION.jar + * share/ozone/lib/hadoop-ozone-filesystem-lib-current-VERSION.jar * share/ozone/lib/hadoop-ozone-filesystem-lib-legacy-VERSION.jar The first one contains all the required dependency to use ozonefs with a diff --git a/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching b/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching index 377540afb3ba1..9a470d5480023 100755 --- a/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching +++ b/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching @@ -110,10 +110,6 @@ run cp "${ROOT}/hadoop-ozone/common/src/main/bin/stop-ozone.sh" "sbin/" run mkdir -p "./share/hadoop/ozoneplugin" run cp "${ROOT}/hadoop-ozone/objectstore-service/target/hadoop-ozone-objectstore-service-${HDDS_VERSION}-plugin.jar" "./share/hadoop/ozoneplugin/hadoop-ozone-datanode-plugin-${HDDS_VERSION}.jar" -#shaded ozone filesystem service -run mkdir -p "./share/hadoop/ozonefs" -run cp "${ROOT}/hadoop-ozone/ozonefs-lib/target/hadoop-ozone-filesystem-lib-${HDDS_VERSION}.jar" "./share/hadoop/ozonefs/hadoop-ozone-filesystem-lib-${HDDS_VERSION}.jar" - # Optional documentation, could be missing cp -r "${ROOT}/hadoop-hdds/docs/target/classes/docs" ./ diff --git a/hadoop-ozone/dist/pom.xml b/hadoop-ozone/dist/pom.xml index e66bbbe90980f..5fa3bf7d2f1a4 100644 --- a/hadoop-ozone/dist/pom.xml +++ b/hadoop-ozone/dist/pom.xml @@ -204,7 +204,7 @@ org.apache.hadoop - hadoop-ozone-filesystem-lib + hadoop-ozone-filesystem-lib-current org.apache.hadoop diff --git a/hadoop-ozone/dist/src/main/compose/ozonefs/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozonefs/docker-compose.yaml index 22055fce5b1eb..93830f4101041 100644 --- a/hadoop-ozone/dist/src/main/compose/ozonefs/docker-compose.yaml +++ b/hadoop-ozone/dist/src/main/compose/ozonefs/docker-compose.yaml @@ -56,7 +56,7 @@ services: env_file: - ./docker-config environment: - HADOOP_CLASSPATH: /opt/ozone/share/ozone/lib/hadoop-ozone-filesystem-lib-0*.jar + HADOOP_CLASSPATH: /opt/ozone/share/ozone/lib/hadoop-ozone-filesystem-lib-current*.jar command: ["watch","-n","100000","ls"] hadoop2: image: flokkr/hadoop:2.9.0 diff --git a/hadoop-ozone/ozonefs-lib/pom.xml b/hadoop-ozone/ozonefs-lib-current/pom.xml similarity index 98% rename from hadoop-ozone/ozonefs-lib/pom.xml rename to hadoop-ozone/ozonefs-lib-current/pom.xml index b1cbaf872d39b..2fe09cb30d0b9 100644 --- a/hadoop-ozone/ozonefs-lib/pom.xml +++ b/hadoop-ozone/ozonefs-lib-current/pom.xml @@ -21,7 +21,7 @@ hadoop-ozone 0.4.0-SNAPSHOT - hadoop-ozone-filesystem-lib + hadoop-ozone-filesystem-lib-current Apache Hadoop Ozone FileSystem Single Jar Library jar This projects creates an uber jar from ozonefs with all the diff --git a/hadoop-ozone/pom.xml b/hadoop-ozone/pom.xml index ac20f1e8d5cb2..6f9ab659aaa17 100644 --- a/hadoop-ozone/pom.xml +++ b/hadoop-ozone/pom.xml @@ -40,7 +40,7 @@ client ozone-manager ozonefs - ozonefs-lib + ozonefs-lib-current ozonefs-lib-legacy tools integration-test @@ -102,7 +102,7 @@ org.apache.hadoop - hadoop-ozone-filesystem-lib + hadoop-ozone-filesystem-lib-current ${ozone.version} From 9dda5cf50e5d3fad6003fbfd630e14b142b00811 Mon Sep 17 00:00:00 2001 From: Vivek Ratnavel Subramanian Date: Wed, 6 Mar 2019 21:43:59 -0800 Subject: [PATCH 3/3] fix typos --- hadoop-hdds/common/src/main/resources/ozone-default.xml | 2 +- hadoop-hdds/docs/content/OzoneFS.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml index a4f49e762b31a..82b9f2c1ff6f3 100644 --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml @@ -1907,7 +1907,7 @@ an isolated classloader. Default depends from the used jar. true for ozone-filesystem-lib-legacy - jar and false for the ozone-filesystem-lib.jar + jar and false for the ozone-filesystem-lib-current.jar diff --git a/hadoop-hdds/docs/content/OzoneFS.md b/hadoop-hdds/docs/content/OzoneFS.md index 84884341ecc5b..23d12b368ba9a 100644 --- a/hadoop-hdds/docs/content/OzoneFS.md +++ b/hadoop-hdds/docs/content/OzoneFS.md @@ -56,7 +56,7 @@ This will make this bucket to be the default file system for HDFS dfs commands a You also need to add the ozone-filesystem.jar file to the classpath: {{< highlight bash >}} -export HADOOP_CLASSPATH=/opt/ozone/share/ozonefs/lib/hadoop-ozone-filesystem-lib-current.*.jar:$HADOOP_CLASSPATH +export HADOOP_CLASSPATH=/opt/ozone/share/ozonefs/lib/hadoop-ozone-filesystem-lib-current*.jar:$HADOOP_CLASSPATH {{< /highlight >}}