From d99d6c8949cd1b54acce72b885de6d014b36e067 Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Fri, 27 Sep 2019 00:11:53 +0200 Subject: [PATCH 1/2] HDDS-2187. ozone-mr test fails with No FileSystem for scheme o3fs --- .../dist/src/main/compose/ozone-mr/hadoop27/docker-compose.yaml | 1 + .../dist/src/main/compose/ozone-mr/hadoop31/docker-compose.yaml | 1 + .../dist/src/main/compose/ozone-mr/hadoop32/docker-compose.yaml | 1 + .../dist/src/main/compose/ozonesecure-mr/docker-compose.yaml | 1 + hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-config | 1 - 5 files changed, 4 insertions(+), 1 deletion(-) diff --git a/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop27/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop27/docker-compose.yaml index a23566b8a4727..96e2504d64683 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop27/docker-compose.yaml +++ b/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop27/docker-compose.yaml @@ -63,6 +63,7 @@ services: - ../common-config environment: ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION + CORE-SITE.xml_fs.o3fs.impl: org.apache.hadoop.fs.ozone.BasicOzoneFileSystem command: ["/opt/hadoop/bin/ozone","scm"] rm: image: ${HADOOP_IMAGE}:${HADOOP_VERSION} diff --git a/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop31/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop31/docker-compose.yaml index af1c960918518..425aecc493be7 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop31/docker-compose.yaml +++ b/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop31/docker-compose.yaml @@ -63,6 +63,7 @@ services: - ../common-config environment: ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION + CORE-SITE.xml_fs.o3fs.impl: org.apache.hadoop.fs.ozone.OzoneFileSystem command: ["/opt/hadoop/bin/ozone","scm"] rm: image: ${HADOOP_IMAGE}:${HADOOP_VERSION} diff --git a/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop32/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop32/docker-compose.yaml index 755b279749ca2..e11a31efc82bd 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop32/docker-compose.yaml +++ b/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop32/docker-compose.yaml @@ -63,6 +63,7 @@ services: - ../common-config environment: ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION + CORE-SITE.xml_fs.o3fs.impl: org.apache.hadoop.fs.ozone.OzoneFileSystem command: ["/opt/hadoop/bin/ozone","scm"] rm: image: ${HADOOP_IMAGE}:${HADOOP_VERSION} diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-compose.yaml index 31f74f087edda..3af9cb531d0cc 100644 --- a/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-compose.yaml +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-compose.yaml @@ -50,6 +50,7 @@ services: - 9874:9874 environment: ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION + CORE-SITE.xml_fs.o3fs.impl: org.apache.hadoop.fs.ozone.OzoneFileSystem env_file: - docker-config command: ["/opt/hadoop/bin/ozone","om"] diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-config b/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-config index f5c5fbd230bda..a4ce236049744 100644 --- a/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-config @@ -62,7 +62,6 @@ HADOOP-POLICY.XML_org.apache.hadoop.yarn.server.api.ResourceTracker.acl=* HDFS-SITE.XML_rpc.metrics.quantile.enable=true HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300 -CORE-SITE.xml_fs.o3fs.impl=org.apache.hadoop.fs.ozone.OzoneFileSystem CORE-SITE.xml_fs.AbstractFileSystem.o3fs.impl=org.apache.hadoop.fs.ozone.OzFs CORE-SITE.xml_fs.defaultFS=o3fs://bucket1.vol1/ From 0aca2b212904edc1fd6842db0662413c202aa62d Mon Sep 17 00:00:00 2001 From: "Doroszlai, Attila" Date: Mon, 30 Sep 2019 10:49:10 +0200 Subject: [PATCH 2/2] HDDS-2187. Define FileSystem service for Ozone Tools --- .../ozone-mr/hadoop27/docker-compose.yaml | 1 - .../ozone-mr/hadoop31/docker-compose.yaml | 1 - .../ozone-mr/hadoop32/docker-compose.yaml | 1 - .../compose/ozonesecure-mr/docker-compose.yaml | 1 - .../main/compose/ozonesecure-mr/docker-config | 2 +- .../services/org.apache.hadoop.fs.FileSystem | 16 ++++++++++++++++ 6 files changed, 17 insertions(+), 5 deletions(-) create mode 100644 hadoop-ozone/tools/src/main/resources/META-INF/services/org.apache.hadoop.fs.FileSystem diff --git a/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop27/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop27/docker-compose.yaml index 96e2504d64683..a23566b8a4727 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop27/docker-compose.yaml +++ b/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop27/docker-compose.yaml @@ -63,7 +63,6 @@ services: - ../common-config environment: ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION - CORE-SITE.xml_fs.o3fs.impl: org.apache.hadoop.fs.ozone.BasicOzoneFileSystem command: ["/opt/hadoop/bin/ozone","scm"] rm: image: ${HADOOP_IMAGE}:${HADOOP_VERSION} diff --git a/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop31/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop31/docker-compose.yaml index 425aecc493be7..af1c960918518 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop31/docker-compose.yaml +++ b/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop31/docker-compose.yaml @@ -63,7 +63,6 @@ services: - ../common-config environment: ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION - CORE-SITE.xml_fs.o3fs.impl: org.apache.hadoop.fs.ozone.OzoneFileSystem command: ["/opt/hadoop/bin/ozone","scm"] rm: image: ${HADOOP_IMAGE}:${HADOOP_VERSION} diff --git a/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop32/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop32/docker-compose.yaml index e11a31efc82bd..755b279749ca2 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop32/docker-compose.yaml +++ b/hadoop-ozone/dist/src/main/compose/ozone-mr/hadoop32/docker-compose.yaml @@ -63,7 +63,6 @@ services: - ../common-config environment: ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION - CORE-SITE.xml_fs.o3fs.impl: org.apache.hadoop.fs.ozone.OzoneFileSystem command: ["/opt/hadoop/bin/ozone","scm"] rm: image: ${HADOOP_IMAGE}:${HADOOP_VERSION} diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-compose.yaml index 3af9cb531d0cc..31f74f087edda 100644 --- a/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-compose.yaml +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-compose.yaml @@ -50,7 +50,6 @@ services: - 9874:9874 environment: ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION - CORE-SITE.xml_fs.o3fs.impl: org.apache.hadoop.fs.ozone.OzoneFileSystem env_file: - docker-config command: ["/opt/hadoop/bin/ozone","om"] diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-config b/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-config index a4ce236049744..5bd3c9206c1ea 100644 --- a/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-config @@ -174,4 +174,4 @@ KERBEROS_SERVER=kdc JAVA_HOME=/usr/lib/jvm/jre JSVC_HOME=/usr/bin SLEEP_SECONDS=5 -KERBEROS_ENABLED=true \ No newline at end of file +KERBEROS_ENABLED=true diff --git a/hadoop-ozone/tools/src/main/resources/META-INF/services/org.apache.hadoop.fs.FileSystem b/hadoop-ozone/tools/src/main/resources/META-INF/services/org.apache.hadoop.fs.FileSystem new file mode 100644 index 0000000000000..03680027d539e --- /dev/null +++ b/hadoop-ozone/tools/src/main/resources/META-INF/services/org.apache.hadoop.fs.FileSystem @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +org.apache.hadoop.fs.ozone.OzoneFileSystem