From da969ccd54884e996f1b729dbf7c9ddc6723fcdd Mon Sep 17 00:00:00 2001 From: Bharat Viswanadham Date: Thu, 15 Aug 2019 14:21:56 -0700 Subject: [PATCH 1/6] HDDS-1972. Provide example ha proxy with multiple s3 servers back end. --- .../src/main/compose/ozones3-haproxy/.env | 18 ++++ .../ozones3-haproxy/docker-compose.yaml | 83 +++++++++++++++++++ .../compose/ozones3-haproxy/docker-config | 79 ++++++++++++++++++ .../ozones3-haproxy/haproxy-conf/haproxy.cfg | 22 +++++ .../src/main/compose/ozones3-haproxy/test.sh | 32 +++++++ 5 files changed, 234 insertions(+) create mode 100644 hadoop-ozone/dist/src/main/compose/ozones3-haproxy/.env create mode 100644 hadoop-ozone/dist/src/main/compose/ozones3-haproxy/docker-compose.yaml create mode 100644 hadoop-ozone/dist/src/main/compose/ozones3-haproxy/docker-config create mode 100644 hadoop-ozone/dist/src/main/compose/ozones3-haproxy/haproxy-conf/haproxy.cfg create mode 100755 hadoop-ozone/dist/src/main/compose/ozones3-haproxy/test.sh diff --git a/hadoop-ozone/dist/src/main/compose/ozones3-haproxy/.env b/hadoop-ozone/dist/src/main/compose/ozones3-haproxy/.env new file mode 100644 index 0000000000000..8753b1dc9bb01 --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/ozones3-haproxy/.env @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +HDDS_VERSION=${hdds.version} +HADOOP_RUNNER_VERSION=${docker.ozone-runner.version} \ No newline at end of file diff --git a/hadoop-ozone/dist/src/main/compose/ozones3-haproxy/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozones3-haproxy/docker-compose.yaml new file mode 100644 index 0000000000000..45d043b298a16 --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/ozones3-haproxy/docker-compose.yaml @@ -0,0 +1,83 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: "3" +services: + s3-proxy: + image: haproxy:latest + volumes: + - ../..:/opt/hadoop + - ./haproxy-conf/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg + ports: + - 8081:5001 + datanode: + image: apache/ozone-runner:${HADOOP_RUNNER_VERSION} + volumes: + - ../..:/opt/hadoop + ports: + - 9864 + command: ["ozone","datanode"] + env_file: + - ./docker-config + om: + image: apache/ozone-runner:${HADOOP_RUNNER_VERSION} + volumes: + - ../..:/opt/hadoop + ports: + - 9874:9874 + environment: + ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION + env_file: + - ./docker-config + command: ["ozone","om"] + scm: + image: apache/ozone-runner:${HADOOP_RUNNER_VERSION} + volumes: + - ../..:/opt/hadoop + ports: + - 9876:9876 + env_file: + - ./docker-config + environment: + ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION + command: ["ozone","scm"] + s3g1: + image: apache/ozone-runner:${HADOOP_RUNNER_VERSION} + volumes: + - ../..:/opt/hadoop + ports: + - 9878:9878 + env_file: + - ./docker-config + command: ["ozone","s3g"] + s3g2: + image: apache/ozone-runner:${HADOOP_RUNNER_VERSION} + volumes: + - ../..:/opt/hadoop + ports: + - 9879:9878 + env_file: + - ./docker-config + command: ["ozone","s3g"] + s3g3: + image: apache/ozone-runner:${HADOOP_RUNNER_VERSION} + volumes: + - ../..:/opt/hadoop + ports: + - 9880:9878 + env_file: + - ./docker-config + command: ["ozone","s3g"] \ No newline at end of file diff --git a/hadoop-ozone/dist/src/main/compose/ozones3-haproxy/docker-config b/hadoop-ozone/dist/src/main/compose/ozones3-haproxy/docker-config new file mode 100644 index 0000000000000..4ffe9a6674c7f --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/ozones3-haproxy/docker-config @@ -0,0 +1,79 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +OZONE-SITE.XML_ozone.om.address=om +OZONE-SITE.XML_ozone.scm.names=scm +OZONE-SITE.XML_ozone.enabled=true +OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data +OZONE-SITE.XML_ozone.scm.block.client.address=scm +OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata +OZONE-SITE.XML_ozone.scm.client.address=scm +OZONE-SITE.XML_ozone.replication=1 +OZONE-SITE.XML_hdds.datanode.dir=/data/hdds + +HDFS-SITE.XML_rpc.metrics.quantile.enable=true +HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300 +LOG4J.PROPERTIES_log4j.rootLogger=INFO, stdout +LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender +LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n +LOG4J.PROPERTIES_log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR +LOG4J.PROPERTIES_log4j.logger.org.apache.ratis.conf.ConfUtils=WARN +LOG4J.PROPERTIES_log4j.logger.org.apache.hadoop.security.ShellBasedUnixGroupsMapping=ERROR +LOG4J.PROPERTIES_log4j.logger.http.requests.s3gateway=INFO,s3gatewayrequestlog +LOG4J.PROPERTIES_log4j.appender.s3gatewayrequestlog=org.apache.hadoop.http.HttpRequestLogAppender +LOG4J.PROPERTIES_log4j.appender.s3gatewayrequestlog.Filename=/tmp/jetty-s3gateway-yyyy_mm_dd.log +LOG4J.PROPERTIES_log4j.appender.s3gatewayrequestlog.RetainDays=3 + +#Enable this variable to print out all hadoop rpc traffic to the stdout. See http://byteman.jboss.org/ to define your own instrumentation. +#BYTEMAN_SCRIPT_URL=https://raw.githubusercontent.com/apache/hadoop/trunk/dev-support/byteman/hadooprpc.btm + +#LOG4J2.PROPERTIES_* are for Ozone Audit Logging +LOG4J2.PROPERTIES_monitorInterval=30 +LOG4J2.PROPERTIES_filter=read,write +LOG4J2.PROPERTIES_filter.read.type=MarkerFilter +LOG4J2.PROPERTIES_filter.read.marker=READ +LOG4J2.PROPERTIES_filter.read.onMatch=DENY +LOG4J2.PROPERTIES_filter.read.onMismatch=NEUTRAL +LOG4J2.PROPERTIES_filter.write.type=MarkerFilter +LOG4J2.PROPERTIES_filter.write.marker=WRITE +LOG4J2.PROPERTIES_filter.write.onMatch=NEUTRAL +LOG4J2.PROPERTIES_filter.write.onMismatch=NEUTRAL +LOG4J2.PROPERTIES_appenders=console, rolling +LOG4J2.PROPERTIES_appender.console.type=Console +LOG4J2.PROPERTIES_appender.console.name=STDOUT +LOG4J2.PROPERTIES_appender.console.layout.type=PatternLayout +LOG4J2.PROPERTIES_appender.console.layout.pattern=%d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n +LOG4J2.PROPERTIES_appender.rolling.type=RollingFile +LOG4J2.PROPERTIES_appender.rolling.name=RollingFile +LOG4J2.PROPERTIES_appender.rolling.fileName=${sys:hadoop.log.dir}/om-audit-${hostName}.log +LOG4J2.PROPERTIES_appender.rolling.filePattern=${sys:hadoop.log.dir}/om-audit-${hostName}-%d{yyyy-MM-dd-HH-mm-ss}-%i.log.gz +LOG4J2.PROPERTIES_appender.rolling.layout.type=PatternLayout +LOG4J2.PROPERTIES_appender.rolling.layout.pattern=%d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n +LOG4J2.PROPERTIES_appender.rolling.policies.type=Policies +LOG4J2.PROPERTIES_appender.rolling.policies.time.type=TimeBasedTriggeringPolicy +LOG4J2.PROPERTIES_appender.rolling.policies.time.interval=86400 +LOG4J2.PROPERTIES_appender.rolling.policies.size.type=SizeBasedTriggeringPolicy +LOG4J2.PROPERTIES_appender.rolling.policies.size.size=64MB +LOG4J2.PROPERTIES_loggers=audit +LOG4J2.PROPERTIES_logger.audit.type=AsyncLogger +LOG4J2.PROPERTIES_logger.audit.name=OMAudit +LOG4J2.PROPERTIES_logger.audit.level=INFO +LOG4J2.PROPERTIES_logger.audit.appenderRefs=rolling +LOG4J2.PROPERTIES_logger.audit.appenderRef.file.ref=RollingFile +LOG4J2.PROPERTIES_rootLogger.level=INFO +LOG4J2.PROPERTIES_rootLogger.appenderRefs=stdout +LOG4J2.PROPERTIES_rootLogger.appenderRef.stdout.ref=STDOUT diff --git a/hadoop-ozone/dist/src/main/compose/ozones3-haproxy/haproxy-conf/haproxy.cfg b/hadoop-ozone/dist/src/main/compose/ozones3-haproxy/haproxy-conf/haproxy.cfg new file mode 100644 index 0000000000000..937bd7b63183e --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/ozones3-haproxy/haproxy-conf/haproxy.cfg @@ -0,0 +1,22 @@ +# Simple configuration for an HTTP proxy listening on port 5001 on all +# interfaces and forwarding requests to a multiple multiple S3 servers in round +# robin fashion. +global + daemon + maxconn 256 + +defaults + mode http + timeout connect 5000ms + timeout client 50000ms + timeout server 50000ms + +frontend http-in + bind *:5001 + default_backend servers + +backend servers + balance roundrobin + server server1 s3g1:9878 maxconn 32 + server server2 s3g2:9878 maxconn 32 + server server3 s3g3:9878 maxconn 32 \ No newline at end of file diff --git a/hadoop-ozone/dist/src/main/compose/ozones3-haproxy/test.sh b/hadoop-ozone/dist/src/main/compose/ozones3-haproxy/test.sh new file mode 100755 index 0000000000000..0160da9e38292 --- /dev/null +++ b/hadoop-ozone/dist/src/main/compose/ozones3-haproxy/test.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +export COMPOSE_DIR + +# shellcheck source=/dev/null +source "$COMPOSE_DIR/../testlib.sh" + +start_docker_env + +execute_robot_test scm basic/basic.robot + +execute_robot_test scm s3 + +stop_docker_env + +generate_report From b6855d87805de4d3eb2df6e1fbec3842a927c587 Mon Sep 17 00:00:00 2001 From: Bharat Viswanadham Date: Fri, 16 Aug 2019 08:32:16 -0700 Subject: [PATCH 2/6] Apply suggestions from code review Co-Authored-By: Doroszlai, Attila <6454655+adoroszlai@users.noreply.github.com> --- .../src/main/compose/ozones3-haproxy/docker-compose.yaml | 6 +++--- .../main/compose/ozones3-haproxy/haproxy-conf/haproxy.cfg | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/hadoop-ozone/dist/src/main/compose/ozones3-haproxy/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozones3-haproxy/docker-compose.yaml index 45d043b298a16..cb9b34773e47a 100644 --- a/hadoop-ozone/dist/src/main/compose/ozones3-haproxy/docker-compose.yaml +++ b/hadoop-ozone/dist/src/main/compose/ozones3-haproxy/docker-compose.yaml @@ -16,13 +16,13 @@ version: "3" services: - s3-proxy: + s3g: image: haproxy:latest volumes: - ../..:/opt/hadoop - ./haproxy-conf/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg ports: - - 8081:5001 + - 8081:9878 datanode: image: apache/ozone-runner:${HADOOP_RUNNER_VERSION} volumes: @@ -80,4 +80,4 @@ services: - 9880:9878 env_file: - ./docker-config - command: ["ozone","s3g"] \ No newline at end of file + command: ["ozone","s3g"] diff --git a/hadoop-ozone/dist/src/main/compose/ozones3-haproxy/haproxy-conf/haproxy.cfg b/hadoop-ozone/dist/src/main/compose/ozones3-haproxy/haproxy-conf/haproxy.cfg index 937bd7b63183e..ab52689fd9fce 100644 --- a/hadoop-ozone/dist/src/main/compose/ozones3-haproxy/haproxy-conf/haproxy.cfg +++ b/hadoop-ozone/dist/src/main/compose/ozones3-haproxy/haproxy-conf/haproxy.cfg @@ -1,4 +1,4 @@ -# Simple configuration for an HTTP proxy listening on port 5001 on all +# Simple configuration for an HTTP proxy listening on port 9878 on all # interfaces and forwarding requests to a multiple multiple S3 servers in round # robin fashion. global @@ -12,11 +12,11 @@ defaults timeout server 50000ms frontend http-in - bind *:5001 + bind *:9878 default_backend servers backend servers balance roundrobin server server1 s3g1:9878 maxconn 32 server server2 s3g2:9878 maxconn 32 - server server3 s3g3:9878 maxconn 32 \ No newline at end of file + server server3 s3g3:9878 maxconn 32 From 2776de9db8179ff6048ee7dcef7566392c1977ee Mon Sep 17 00:00:00 2001 From: Bharat Viswanadham Date: Fri, 16 Aug 2019 08:33:33 -0700 Subject: [PATCH 3/6] Change S3 gateway ports. As s3g port is used for s3 proxy. --- .../src/main/compose/ozones3-haproxy/docker-compose.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hadoop-ozone/dist/src/main/compose/ozones3-haproxy/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozones3-haproxy/docker-compose.yaml index cb9b34773e47a..504b32fa42d25 100644 --- a/hadoop-ozone/dist/src/main/compose/ozones3-haproxy/docker-compose.yaml +++ b/hadoop-ozone/dist/src/main/compose/ozones3-haproxy/docker-compose.yaml @@ -59,7 +59,7 @@ services: volumes: - ../..:/opt/hadoop ports: - - 9878:9878 + - 9879:9878 env_file: - ./docker-config command: ["ozone","s3g"] @@ -68,7 +68,7 @@ services: volumes: - ../..:/opt/hadoop ports: - - 9879:9878 + - 9880:9878 env_file: - ./docker-config command: ["ozone","s3g"] @@ -77,7 +77,7 @@ services: volumes: - ../..:/opt/hadoop ports: - - 9880:9878 + - 9881:9878 env_file: - ./docker-config command: ["ozone","s3g"] From 7ab3179eeeb4fce7eac1370182c61842ddbb777b Mon Sep 17 00:00:00 2001 From: Bharat Viswanadham Date: Fri, 16 Aug 2019 09:36:05 -0700 Subject: [PATCH 4/6] Trigger notification From 288eef646aae0c16a1c1234f51b286a9cb29ca54 Mon Sep 17 00:00:00 2001 From: Bharat Viswanadham Date: Sun, 18 Aug 2019 18:25:21 -0700 Subject: [PATCH 5/6] Update hadoop-ozone/dist/src/main/compose/ozones3-haproxy/docker-compose.yaml Co-Authored-By: Doroszlai, Attila <6454655+adoroszlai@users.noreply.github.com> --- .../dist/src/main/compose/ozones3-haproxy/docker-compose.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-ozone/dist/src/main/compose/ozones3-haproxy/docker-compose.yaml b/hadoop-ozone/dist/src/main/compose/ozones3-haproxy/docker-compose.yaml index 504b32fa42d25..829792950e593 100644 --- a/hadoop-ozone/dist/src/main/compose/ozones3-haproxy/docker-compose.yaml +++ b/hadoop-ozone/dist/src/main/compose/ozones3-haproxy/docker-compose.yaml @@ -22,7 +22,7 @@ services: - ../..:/opt/hadoop - ./haproxy-conf/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg ports: - - 8081:9878 + - 9878:9878 datanode: image: apache/ozone-runner:${HADOOP_RUNNER_VERSION} volumes: From 682e36752d456eddad021821df04ad2f2993bb4d Mon Sep 17 00:00:00 2001 From: Bharat Viswanadham Date: Mon, 19 Aug 2019 10:34:10 -0700 Subject: [PATCH 6/6] remove running of s3 test in s3-proxy. --- hadoop-ozone/dist/src/main/compose/ozones3-haproxy/test.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/hadoop-ozone/dist/src/main/compose/ozones3-haproxy/test.sh b/hadoop-ozone/dist/src/main/compose/ozones3-haproxy/test.sh index 0160da9e38292..f4bfcc3d57f99 100755 --- a/hadoop-ozone/dist/src/main/compose/ozones3-haproxy/test.sh +++ b/hadoop-ozone/dist/src/main/compose/ozones3-haproxy/test.sh @@ -25,8 +25,6 @@ start_docker_env execute_robot_test scm basic/basic.robot -execute_robot_test scm s3 - stop_docker_env generate_report