Skip to content

Commit 551fd71

Browse files
bharatviswa504bharatviswa504
committed
HDDS-1972. Provide example ha proxy with multiple s3 servers back end. (#1304)
(cherry picked from commit 4f925af)
1 parent e981204 commit 551fd71

File tree

5 files changed

+232
-0
lines changed

5 files changed

+232
-0
lines changed

‎hadoop-ozone/dist/src/main/compose/ozones3-haproxy/.env‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
HDDS_VERSION=${hdds.version}
18+
HADOOP_RUNNER_VERSION=${docker.ozone-runner.version}

‎hadoop-ozone/dist/src/main/compose/ozones3-haproxy/docker-compose.yaml‎

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
version: "3"
18+
services:
19+
s3g:
20+
image: haproxy:latest
21+
volumes:
22+
- ../..:/opt/hadoop
23+
- ./haproxy-conf/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg
24+
ports:
25+
- 9878:9878
26+
datanode:
27+
image: apache/ozone-runner:${HADOOP_RUNNER_VERSION}
28+
volumes:
29+
- ../..:/opt/hadoop
30+
ports:
31+
- 9864
32+
command: ["ozone","datanode"]
33+
env_file:
34+
- ./docker-config
35+
om:
36+
image: apache/ozone-runner:${HADOOP_RUNNER_VERSION}
37+
volumes:
38+
- ../..:/opt/hadoop
39+
ports:
40+
- 9874:9874
41+
environment:
42+
ENSURE_OM_INITIALIZED: /data/metadata/om/current/VERSION
43+
env_file:
44+
- ./docker-config
45+
command: ["ozone","om"]
46+
scm:
47+
image: apache/ozone-runner:${HADOOP_RUNNER_VERSION}
48+
volumes:
49+
- ../..:/opt/hadoop
50+
ports:
51+
- 9876:9876
52+
env_file:
53+
- ./docker-config
54+
environment:
55+
ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION
56+
command: ["ozone","scm"]
57+
s3g1:
58+
image: apache/ozone-runner:${HADOOP_RUNNER_VERSION}
59+
volumes:
60+
- ../..:/opt/hadoop
61+
ports:
62+
- 9879:9878
63+
env_file:
64+
- ./docker-config
65+
command: ["ozone","s3g"]
66+
s3g2:
67+
image: apache/ozone-runner:${HADOOP_RUNNER_VERSION}
68+
volumes:
69+
- ../..:/opt/hadoop
70+
ports:
71+
- 9880:9878
72+
env_file:
73+
- ./docker-config
74+
command: ["ozone","s3g"]
75+
s3g3:
76+
image: apache/ozone-runner:${HADOOP_RUNNER_VERSION}
77+
volumes:
78+
- ../..:/opt/hadoop
79+
ports:
80+
- 9881:9878
81+
env_file:
82+
- ./docker-config
83+
command: ["ozone","s3g"]

‎hadoop-ozone/dist/src/main/compose/ozones3-haproxy/docker-config‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
OZONE-SITE.XML_ozone.om.address=om
18+
OZONE-SITE.XML_ozone.scm.names=scm
19+
OZONE-SITE.XML_ozone.enabled=true
20+
OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data
21+
OZONE-SITE.XML_ozone.scm.block.client.address=scm
22+
OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata
23+
OZONE-SITE.XML_ozone.scm.client.address=scm
24+
OZONE-SITE.XML_ozone.replication=1
25+
OZONE-SITE.XML_hdds.datanode.dir=/data/hdds
26+
27+
HDFS-SITE.XML_rpc.metrics.quantile.enable=true
28+
HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300
29+
LOG4J.PROPERTIES_log4j.rootLogger=INFO, stdout
30+
LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender
31+
LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
32+
LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
33+
LOG4J.PROPERTIES_log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR
34+
LOG4J.PROPERTIES_log4j.logger.org.apache.ratis.conf.ConfUtils=WARN
35+
LOG4J.PROPERTIES_log4j.logger.org.apache.hadoop.security.ShellBasedUnixGroupsMapping=ERROR
36+
LOG4J.PROPERTIES_log4j.logger.http.requests.s3gateway=INFO,s3gatewayrequestlog
37+
LOG4J.PROPERTIES_log4j.appender.s3gatewayrequestlog=org.apache.hadoop.http.HttpRequestLogAppender
38+
LOG4J.PROPERTIES_log4j.appender.s3gatewayrequestlog.Filename=/tmp/jetty-s3gateway-yyyy_mm_dd.log
39+
LOG4J.PROPERTIES_log4j.appender.s3gatewayrequestlog.RetainDays=3
40+
41+
#Enable this variable to print out all hadoop rpc traffic to the stdout. See http://byteman.jboss.org/ to define your own instrumentation.
42+
#BYTEMAN_SCRIPT_URL=https://raw.githubusercontent.com/apache/hadoop/trunk/dev-support/byteman/hadooprpc.btm
43+
44+
#LOG4J2.PROPERTIES_* are for Ozone Audit Logging
45+
LOG4J2.PROPERTIES_monitorInterval=30
46+
LOG4J2.PROPERTIES_filter=read,write
47+
LOG4J2.PROPERTIES_filter.read.type=MarkerFilter
48+
LOG4J2.PROPERTIES_filter.read.marker=READ
49+
LOG4J2.PROPERTIES_filter.read.onMatch=DENY
50+
LOG4J2.PROPERTIES_filter.read.onMismatch=NEUTRAL
51+
LOG4J2.PROPERTIES_filter.write.type=MarkerFilter
52+
LOG4J2.PROPERTIES_filter.write.marker=WRITE
53+
LOG4J2.PROPERTIES_filter.write.onMatch=NEUTRAL
54+
LOG4J2.PROPERTIES_filter.write.onMismatch=NEUTRAL
55+
LOG4J2.PROPERTIES_appenders=console, rolling
56+
LOG4J2.PROPERTIES_appender.console.type=Console
57+
LOG4J2.PROPERTIES_appender.console.name=STDOUT
58+
LOG4J2.PROPERTIES_appender.console.layout.type=PatternLayout
59+
LOG4J2.PROPERTIES_appender.console.layout.pattern=%d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n
60+
LOG4J2.PROPERTIES_appender.rolling.type=RollingFile
61+
LOG4J2.PROPERTIES_appender.rolling.name=RollingFile
62+
LOG4J2.PROPERTIES_appender.rolling.fileName=${sys:hadoop.log.dir}/om-audit-${hostName}.log
63+
LOG4J2.PROPERTIES_appender.rolling.filePattern=${sys:hadoop.log.dir}/om-audit-${hostName}-%d{yyyy-MM-dd-HH-mm-ss}-%i.log.gz
64+
LOG4J2.PROPERTIES_appender.rolling.layout.type=PatternLayout
65+
LOG4J2.PROPERTIES_appender.rolling.layout.pattern=%d{DEFAULT} | %-5level | %c{1} | %msg | %throwable{3} %n
66+
LOG4J2.PROPERTIES_appender.rolling.policies.type=Policies
67+
LOG4J2.PROPERTIES_appender.rolling.policies.time.type=TimeBasedTriggeringPolicy
68+
LOG4J2.PROPERTIES_appender.rolling.policies.time.interval=86400
69+
LOG4J2.PROPERTIES_appender.rolling.policies.size.type=SizeBasedTriggeringPolicy
70+
LOG4J2.PROPERTIES_appender.rolling.policies.size.size=64MB
71+
LOG4J2.PROPERTIES_loggers=audit
72+
LOG4J2.PROPERTIES_logger.audit.type=AsyncLogger
73+
LOG4J2.PROPERTIES_logger.audit.name=OMAudit
74+
LOG4J2.PROPERTIES_logger.audit.level=INFO
75+
LOG4J2.PROPERTIES_logger.audit.appenderRefs=rolling
76+
LOG4J2.PROPERTIES_logger.audit.appenderRef.file.ref=RollingFile
77+
LOG4J2.PROPERTIES_rootLogger.level=INFO
78+
LOG4J2.PROPERTIES_rootLogger.appenderRefs=stdout
79+
LOG4J2.PROPERTIES_rootLogger.appenderRef.stdout.ref=STDOUT

‎hadoop-ozone/dist/src/main/compose/ozones3-haproxy/haproxy-conf/haproxy.cfg‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
# Simple configuration for an HTTP proxy listening on port 9878 on all
2+
# interfaces and forwarding requests to a multiple multiple S3 servers in round
3+
# robin fashion.
4+
global
5+
daemon
6+
maxconn 256
7+
8+
defaults
9+
mode http
10+
timeout connect 5000ms
11+
timeout client 50000ms
12+
timeout server 50000ms
13+
14+
frontend http-in
15+
bind *:9878
16+
default_backend servers
17+
18+
backend servers
19+
balance roundrobin
20+
server server1 s3g1:9878 maxconn 32
21+
server server2 s3g2:9878 maxconn 32
22+
server server3 s3g3:9878 maxconn 32

‎hadoop-ozone/dist/src/main/compose/ozones3-haproxy/test.sh‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
#!/usr/bin/env bash
2+
# Licensed to the Apache Software Foundation (ASF) under one
3+
# or more contributor license agreements. See the NOTICE file
4+
# distributed with this work for additional information
5+
# regarding copyright ownership. The ASF licenses this file
6+
# to you under the Apache License, Version 2.0 (the
7+
# "License"); you may not use this file except in compliance
8+
# with the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing, software
13+
# distributed under the License is distributed on an "AS IS" BASIS,
14+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
# See the License for the specific language governing permissions and
16+
# limitations under the License.
17+
18+
COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
19+
export COMPOSE_DIR
20+
21+
# shellcheck source=/dev/null
22+
source "$COMPOSE_DIR/../testlib.sh"
23+
24+
start_docker_env
25+
26+
execute_robot_test scm basic/basic.robot
27+
28+
stop_docker_env
29+
30+
generate_report

0 commit comments

Comments
 (0)