From d3011100c1894785854cab77399e5e08a5649171 Mon Sep 17 00:00:00 2001 From: "shwetayakkali@cloudera.com" Date: Fri, 7 Jun 2019 16:12:54 -0700 Subject: [PATCH 1/3] HDDS-1651. Create a http.policy config for Ozone Change-Id: Ia284f685f6d39a512124e6055537615d325ae96b --- .../java/org/apache/hadoop/ozone/OzoneConfigKeys.java | 2 ++ .../common/src/main/resources/ozone-default.xml | 10 ++++++++++ 2 files changed, 12 insertions(+) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java index 1463c43e830f3..cc6231726972a 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java @@ -64,6 +64,8 @@ public final class OzoneConfigKeys { "dfs.container.ratis.ipc"; public static final int DFS_CONTAINER_RATIS_IPC_PORT_DEFAULT = 9858; + public static final String OZONE_HTTP_POLICY = "ozone.http.policy"; + /** * When set to true, allocate a random free port for ozone container, so that * a mini cluster is able to launch multiple containers on a node. diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml index 33f058486c7fe..755ff6025170d 100644 --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml @@ -2487,4 +2487,14 @@ The number of Recon Tasks that are waiting on updates from OM. + + ozone.http.policy + HTTP_ONLY + This configures the HTTP endpoint for Ozone daemons: + The following values are supported: + - HTTP_ONLY : Service is provided only on http + - HTTPS_ONLY : Service is provided only on https + - HTTP_AND_HTTPS : Service is provided both on http and https + + From 42b684f2aef026126f6fac1a58ddbc4d4b602f40 Mon Sep 17 00:00:00 2001 From: "shwetayakkali@cloudera.com" Date: Thu, 13 Jun 2019 22:11:11 -0700 Subject: [PATCH 2/3] HDDS-1651. Create a http.policy config for Ozone Change-Id: I047aafc733c936fb82f926bcde489595cf51d928 --- .../java/org/apache/hadoop/hdds/HddsUtils.java | 18 ++++++++++++++++++ .../hadoop/hdds/server/BaseHttpServer.java | 3 ++- .../ozone/om/TestOzoneManagerHttpServer.java | 6 ++++-- 3 files changed, 24 insertions(+), 3 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java index a284caaf66579..7041e355c81eb 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java @@ -31,6 +31,7 @@ import java.util.Optional; import java.util.TimeZone; +import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; @@ -42,6 +43,8 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.SCMSecurityProtocol; import org.apache.hadoop.hdds.scm.protocolPB.ScmBlockLocationProtocolPB; +import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.http.HttpConfig.Policy; import org.apache.hadoop.ipc.Client; import org.apache.hadoop.ipc.ProtobufRpcEngine; import org.apache.hadoop.ipc.RPC; @@ -56,6 +59,7 @@ import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ENABLED; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ENABLED_DEFAULT; +import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.security.UserGroupInformation; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -426,6 +430,20 @@ public static long getUtcTime() { return Calendar.getInstance(UTC_ZONE).getTimeInMillis(); } + public static Policy getHttpPolicy(Configuration conf) { + String policyStr = conf.get("ozone.http.policy", OzoneConfigKeys.OZONE_HTTP_POLICY); + if(policyStr == null || policyStr.length() == 0) { + policyStr = conf.get("dfs.http.policy", DFSConfigKeys.DFS_HTTP_POLICY_DEFAULT); + } + Policy policy = Policy.fromString(policyStr); + if (policy == null) { + throw new HadoopIllegalArgumentException("Unrecognized value '" + policyStr + "' for " + "dfs.http.policy"); + } else { + conf.set("dfs.http.policy", policy.name()); + return policy; + } + } + /** * Retrieve the socket address that should be used by clients to connect * to the SCM for diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/BaseHttpServer.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/BaseHttpServer.java index 9a1d4b3c77992..5a1c33d3f407f 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/BaseHttpServer.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/BaseHttpServer.java @@ -19,6 +19,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdds.HddsConfigKeys; +import org.apache.hadoop.hdds.HddsUtils; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdds.conf.HddsConfServlet; @@ -65,7 +66,7 @@ public abstract class BaseHttpServer { public BaseHttpServer(Configuration conf, String name) throws IOException { this.name = name; this.conf = conf; - policy = DFSUtil.getHttpPolicy(conf); + policy = HddsUtils.getHttpPolicy(conf); if (isEnabled()) { this.httpAddress = getHttpBindAddress(); this.httpsAddress = getHttpsBindAddress(); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHttpServer.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHttpServer.java index b071e27302a3a..a1aa3724ae0cc 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHttpServer.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHttpServer.java @@ -25,6 +25,7 @@ import org.apache.hadoop.http.HttpConfig; import org.apache.hadoop.http.HttpConfig.Policy; import org.apache.hadoop.net.NetUtils; +import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.security.ssl.KeyStoreTestUtil; import org.apache.hadoop.test.GenericTestUtils; import org.junit.AfterClass; @@ -91,8 +92,9 @@ public TestOzoneManagerHttpServer(Policy policy) { KeyStoreTestUtil.cleanupSSLConfig(keystoresDir, sslConfDir); } - @Test public void testHttpPolicy() throws Exception { - conf.set(DFSConfigKeys.DFS_HTTP_POLICY_KEY, policy.name()); + @Test + public void testHttpPolicy() throws Exception { + conf.set(OzoneConfigKeys.OZONE_HTTP_POLICY, policy.name()); conf.set(OMConfigKeys.OZONE_OM_HTTP_ADDRESS_KEY, "localhost:0"); conf.set(OMConfigKeys.OZONE_OM_HTTPS_ADDRESS_KEY, "localhost:0"); From 788b79d2f5502f8a102aec66fb29af68e9f75c7f Mon Sep 17 00:00:00 2001 From: "shwetayakkali@cloudera.com" Date: Tue, 18 Jun 2019 15:34:33 -0700 Subject: [PATCH 3/3] HDDS-1651. Create a http.policy config for Ozone Change-Id: I23b1ddc619488bb290be9136a18ec5e2aab7dd77 --- .../org/apache/hadoop/hdds/HddsUtils.java | 14 +++------ .../ozone/om/TestOzoneManagerHttpServer.java | 30 ++++++++++++++++++- 2 files changed, 33 insertions(+), 11 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java index 7041e355c81eb..fab622039a66e 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java @@ -431,17 +431,11 @@ public static long getUtcTime() { } public static Policy getHttpPolicy(Configuration conf) { - String policyStr = conf.get("ozone.http.policy", OzoneConfigKeys.OZONE_HTTP_POLICY); - if(policyStr == null || policyStr.length() == 0) { - policyStr = conf.get("dfs.http.policy", DFSConfigKeys.DFS_HTTP_POLICY_DEFAULT); - } + String policyStr = conf.get(OzoneConfigKeys.OZONE_HTTP_POLICY, + DFSConfigKeys.DFS_HTTP_POLICY_DEFAULT); Policy policy = Policy.fromString(policyStr); - if (policy == null) { - throw new HadoopIllegalArgumentException("Unrecognized value '" + policyStr + "' for " + "dfs.http.policy"); - } else { - conf.set("dfs.http.policy", policy.name()); - return policy; - } + conf.set("dfs.http.policy", policy.name()); + return policy; } /** diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHttpServer.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHttpServer.java index a1aa3724ae0cc..816f505fde58d 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHttpServer.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHttpServer.java @@ -94,7 +94,7 @@ public TestOzoneManagerHttpServer(Policy policy) { @Test public void testHttpPolicy() throws Exception { - conf.set(OzoneConfigKeys.OZONE_HTTP_POLICY, policy.name()); + conf.set(DFSConfigKeys.DFS_HTTP_POLICY_KEY, policy.name()); conf.set(OMConfigKeys.OZONE_OM_HTTP_ADDRESS_KEY, "localhost:0"); conf.set(OMConfigKeys.OZONE_OM_HTTPS_ADDRESS_KEY, "localhost:0"); @@ -121,6 +121,34 @@ public void testHttpPolicy() throws Exception { } } + @Test + public void tesOzonetHttpPolicy() throws Exception { + conf.set(OzoneConfigKeys.OZONE_HTTP_POLICY, policy.name()); + conf.set(OMConfigKeys.OZONE_OM_HTTP_ADDRESS_KEY, "localhost:0"); + conf.set(OMConfigKeys.OZONE_OM_HTTPS_ADDRESS_KEY, "localhost:0"); + + OzoneManagerHttpServer server = null; + try { + server = new OzoneManagerHttpServer(conf, null); + server.start(); + + Assert.assertTrue(implies(policy.isHttpEnabled(), + canAccess("http", server.getHttpAddress()))); + Assert.assertTrue(implies(policy.isHttpEnabled() && + !policy.isHttpsEnabled(), + !canAccess("https", server.getHttpsAddress()))); + + Assert.assertTrue(implies(policy.isHttpsEnabled(), + canAccess("https", server.getHttpsAddress()))); + Assert.assertTrue(implies(policy.isHttpsEnabled(), + !canAccess("http", server.getHttpsAddress()))); + + } finally { + if (server != null) { + server.stop(); + } + } + } private static boolean canAccess(String scheme, InetSocketAddress addr) { if (addr == null) { return false;