From 091b41795af985cb0526b48c9cf23e2123552ff8 Mon Sep 17 00:00:00 2001 From: Bharat Viswanadham Date: Fri, 20 Sep 2019 14:32:54 -0700 Subject: [PATCH 1/6] HDDS-2019. Handle Set DtService of token in S3Gateway for OM HA. --- .../hadoop/ozone/s3/OzoneClientProducer.java | 12 ++++- .../hadoop/ozone/s3/OzoneServiceProvider.java | 48 ++++++++++++++++++- .../hadoop/ozone/s3/util/OzoneS3Util.java | 27 +++++++++++ 3 files changed, 84 insertions(+), 3 deletions(-) diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneClientProducer.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneClientProducer.java index d9afaf162a457..75edf1da2ea6c 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneClientProducer.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneClientProducer.java @@ -20,6 +20,7 @@ import com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.io.Text; +import org.apache.hadoop.ozone.OmUtils; import org.apache.hadoop.ozone.OzoneSecurityUtil; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.client.OzoneClientFactory; @@ -62,6 +63,9 @@ public class OzoneClientProducer { @Inject private Text omService; + @Inject + private String omServiceID; + @Produces public OzoneClient createClient() throws IOException { @@ -105,7 +109,13 @@ private OzoneClient getClient(OzoneConfiguration config) throws IOException { } catch (Exception e) { LOG.error("Error: ", e); } - return OzoneClientFactory.getClient(ozoneConfiguration); + + if (omServiceID == null) { + return OzoneClientFactory.getClient(ozoneConfiguration); + } else { + // As in HA case, we need to pass om service ID. + return OzoneClientFactory.getRpcClient(omServiceID, ozoneConfiguration); + } } @VisibleForTesting diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneServiceProvider.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneServiceProvider.java index f4342f62f4820..f73130a626685 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneServiceProvider.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneServiceProvider.java @@ -20,12 +20,19 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.io.Text; import org.apache.hadoop.ozone.OmUtils; +import org.apache.hadoop.ozone.s3.util.OzoneS3Util; import org.apache.hadoop.security.SecurityUtil; import javax.annotation.PostConstruct; import javax.enterprise.context.ApplicationScoped; import javax.enterprise.inject.Produces; import javax.inject.Inject; + +import java.util.Collection; + +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_NODES_KEY; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SERVICE_IDS_KEY; + /** * This class creates the OM service . */ @@ -34,13 +41,45 @@ public class OzoneServiceProvider { private Text omServiceAdd; + private String omserviceID; + @Inject private OzoneConfiguration conf; @PostConstruct public void init() { - omServiceAdd = SecurityUtil.buildTokenService(OmUtils. - getOmAddressForClients(conf)); + Collection serviceIdList = + conf.getTrimmedStringCollection(OZONE_OM_SERVICE_IDS_KEY); + if (serviceIdList.size() == 0) { + // Non-HA cluster + omServiceAdd = SecurityUtil.buildTokenService(OmUtils. + getOmAddressForClients(conf)); + } else { + // HA cluster + Collection serviceIds = + conf.getTrimmedStringCollection(OZONE_OM_SERVICE_IDS_KEY); + + //For now if multiple service id's are configured we throw exception. + // As if multiple service id's are configured, S3Gateway will not be + // knowing which one to talk to. In future, if OM federation is supported + // we can resolve this by having another property like + // ozone.om.internal.service.id. + // TODO: Revisit this later. + if (serviceIds.size() > 1) { + throw new IllegalArgumentException("Multiple serviceIds are " + + "configured. " + serviceIds.toArray().toString()); + } else { + String serviceId = serviceIds.iterator().next(); + Collection omNodeIds = OmUtils.getOMNodeIds(conf, serviceId); + if (omNodeIds.size() == 0) { + throw new IllegalArgumentException(OZONE_OM_NODES_KEY + + "." + serviceId + " is not defined"); + } + omServiceAdd = new Text(OzoneS3Util.buildServiceNameForToken(conf, + serviceId, omNodeIds)); + omserviceID = serviceId; + } + } } @@ -49,4 +88,9 @@ public Text getService() { return omServiceAdd; } + @Produces + public String getOmServiceID() { + return omserviceID; + } + } diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/OzoneS3Util.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/OzoneS3Util.java index 129ea2d293d5e..7fa2543551fbd 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/OzoneS3Util.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/OzoneS3Util.java @@ -19,8 +19,17 @@ package org.apache.hadoop.ozone.s3.util; import org.apache.commons.codec.digest.DigestUtils; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.net.NetUtils; +import org.apache.hadoop.ozone.OmUtils; +import org.apache.hadoop.security.SecurityUtil; + +import javax.annotation.Nonnull; +import java.util.Collection; import java.util.Objects; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY; + /** * Ozone util for S3 related operations. */ @@ -33,4 +42,22 @@ public static String getVolumeName(String userName) { Objects.requireNonNull(userName); return DigestUtils.md5Hex(userName); } + + public static String buildServiceNameForToken( + @Nonnull OzoneConfiguration configuration, @Nonnull String serviceId, + @Nonnull Collection omNodeIds) { + StringBuilder rpcAddress = new StringBuilder(); + for (String nodeId : omNodeIds) { + String rpcAddrKey = OmUtils.addKeySuffixes(OZONE_OM_ADDRESS_KEY, + serviceId, nodeId); + String rpcAddrStr = OmUtils.getOmRpcAddress(configuration, rpcAddrKey); + if (rpcAddrStr == null) { + throw new IllegalArgumentException("Could not find rpcAddress for " + + OZONE_OM_ADDRESS_KEY + "." + serviceId + "." + nodeId); + } + rpcAddress.append(SecurityUtil.buildTokenService( + NetUtils.createSocketAddr(rpcAddrStr))); + } + return rpcAddress.toString(); + } } From 921d5211a73184c8f461c97a23c687c5972fe8e7 Mon Sep 17 00:00:00 2001 From: Bharat Viswanadham Date: Fri, 20 Sep 2019 14:39:58 -0700 Subject: [PATCH 2/6] few minor changes.: --- .../java/org/apache/hadoop/ozone/s3/util/OzoneS3Util.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/OzoneS3Util.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/OzoneS3Util.java index 7fa2543551fbd..627b32338c6d0 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/OzoneS3Util.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/OzoneS3Util.java @@ -43,6 +43,13 @@ public static String getVolumeName(String userName) { return DigestUtils.md5Hex(userName); } + /** + * Generate service Name for token. + * @param configuration + * @param serviceId - ozone manager service ID + * @param omNodeIds - list of node ids for the given OM service. + * @return service Name. + */ public static String buildServiceNameForToken( @Nonnull OzoneConfiguration configuration, @Nonnull String serviceId, @Nonnull Collection omNodeIds) { From 0f88603d2b0b0173dd094e45e3650990171bebd8 Mon Sep 17 00:00:00 2001 From: Bharat Viswanadham Date: Tue, 24 Sep 2019 16:13:14 -0700 Subject: [PATCH 3/6] fix review comments. --- .../hadoop/ozone/s3/OzoneServiceProvider.java | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneServiceProvider.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneServiceProvider.java index f73130a626685..032e1ee7ffa52 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneServiceProvider.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneServiceProvider.java @@ -39,7 +39,7 @@ @ApplicationScoped public class OzoneServiceProvider { - private Text omServiceAdd; + private Text omServiceAddr; private String omserviceID; @@ -52,30 +52,27 @@ public void init() { conf.getTrimmedStringCollection(OZONE_OM_SERVICE_IDS_KEY); if (serviceIdList.size() == 0) { // Non-HA cluster - omServiceAdd = SecurityUtil.buildTokenService(OmUtils. + omServiceAddr = SecurityUtil.buildTokenService(OmUtils. getOmAddressForClients(conf)); } else { - // HA cluster - Collection serviceIds = - conf.getTrimmedStringCollection(OZONE_OM_SERVICE_IDS_KEY); - + // HA cluster. //For now if multiple service id's are configured we throw exception. // As if multiple service id's are configured, S3Gateway will not be // knowing which one to talk to. In future, if OM federation is supported // we can resolve this by having another property like // ozone.om.internal.service.id. // TODO: Revisit this later. - if (serviceIds.size() > 1) { + if (serviceIdList.size() > 1) { throw new IllegalArgumentException("Multiple serviceIds are " + - "configured. " + serviceIds.toArray().toString()); + "configured. " + serviceIdList.toArray().toString()); } else { - String serviceId = serviceIds.iterator().next(); + String serviceId = serviceIdList.iterator().next(); Collection omNodeIds = OmUtils.getOMNodeIds(conf, serviceId); if (omNodeIds.size() == 0) { throw new IllegalArgumentException(OZONE_OM_NODES_KEY + "." + serviceId + " is not defined"); } - omServiceAdd = new Text(OzoneS3Util.buildServiceNameForToken(conf, + omServiceAddr = new Text(OzoneS3Util.buildServiceNameForToken(conf, serviceId, omNodeIds)); omserviceID = serviceId; } @@ -85,7 +82,7 @@ public void init() { @Produces public Text getService() { - return omServiceAdd; + return omServiceAddr; } @Produces From 150720943d7c0a10e941aa0e4e94b171312412f2 Mon Sep 17 00:00:00 2001 From: Bharat Viswanadham Date: Tue, 24 Sep 2019 16:47:41 -0700 Subject: [PATCH 4/6] add test --- .../hadoop/ozone/s3/util/OzoneS3Util.java | 14 +- .../hadoop/ozone/s3/util/TestOzoneS3Util.java | 129 ++++++++++++++++++ 2 files changed, 141 insertions(+), 2 deletions(-) create mode 100644 hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/util/TestOzoneS3Util.java diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/OzoneS3Util.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/OzoneS3Util.java index 627b32338c6d0..6504c967ceb4a 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/OzoneS3Util.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/OzoneS3Util.java @@ -54,7 +54,11 @@ public static String buildServiceNameForToken( @Nonnull OzoneConfiguration configuration, @Nonnull String serviceId, @Nonnull Collection omNodeIds) { StringBuilder rpcAddress = new StringBuilder(); + + int nodesLength = omNodeIds.size(); + int counter = 0; for (String nodeId : omNodeIds) { + counter++; String rpcAddrKey = OmUtils.addKeySuffixes(OZONE_OM_ADDRESS_KEY, serviceId, nodeId); String rpcAddrStr = OmUtils.getOmRpcAddress(configuration, rpcAddrKey); @@ -62,8 +66,14 @@ public static String buildServiceNameForToken( throw new IllegalArgumentException("Could not find rpcAddress for " + OZONE_OM_ADDRESS_KEY + "." + serviceId + "." + nodeId); } - rpcAddress.append(SecurityUtil.buildTokenService( - NetUtils.createSocketAddr(rpcAddrStr))); + + if (counter != nodesLength) { + rpcAddress.append(SecurityUtil.buildTokenService( + NetUtils.createSocketAddr(rpcAddrStr)) + ","); + } else { + rpcAddress.append(SecurityUtil.buildTokenService( + NetUtils.createSocketAddr(rpcAddrStr))); + } } return rpcAddress.toString(); } diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/util/TestOzoneS3Util.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/util/TestOzoneS3Util.java new file mode 100644 index 0000000000000..b2ff39e697364 --- /dev/null +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/util/TestOzoneS3Util.java @@ -0,0 +1,129 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.s3.util; + +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.ozone.OmUtils; +import org.apache.hadoop.ozone.om.OMConfigKeys; +import org.apache.hadoop.security.SecurityUtil; +import org.apache.hadoop.test.GenericTestUtils; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.util.Collection; + +import static org.apache.hadoop.fs.CommonConfigurationKeys.HADOOP_SECURITY_TOKEN_SERVICE_USE_IP; +import static org.junit.Assert.fail; + +/** + * Class used to test OzoneS3Util. + */ +public class TestOzoneS3Util { + + + private OzoneConfiguration configuration; + + @Before + public void setConf() { + configuration = new OzoneConfiguration(); + String serviceID = "omService"; + String nodeIDs = "om1,om2,om3"; + configuration.set(OMConfigKeys.OZONE_OM_SERVICE_IDS_KEY, serviceID); + configuration.set(OMConfigKeys.OZONE_OM_NODE_ID_KEY, nodeIDs); + configuration.setBoolean(HADOOP_SECURITY_TOKEN_SERVICE_USE_IP, false); + } + + @Test + public void testBuildServiceNameForToken() { + String serviceID = "omService"; + String nodeIDs = "om1,om2,om3"; + configuration.set(OMConfigKeys.OZONE_OM_SERVICE_IDS_KEY, serviceID); + configuration.set(OMConfigKeys.OZONE_OM_NODE_ID_KEY, nodeIDs); + configuration.setBoolean(HADOOP_SECURITY_TOKEN_SERVICE_USE_IP, false); + + Collection nodeIDList = configuration.getStringCollection( + OMConfigKeys.OZONE_OM_NODE_ID_KEY); + + String expectedOmServiceAddress = buildOMNodeAddresses(nodeIDList, + serviceID); + + SecurityUtil.setConfiguration(configuration); + String omserviceAddr = OzoneS3Util.buildServiceNameForToken(configuration, + serviceID, nodeIDList); + + Assert.assertEquals(expectedOmServiceAddress, omserviceAddr); + } + + @Test + public void testBuildServiceNameForTokenIncorrectConfig() { + String serviceID = "omService"; + String nodeIDs = "om1,om2,om3"; + configuration.set(OMConfigKeys.OZONE_OM_SERVICE_IDS_KEY, serviceID); + configuration.set(OMConfigKeys.OZONE_OM_NODE_ID_KEY, nodeIDs); + configuration.setBoolean(HADOOP_SECURITY_TOKEN_SERVICE_USE_IP, false); + + Collection nodeIDList = configuration.getStringCollection( + OMConfigKeys.OZONE_OM_NODE_ID_KEY); + + // Don't set om3 node rpc address. + configuration.set(OmUtils.addKeySuffixes(OMConfigKeys.OZONE_OM_ADDRESS_KEY, + serviceID, "om1"), "om1:9862"); + configuration.set(OmUtils.addKeySuffixes(OMConfigKeys.OZONE_OM_ADDRESS_KEY, + serviceID, "om2"), "om2:9862"); + + + SecurityUtil.setConfiguration(configuration); + + try { + OzoneS3Util.buildServiceNameForToken(configuration, + serviceID, nodeIDList); + fail("testBuildServiceNameForTokenIncorrectConfig failed"); + } catch (IllegalArgumentException ex) { + GenericTestUtils.assertExceptionContains("Could not find rpcAddress " + + "for", ex); + } + + + } + + + private String buildOMNodeAddresses(Collection nodeIDList, + String serviceID) { + StringBuilder omServiceAddrBuilder = new StringBuilder(); + int port = 9862; + int nodesLength = nodeIDList.size(); + int counter = 0; + for (String nodeID : nodeIDList) { + counter++; + String addr = nodeID + ":" + port++; + configuration.set(OmUtils.addKeySuffixes(OMConfigKeys.OZONE_OM_ADDRESS_KEY, + serviceID, nodeID), addr); + + if (counter != nodesLength) { + omServiceAddrBuilder.append(addr + ","); + } else { + omServiceAddrBuilder.append(addr); + } + } + + return omServiceAddrBuilder.toString(); + } + +} From 5965226f0f8739f2bcc4e6e24ee0adbda7ecc8f1 Mon Sep 17 00:00:00 2001 From: Bharat Viswanadham Date: Fri, 27 Sep 2019 14:28:16 -0700 Subject: [PATCH 5/6] fix jenkins issues. --- .../java/org/apache/hadoop/ozone/s3/OzoneClientProducer.java | 1 - .../java/org/apache/hadoop/ozone/s3/OzoneServiceProvider.java | 3 ++- .../java/org/apache/hadoop/ozone/s3/util/TestOzoneS3Util.java | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneClientProducer.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneClientProducer.java index 75edf1da2ea6c..abaca03908240 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneClientProducer.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneClientProducer.java @@ -20,7 +20,6 @@ import com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.io.Text; -import org.apache.hadoop.ozone.OmUtils; import org.apache.hadoop.ozone.OzoneSecurityUtil; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.client.OzoneClientFactory; diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneServiceProvider.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneServiceProvider.java index 032e1ee7ffa52..b98426c20149e 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneServiceProvider.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneServiceProvider.java @@ -28,6 +28,7 @@ import javax.enterprise.inject.Produces; import javax.inject.Inject; +import java.util.Arrays; import java.util.Collection; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_NODES_KEY; @@ -64,7 +65,7 @@ public void init() { // TODO: Revisit this later. if (serviceIdList.size() > 1) { throw new IllegalArgumentException("Multiple serviceIds are " + - "configured. " + serviceIdList.toArray().toString()); + "configured. " + Arrays.toString(serviceIdList.toArray())); } else { String serviceId = serviceIdList.iterator().next(); Collection omNodeIds = OmUtils.getOMNodeIds(conf, serviceId); diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/util/TestOzoneS3Util.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/util/TestOzoneS3Util.java index b2ff39e697364..78ee200a1122d 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/util/TestOzoneS3Util.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/util/TestOzoneS3Util.java @@ -113,8 +113,8 @@ private String buildOMNodeAddresses(Collection nodeIDList, for (String nodeID : nodeIDList) { counter++; String addr = nodeID + ":" + port++; - configuration.set(OmUtils.addKeySuffixes(OMConfigKeys.OZONE_OM_ADDRESS_KEY, - serviceID, nodeID), addr); + configuration.set(OmUtils.addKeySuffixes( + OMConfigKeys.OZONE_OM_ADDRESS_KEY, serviceID, nodeID), addr); if (counter != nodesLength) { omServiceAddrBuilder.append(addr + ","); From 9aebf0c8765f7edc85d08aae3d40597bf1b97f93 Mon Sep 17 00:00:00 2001 From: Bharat Viswanadham Date: Tue, 1 Oct 2019 12:50:25 -0700 Subject: [PATCH 6/6] address review comments. --- .../hadoop/ozone/s3/util/OzoneS3Util.java | 2 +- .../hadoop/ozone/s3/util/TestOzoneS3Util.java | 53 ++++++++++--------- 2 files changed, 28 insertions(+), 27 deletions(-) diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/OzoneS3Util.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/OzoneS3Util.java index 6504c967ceb4a..ce7d4f2876ab8 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/OzoneS3Util.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/OzoneS3Util.java @@ -62,7 +62,7 @@ public static String buildServiceNameForToken( String rpcAddrKey = OmUtils.addKeySuffixes(OZONE_OM_ADDRESS_KEY, serviceId, nodeId); String rpcAddrStr = OmUtils.getOmRpcAddress(configuration, rpcAddrKey); - if (rpcAddrStr == null) { + if (rpcAddrStr == null || rpcAddrStr.isEmpty()) { throw new IllegalArgumentException("Could not find rpcAddress for " + OZONE_OM_ADDRESS_KEY + "." + serviceId + "." + nodeId); } diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/util/TestOzoneS3Util.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/util/TestOzoneS3Util.java index 78ee200a1122d..8892a9784dbf6 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/util/TestOzoneS3Util.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/util/TestOzoneS3Util.java @@ -39,31 +39,34 @@ public class TestOzoneS3Util { private OzoneConfiguration configuration; + private String serviceID = "omService"; @Before public void setConf() { configuration = new OzoneConfiguration(); - String serviceID = "omService"; + String nodeIDs = "om1,om2,om3"; configuration.set(OMConfigKeys.OZONE_OM_SERVICE_IDS_KEY, serviceID); - configuration.set(OMConfigKeys.OZONE_OM_NODE_ID_KEY, nodeIDs); + configuration.set(OMConfigKeys.OZONE_OM_NODES_KEY + "." + serviceID, + nodeIDs); configuration.setBoolean(HADOOP_SECURITY_TOKEN_SERVICE_USE_IP, false); } @Test public void testBuildServiceNameForToken() { - String serviceID = "omService"; - String nodeIDs = "om1,om2,om3"; - configuration.set(OMConfigKeys.OZONE_OM_SERVICE_IDS_KEY, serviceID); - configuration.set(OMConfigKeys.OZONE_OM_NODE_ID_KEY, nodeIDs); - configuration.setBoolean(HADOOP_SECURITY_TOKEN_SERVICE_USE_IP, false); - - Collection nodeIDList = configuration.getStringCollection( - OMConfigKeys.OZONE_OM_NODE_ID_KEY); - String expectedOmServiceAddress = buildOMNodeAddresses(nodeIDList, + Collection nodeIDList = OmUtils.getOMNodeIds(configuration, serviceID); + configuration.set(OmUtils.addKeySuffixes(OMConfigKeys.OZONE_OM_ADDRESS_KEY, + serviceID, "om1"), "om1:9862"); + configuration.set(OmUtils.addKeySuffixes(OMConfigKeys.OZONE_OM_ADDRESS_KEY, + serviceID, "om2"), "om2:9862"); + configuration.set(OmUtils.addKeySuffixes(OMConfigKeys.OZONE_OM_ADDRESS_KEY, + serviceID, "om3"), "om3:9862"); + + String expectedOmServiceAddress = buildServiceAddress(nodeIDList); + SecurityUtil.setConfiguration(configuration); String omserviceAddr = OzoneS3Util.buildServiceNameForToken(configuration, serviceID, nodeIDList); @@ -71,18 +74,15 @@ public void testBuildServiceNameForToken() { Assert.assertEquals(expectedOmServiceAddress, omserviceAddr); } + @Test public void testBuildServiceNameForTokenIncorrectConfig() { - String serviceID = "omService"; - String nodeIDs = "om1,om2,om3"; - configuration.set(OMConfigKeys.OZONE_OM_SERVICE_IDS_KEY, serviceID); - configuration.set(OMConfigKeys.OZONE_OM_NODE_ID_KEY, nodeIDs); - configuration.setBoolean(HADOOP_SECURITY_TOKEN_SERVICE_USE_IP, false); - Collection nodeIDList = configuration.getStringCollection( - OMConfigKeys.OZONE_OM_NODE_ID_KEY); + Collection nodeIDList = OmUtils.getOMNodeIds(configuration, + serviceID); - // Don't set om3 node rpc address. + // Don't set om3 node rpc address. Here we are skipping setting of one of + // the OM address. So buildServiceNameForToken will fail. configuration.set(OmUtils.addKeySuffixes(OMConfigKeys.OZONE_OM_ADDRESS_KEY, serviceID, "om1"), "om1:9862"); configuration.set(OmUtils.addKeySuffixes(OMConfigKeys.OZONE_OM_ADDRESS_KEY, @@ -103,18 +103,19 @@ public void testBuildServiceNameForTokenIncorrectConfig() { } - - private String buildOMNodeAddresses(Collection nodeIDList, - String serviceID) { + /** + * Build serviceName from list of node ids. + * @param nodeIDList + * @return service name for token. + */ + private String buildServiceAddress(Collection nodeIDList) { StringBuilder omServiceAddrBuilder = new StringBuilder(); - int port = 9862; int nodesLength = nodeIDList.size(); int counter = 0; for (String nodeID : nodeIDList) { counter++; - String addr = nodeID + ":" + port++; - configuration.set(OmUtils.addKeySuffixes( - OMConfigKeys.OZONE_OM_ADDRESS_KEY, serviceID, nodeID), addr); + String addr = configuration.get(OmUtils.addKeySuffixes( + OMConfigKeys.OZONE_OM_ADDRESS_KEY, serviceID, nodeID)); if (counter != nodesLength) { omServiceAddrBuilder.append(addr + ",");