From 328577e9a8dc476b2723e07cc6560e4270ae00c6 Mon Sep 17 00:00:00 2001 From: prasad-acit Date: Mon, 2 Aug 2021 19:22:04 +0530 Subject: [PATCH 1/2] HADOOP-17542. IPV6 support in Netutils#createSocketAddress --- .../apache/hadoop/fs/ChecksumFileSystem.java | 7 ++++ .../main/java/org/apache/hadoop/fs/Path.java | 11 ++++-- .../java/org/apache/hadoop/net/NetUtils.java | 35 ++++++++++++++++++ .../org/apache/hadoop/net/TestNetUtils.java | 36 +++++++++++++++++++ 4 files changed, 87 insertions(+), 2 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java index 0256a58f46368..e06fce2fd8f02 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java @@ -30,6 +30,7 @@ import java.util.concurrent.CompletableFuture; import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; +import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; @@ -104,12 +105,18 @@ public FileSystem getRawFileSystem() { /** Return the name of the checksum file associated with a file.*/ public Path getChecksumFile(Path file) { + if (StringUtils.countMatches(file.toString(), ":") > 2) { + return new Path(file.getParent(), "./" + file.getName() + ".crc"); + } return new Path(file.getParent(), "." + file.getName() + ".crc"); } /** Return true iff file is a checksum file name.*/ public static boolean isChecksumFile(Path file) { String name = file.getName(); + if (StringUtils.countMatches(file.toString(), ":") > 2) { + return name.startsWith("./") && name.endsWith(".crc"); + } return name.startsWith(".") && name.endsWith(".crc"); } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java index f70ff01e4a0da..8ef7311c8bb9d 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java @@ -199,8 +199,15 @@ public Path(String pathString) throws IllegalArgumentException { int start = 0; // parse uri scheme, if any - int colon = pathString.indexOf(':'); - int slash = pathString.indexOf('/'); + int colon = -1; + int slash = -1; + if (StringUtils.countMatches(pathString, ":") > 2) { + colon = pathString.indexOf(":/"); + slash = pathString.indexOf('/'); + } else { + colon = pathString.indexOf(':'); + slash = pathString.indexOf('/'); + } if ((colon != -1) && ((slash == -1) || (colon < slash))) { // has a scheme scheme = pathString.substring(0, colon); diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java index 6d58ee42e0c24..19cdbc073672c 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java @@ -45,6 +45,7 @@ import javax.net.SocketFactory; +import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.thirdparty.com.google.common.cache.Cache; import org.apache.hadoop.thirdparty.com.google.common.cache.CacheBuilder; @@ -223,6 +224,22 @@ public static InetSocketAddress createSocketAddr(String target, } target = target.trim(); boolean hasScheme = target.contains("://"); + if (StringUtils.countMatches(target, ":") > 2) { + // if scheme exists in the target + // for example : https://ffff:ffff:ffff:ffff::1:XXXXX + // we have to form https://[ffff:ffff:ffff:ffff::1]:XXXXX + if (hasScheme) { + int i = target.lastIndexOf("/"); + String scheme = target.substring(0, i + 1); + String ipAddrWithPort = target.substring(i + 1); + target = scheme + normalizeV6Address(ipAddrWithPort); + } else { + // if scheme does not exists in the target + // for example : ffff:ffff:ffff:ffff::1:XXXXX + // we have to form [ffff:ffff:ffff:ffff::1]:XXXXX + target = normalizeV6Address(target); + } + } URI uri = createURI(target, hasScheme, helpText, useCacheIfPresent); String host = uri.getHost(); @@ -275,6 +292,24 @@ private static URI createURI(String target, return uri; } + public static String normalizeV6Address(String target) { + if (!target.startsWith("[")) { + if (target.contains("%")) { + int i = target.lastIndexOf('%'); + target = target.trim(); + String port = target.substring(target.lastIndexOf(":") + 1); + String addr = target.substring(0, i); + target = "[" + addr + "]" + ":" + port; + } else { + int i = target.lastIndexOf(':'); + String port = target.substring(target.lastIndexOf(":") + 1); + String addr = target.substring(0, i); + target = "[" + addr + "]" + ":" + port; + } + } + return target; + } + /** * Create a socket address with the given host and port. The hostname * might be replaced with another host that was set via diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetUtils.java index f8599b2fa8ede..a3479044121be 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetUtils.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetUtils.java @@ -817,4 +817,40 @@ private void assertBetterArrayEquals(T[] expect, T[]got) { String gotStr = StringUtils.join(got, ", "); assertEquals(expectStr, gotStr); } + + @Test + public void testCreateSocketAddressWithIPV6() throws Throwable { + String IPV6_SAMPLE_ADDRESS = "2a03:2880:2130:cf05:face:b00c:0:1"; + String IPV6_SAMPLE_WITH_PORT = IPV6_SAMPLE_ADDRESS + ":12345"; + + InetSocketAddress addr = NetUtils.createSocketAddr(IPV6_SAMPLE_WITH_PORT, + 1000, "myconfig"); + assertEquals("[" + IPV6_SAMPLE_ADDRESS + "]", addr.getHostName()); + assertEquals(12345, addr.getPort()); + + String IPV6_SAMPLE_ADDRESS_WITHSCOPE = IPV6_SAMPLE_ADDRESS + "%2"; + IPV6_SAMPLE_WITH_PORT = IPV6_SAMPLE_ADDRESS_WITHSCOPE + ":12345"; + addr = NetUtils.createSocketAddr(IPV6_SAMPLE_WITH_PORT, 1000, "myconfig"); + assertEquals("[" + IPV6_SAMPLE_ADDRESS + "]", addr.getHostName()); + assertEquals(12345, addr.getPort()); + + IPV6_SAMPLE_ADDRESS = "[2a03:2880:2130:cf05:face:b00c:0:1]"; + IPV6_SAMPLE_WITH_PORT = IPV6_SAMPLE_ADDRESS + ":12345"; + + addr = NetUtils.createSocketAddr(IPV6_SAMPLE_WITH_PORT, 1000, "myconfig"); + assertEquals(IPV6_SAMPLE_ADDRESS, addr.getHostName()); + assertEquals(12345, addr.getPort()); + + String IPV6_ADDRESS_WITH_SCHEME = "https://2a03:2880:2130:cf05:face:b00c:0:1:12345"; + addr = NetUtils.createSocketAddr(IPV6_ADDRESS_WITH_SCHEME, 1000, + "myconfig"); + assertEquals(IPV6_SAMPLE_ADDRESS, addr.getHostName()); + assertEquals(12345, addr.getPort()); + + IPV6_ADDRESS_WITH_SCHEME = "https://[2a03:2880:2130:cf05:face:b00c:0:1]:12345"; + addr = NetUtils.createSocketAddr(IPV6_ADDRESS_WITH_SCHEME, 1000, + "myconfig"); + assertEquals(IPV6_SAMPLE_ADDRESS, addr.getHostName()); + assertEquals(12345, addr.getPort()); + } } From bf0581faaa20df2b552df585ed1715bcf8811c7b Mon Sep 17 00:00:00 2001 From: prasad-acit Date: Tue, 3 Aug 2021 12:59:12 +0530 Subject: [PATCH 2/2] HADOOP-17542. IPV6 support in Netutils#createSocketAddress, review comments --- .../apache/hadoop/fs/ChecksumFileSystem.java | 7 ---- .../main/java/org/apache/hadoop/fs/Path.java | 7 ++-- .../org/apache/hadoop/net/TestNetUtils.java | 37 ++++++++++--------- 3 files changed, 23 insertions(+), 28 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java index e06fce2fd8f02..0256a58f46368 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java @@ -30,7 +30,6 @@ import java.util.concurrent.CompletableFuture; import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions; -import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; @@ -105,18 +104,12 @@ public FileSystem getRawFileSystem() { /** Return the name of the checksum file associated with a file.*/ public Path getChecksumFile(Path file) { - if (StringUtils.countMatches(file.toString(), ":") > 2) { - return new Path(file.getParent(), "./" + file.getName() + ".crc"); - } return new Path(file.getParent(), "." + file.getName() + ".crc"); } /** Return true iff file is a checksum file name.*/ public static boolean isChecksumFile(Path file) { String name = file.getName(); - if (StringUtils.countMatches(file.toString(), ":") > 2) { - return name.startsWith("./") && name.endsWith(".crc"); - } return name.startsWith(".") && name.endsWith(".crc"); } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java index 8ef7311c8bb9d..853b7d971fc9f 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Path.java @@ -200,13 +200,14 @@ public Path(String pathString) throws IllegalArgumentException { // parse uri scheme, if any int colon = -1; - int slash = -1; + int slash = pathString.indexOf('/'); if (StringUtils.countMatches(pathString, ":") > 2) { + //In case of IPv6 address, we should be able to parse the scheme + // correctly (This will ensure to parse path with & without scheme + // correctly in IPv6). colon = pathString.indexOf(":/"); - slash = pathString.indexOf('/'); } else { colon = pathString.indexOf(':'); - slash = pathString.indexOf('/'); } if ((colon != -1) && ((slash == -1) || (colon < slash))) { // has a scheme diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetUtils.java index a3479044121be..e602e66a771c9 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetUtils.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetUtils.java @@ -820,37 +820,38 @@ private void assertBetterArrayEquals(T[] expect, T[]got) { @Test public void testCreateSocketAddressWithIPV6() throws Throwable { - String IPV6_SAMPLE_ADDRESS = "2a03:2880:2130:cf05:face:b00c:0:1"; - String IPV6_SAMPLE_WITH_PORT = IPV6_SAMPLE_ADDRESS + ":12345"; + String ipv6Address = "2a03:2880:2130:cf05:face:b00c:0:1"; + String ipv6WithPort = ipv6Address + ":12345"; - InetSocketAddress addr = NetUtils.createSocketAddr(IPV6_SAMPLE_WITH_PORT, + InetSocketAddress addr = NetUtils.createSocketAddr(ipv6WithPort, 1000, "myconfig"); - assertEquals("[" + IPV6_SAMPLE_ADDRESS + "]", addr.getHostName()); + assertEquals("[" + ipv6Address + "]", addr.getHostName()); assertEquals(12345, addr.getPort()); - String IPV6_SAMPLE_ADDRESS_WITHSCOPE = IPV6_SAMPLE_ADDRESS + "%2"; - IPV6_SAMPLE_WITH_PORT = IPV6_SAMPLE_ADDRESS_WITHSCOPE + ":12345"; - addr = NetUtils.createSocketAddr(IPV6_SAMPLE_WITH_PORT, 1000, "myconfig"); - assertEquals("[" + IPV6_SAMPLE_ADDRESS + "]", addr.getHostName()); + String ipv6SampleAddressWithScope = ipv6Address + "%2"; + ipv6WithPort = ipv6SampleAddressWithScope + ":12345"; + addr = NetUtils.createSocketAddr(ipv6WithPort, 1000, "myconfig"); + assertEquals("[" + ipv6Address + "]", addr.getHostName()); assertEquals(12345, addr.getPort()); - IPV6_SAMPLE_ADDRESS = "[2a03:2880:2130:cf05:face:b00c:0:1]"; - IPV6_SAMPLE_WITH_PORT = IPV6_SAMPLE_ADDRESS + ":12345"; + ipv6Address = "[2a03:2880:2130:cf05:face:b00c:0:1]"; + ipv6WithPort = ipv6Address + ":12345"; - addr = NetUtils.createSocketAddr(IPV6_SAMPLE_WITH_PORT, 1000, "myconfig"); - assertEquals(IPV6_SAMPLE_ADDRESS, addr.getHostName()); + addr = NetUtils.createSocketAddr(ipv6WithPort, 1000, "myconfig"); + assertEquals(ipv6Address, addr.getHostName()); assertEquals(12345, addr.getPort()); - String IPV6_ADDRESS_WITH_SCHEME = "https://2a03:2880:2130:cf05:face:b00c:0:1:12345"; - addr = NetUtils.createSocketAddr(IPV6_ADDRESS_WITH_SCHEME, 1000, + String ipv6AddressWithScheme = + "https://2a03:2880:2130:cf05:face:b00c:0:1:12345"; + addr = NetUtils.createSocketAddr(ipv6AddressWithScheme, 1000, "myconfig"); - assertEquals(IPV6_SAMPLE_ADDRESS, addr.getHostName()); + assertEquals(ipv6Address, addr.getHostName()); assertEquals(12345, addr.getPort()); - IPV6_ADDRESS_WITH_SCHEME = "https://[2a03:2880:2130:cf05:face:b00c:0:1]:12345"; - addr = NetUtils.createSocketAddr(IPV6_ADDRESS_WITH_SCHEME, 1000, + ipv6AddressWithScheme = "https://[2a03:2880:2130:cf05:face:b00c:0:1]:12345"; + addr = NetUtils.createSocketAddr(ipv6AddressWithScheme, 1000, "myconfig"); - assertEquals(IPV6_SAMPLE_ADDRESS, addr.getHostName()); + assertEquals(ipv6Address, addr.getHostName()); assertEquals(12345, addr.getPort()); } }