Skip to content

Commit 85aa877

Browse files
Daniel-009497Daniel-009497
committed
MAPREDUCE-7429 Application log link is not accessible via TimelineServer WebUI in IPV6 scenario
1 parent ca3526d commit 85aa877

File tree

4 files changed

+125
-0
lines changed

4 files changed

+125
-0
lines changed

‎hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@
4646

4747
import javax.net.SocketFactory;
4848

49+
import org.apache.commons.lang3.StringUtils;
4950
import org.apache.hadoop.security.AccessControlException;
5051
import org.apache.hadoop.thirdparty.com.google.common.cache.Cache;
5152
import org.apache.hadoop.thirdparty.com.google.common.cache.CacheBuilder;
@@ -229,6 +230,23 @@ public static InetSocketAddress createSocketAddr(String target,
229230
}
230231
target = target.trim();
231232
boolean hasScheme = target.contains("://");
233+
if (NetUtils.isIPV6Address(target)) {
234+
// if scheme exists in the target, for example:
235+
// https://ffff:ffff:ffff:ffff::1:8088 will be formed like
236+
// https://[ffff:ffff:ffff:ffff::1]:8088
237+
if (hasScheme) {
238+
int i = target.lastIndexOf("/");
239+
String scheme = target.substring(0, i + 1);
240+
String ipAddrWithPort = target.substring(i + 1);
241+
target = scheme + normalizeV6Address(ipAddrWithPort);
242+
} else {
243+
// if scheme does not exists in the target
244+
// for example : ffff:ffff:ffff:ffff::1:8088 will be formed like
245+
// [ffff:ffff:ffff:ffff::1]:8088
246+
target = normalizeV6Address(target);
247+
}
248+
}
249+
232250
URI uri = createURI(target, hasScheme, helpText, useCacheIfPresent);
233251

234252
String host = uri.getHost();
@@ -247,6 +265,28 @@ public static InetSocketAddress createSocketAddr(String target,
247265
return createSocketAddrForHost(host, port);
248266
}
249267

268+
public static String normalizeV6Address(final String target) {
269+
String normalizedAddr = target;
270+
if (!target.startsWith("[")) {
271+
if (target.contains("%")) {
272+
int i = target.lastIndexOf('%');
273+
String port = target.trim().substring(target.lastIndexOf(":") + 1);
274+
String addr = target.trim().substring(0, i);
275+
normalizedAddr = "[" + addr + "]" + ":" + port;
276+
} else {
277+
int i = target.lastIndexOf(':');
278+
String port = target.substring(target.lastIndexOf(":") + 1);
279+
String addr = target.substring(0, i);
280+
normalizedAddr = "[" + addr + "]" + ":" + port;
281+
}
282+
}
283+
return normalizedAddr;
284+
}
285+
286+
public static boolean isIPV6Address(String addr) {
287+
return StringUtils.countMatches(addr, ":") > 2;
288+
}
289+
250290
private static final long URI_CACHE_SIZE_DEFAULT = 1000;
251291
private static final long URI_CACHE_EXPIRE_TIME_DEFAULT = 12;
252292
private static final Cache<String, URI> URI_CACHE = CacheBuilder.newBuilder()

‎hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetUtils.java‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -798,4 +798,74 @@ private <T> void assertBetterArrayEquals(T[] expect, T[]got) {
798798
String gotStr = StringUtils.join(got, ", ");
799799
assertEquals(expectStr, gotStr);
800800
}
801+
802+
@Test
803+
public void testIPV6Address() {
804+
final String IPV6_SAMPLE_ADDRESS1 = "FFFF:FFFF:FFFF:FFFF::1:12345";
805+
final String IPV6_SAMPLE_ADDRESS2 = "FFFF:FFFF:FFFF::1:FFFF:12345";
806+
final String IPV6_SAMPLE_ADDRESS3 = "FFFF:FFFF::1:FFFF:FFFF:12345";
807+
final String IPV4_SAMPLE_ADDRESS = "192.168.0.1:12345";
808+
assertTrue(NetUtils.isIPV6Address(IPV6_SAMPLE_ADDRESS1));
809+
assertTrue(NetUtils.isIPV6Address(IPV6_SAMPLE_ADDRESS2));
810+
assertTrue(NetUtils.isIPV6Address(IPV6_SAMPLE_ADDRESS3));
811+
assertFalse(NetUtils.isIPV6Address(IPV4_SAMPLE_ADDRESS));
812+
}
813+
814+
@Test
815+
public void testNormalizeIPV6Address() {
816+
final String IPV6_SAMPLE_ADDRESS1 = "FFFF:FFFF:FFFF:FFFF::1";
817+
final String IPV6_SAMPLE_ADDRESS2 = "FFFF:FFFF:FFFF::1:FFFF";
818+
final String IPV6_SAMPLE_ADDRESS3 = "FFFF:FFFF::1:FFFF:FFFF";
819+
final String PORT_SUFFIX = ":12345";
820+
String IPV6_SAMPLE_ADDRESS1_WITH_PORT = IPV6_SAMPLE_ADDRESS1 + PORT_SUFFIX;
821+
String IPV6_SAMPLE_ADDRESS2_WITH_PORT = IPV6_SAMPLE_ADDRESS2 + PORT_SUFFIX;
822+
String IPV6_SAMPLE_ADDRESS3_WITH_PORT = IPV6_SAMPLE_ADDRESS3 + PORT_SUFFIX;
823+
assertEquals("[" + IPV6_SAMPLE_ADDRESS1 + "]" + PORT_SUFFIX , NetUtils.normalizeV6Address(IPV6_SAMPLE_ADDRESS1_WITH_PORT));
824+
assertEquals("[" + IPV6_SAMPLE_ADDRESS2 + "]" + PORT_SUFFIX , NetUtils.normalizeV6Address(IPV6_SAMPLE_ADDRESS2_WITH_PORT));
825+
assertEquals("[" + IPV6_SAMPLE_ADDRESS3 + "]" + PORT_SUFFIX , NetUtils.normalizeV6Address(IPV6_SAMPLE_ADDRESS3_WITH_PORT));
826+
}
827+
828+
@Test
829+
public void testCreateSocketAddressWithIPV6() throws Throwable {
830+
final String IPV6_SAMPLE_ADDRESS1 = "FFFF:FFFF:FFFF:FFFF::1";
831+
final String IPV6_SAMPLE_ADDRESS2 = "FFFF:FFFF:FFFF::1:FFFF";
832+
final String IPV6_SAMPLE_ADDRESS3 = "FFFF:FFFF::1:FFFF:FFFF";
833+
final String IPV4_SAMPLE_ADDRESS = "192.168.0.1";
834+
final String PORT_SUFFIX = ":12345";
835+
final int DEFAULT_PORT = 10000;
836+
837+
String IPV6_SAMPLE_ADDRESS1_WITH_PORT = IPV6_SAMPLE_ADDRESS1 + PORT_SUFFIX;
838+
String IPV6_SAMPLE_ADDRESS2_WITH_PORT = IPV6_SAMPLE_ADDRESS2 + PORT_SUFFIX;
839+
String IPV6_SAMPLE_ADDRESS3_WITH_PORT = IPV6_SAMPLE_ADDRESS3 + PORT_SUFFIX;
840+
String IPV4_SAMPLE_ADDRESS_WITH_PORT = IPV4_SAMPLE_ADDRESS + PORT_SUFFIX;
841+
842+
InetSocketAddress addr1 = NetUtils.createSocketAddr(IPV6_SAMPLE_ADDRESS1_WITH_PORT, DEFAULT_PORT, "myconfig1");
843+
InetSocketAddress addr2 = NetUtils.createSocketAddr(IPV6_SAMPLE_ADDRESS2_WITH_PORT, DEFAULT_PORT, "myconfig2");
844+
InetSocketAddress addr3 = NetUtils.createSocketAddr(IPV6_SAMPLE_ADDRESS3_WITH_PORT, DEFAULT_PORT, "myconfig3");
845+
InetSocketAddress addr4 = NetUtils.createSocketAddr(IPV4_SAMPLE_ADDRESS_WITH_PORT, DEFAULT_PORT, "myconfig4");
846+
assertEquals("[" + IPV6_SAMPLE_ADDRESS1 + "]", addr1.getHostName());
847+
assertEquals("[" + IPV6_SAMPLE_ADDRESS2 + "]", addr2.getHostName());
848+
assertEquals("[" + IPV6_SAMPLE_ADDRESS3 + "]", addr3.getHostName());
849+
assertEquals("[" + IPV4_SAMPLE_ADDRESS + "]", addr4.getHostName());
850+
assertEquals(12345, addr1.getPort());
851+
assertEquals(12345, addr2.getPort());
852+
assertEquals(12345, addr3.getPort());
853+
assertEquals(12345, addr4.getPort());
854+
855+
final String IPV6_SAMPLE_ADDRESS1_WITHSCOPE = IPV6_SAMPLE_ADDRESS1 + "%2";
856+
final String IPV6_SAMPLE_ADDRESS2_WITHSCOPE = IPV6_SAMPLE_ADDRESS2 + "%2";
857+
final String IPV6_SAMPLE_ADDRESS3_WITHSCOPE = IPV6_SAMPLE_ADDRESS3 + "%2";
858+
IPV6_SAMPLE_ADDRESS1_WITH_PORT = IPV6_SAMPLE_ADDRESS1_WITHSCOPE + PORT_SUFFIX;
859+
IPV6_SAMPLE_ADDRESS2_WITH_PORT = IPV6_SAMPLE_ADDRESS2_WITHSCOPE + PORT_SUFFIX;
860+
IPV6_SAMPLE_ADDRESS3_WITH_PORT = IPV6_SAMPLE_ADDRESS3_WITHSCOPE + PORT_SUFFIX;
861+
addr1 = NetUtils.createSocketAddr(IPV6_SAMPLE_ADDRESS1_WITH_PORT, DEFAULT_PORT, "myconfig1");
862+
addr2 = NetUtils.createSocketAddr(IPV6_SAMPLE_ADDRESS2_WITH_PORT, DEFAULT_PORT, "myconfig2");
863+
addr3 = NetUtils.createSocketAddr(IPV6_SAMPLE_ADDRESS3_WITH_PORT, DEFAULT_PORT, "myconfig3");
864+
assertEquals("[" + IPV6_SAMPLE_ADDRESS1 + "]", addr1.getHostName());
865+
assertEquals("[" + IPV6_SAMPLE_ADDRESS2 + "]", addr2.getHostName());
866+
assertEquals("[" + IPV6_SAMPLE_ADDRESS3 + "]", addr3.getHostName());
867+
assertEquals(12345, addr1.getPort());
868+
assertEquals(12345, addr2.getPort());
869+
assertEquals(12345, addr3.getPort());
870+
}
801871
}

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineConnector.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,8 @@
3535
import javax.net.ssl.HttpsURLConnection;
3636
import javax.net.ssl.SSLSocketFactory;
3737

38+
import org.apache.commons.lang3.StringUtils;
39+
import org.apache.hadoop.net.NetUtils;
3840
import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler;
3941
import org.apache.hadoop.security.authentication.server.PseudoAuthenticationHandler;
4042
import org.slf4j.Logger;
@@ -197,6 +199,12 @@ private static void setTimeouts(URLConnection connection, int socketTimeout) {
197199

198200
public static URI constructResURI(Configuration conf, String address,
199201
String uri) {
202+
if (NetUtils.isIPV6Address(address)) {
203+
// normalize the ip: port with ipv6 scheme
204+
// For an address like ffff:ffff:ffff::0001:8080 will be transformed into
205+
// [ffff:ffff:ffff::0001]:8080
206+
address = NetUtils.normalizeV6Address(address);
207+
}
200208
return URI.create(
201209
JOINER.join(YarnConfiguration.useHttps(conf) ? "https://" : "http://",
202210
address, uri));

‎hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
import java.util.Iterator;
3131
import java.util.List;
3232

33+
import org.apache.commons.lang3.StringUtils;
3334
import org.apache.hadoop.classification.InterfaceAudience.Private;
3435
import org.apache.hadoop.classification.InterfaceStability.Evolving;
3536
import org.apache.hadoop.conf.Configuration;
@@ -428,6 +429,12 @@ public static String getAggregatedLogURL(String serverHttpAddress,
428429
user == null || user.isEmpty()) {
429430
return null;
430431
}
432+
String[] ipAdds = serverHttpAddress.split("//");
433+
if (ipAdds.length >= 2 && NetUtils.isIPV6Address(ipAdds[1])) {
434+
ipAdds[1] = NetUtils.normalizeV6Address(ipAdds[1]);
435+
serverHttpAddress = ipAdds[0] + "//" + ipAdds[1];
436+
}
437+
431438
return PATH_JOINER.join(serverHttpAddress, "applicationhistory", "logs",
432439
allocatedNode, containerId, entity, user);
433440
}

0 commit comments

Comments
 (0)