Skip to content

MAPREDUCE-7429 Application log link is not accessible via TimelineServer WebUI in IPV6 scenario #5236

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: trunk
Choose a base branch
from
+125 −0
Open

MAPREDUCE-7429 Application log link is not accessible via TimelineServer WebUI in IPV6 scenario #5236

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 40 additions & 0 deletions hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@

import javax.net.SocketFactory;

import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.thirdparty.com.google.common.cache.Cache;
import org.apache.hadoop.thirdparty.com.google.common.cache.CacheBuilder;
Expand Down Expand Up @@ -229,6 +230,23 @@ public static InetSocketAddress createSocketAddr(String target,
}
target = target.trim();
boolean hasScheme = target.contains("://");
if (NetUtils.isIPV6Address(target)) {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

NetUtils.isIPV6Address(target) is slow

image

// if scheme exists in the target, for example:
// https://ffff:ffff:ffff:ffff::1:8088 will be formed like
// https://[ffff:ffff:ffff:ffff::1]:8088
if (hasScheme) {
int i = target.lastIndexOf("/");
String scheme = target.substring(0, i + 1);
String ipAddrWithPort = target.substring(i + 1);
target = scheme + normalizeV6Address(ipAddrWithPort);
} else {
// if scheme does not exists in the target
// for example : ffff:ffff:ffff:ffff::1:8088 will be formed like
// [ffff:ffff:ffff:ffff::1]:8088
target = normalizeV6Address(target);
}
}

URI uri = createURI(target, hasScheme, helpText, useCacheIfPresent);

String host = uri.getHost();
Expand All @@ -247,6 +265,28 @@ public static InetSocketAddress createSocketAddr(String target,
return createSocketAddrForHost(host, port);
}

public static String normalizeV6Address(final String target) {
String normalizedAddr = target;
if (!target.startsWith("[")) {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm pretty sure regex would help here.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No?

if (target.contains("%")) {
int i = target.lastIndexOf('%');
String port = target.trim().substring(target.lastIndexOf(":") + 1);
String addr = target.trim().substring(0, i);
normalizedAddr = "[" + addr + "]" + ":" + port;
} else {
int i = target.lastIndexOf(':');
String port = target.substring(target.lastIndexOf(":") + 1);
String addr = target.substring(0, i);
normalizedAddr = "[" + addr + "]" + ":" + port;
}
}
return normalizedAddr;
}

public static boolean isIPV6Address(String addr) {
return StringUtils.countMatches(addr, ":") > 2;
}

private static final long URI_CACHE_SIZE_DEFAULT = 1000;
private static final long URI_CACHE_EXPIRE_TIME_DEFAULT = 12;
private static final Cache<String, URI> URI_CACHE = CacheBuilder.newBuilder()
Expand Down
70 changes: 70 additions & 0 deletions hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -798,4 +798,74 @@ private <T> void assertBetterArrayEquals(T[] expect, T[]got) {
String gotStr = StringUtils.join(got, ", ");
assertEquals(expectStr, gotStr);
}

@Test
public void testIPV6Address() {
final String IPV6_SAMPLE_ADDRESS1 = "FFFF:FFFF:FFFF:FFFF::1:12345";
final String IPV6_SAMPLE_ADDRESS2 = "FFFF:FFFF:FFFF::1:FFFF:12345";
final String IPV6_SAMPLE_ADDRESS3 = "FFFF:FFFF::1:FFFF:FFFF:12345";
final String IPV4_SAMPLE_ADDRESS = "192.168.0.1:12345";
assertTrue(NetUtils.isIPV6Address(IPV6_SAMPLE_ADDRESS1));
assertTrue(NetUtils.isIPV6Address(IPV6_SAMPLE_ADDRESS2));
assertTrue(NetUtils.isIPV6Address(IPV6_SAMPLE_ADDRESS3));
assertFalse(NetUtils.isIPV6Address(IPV4_SAMPLE_ADDRESS));
}

@Test
public void testNormalizeIPV6Address() {
final String IPV6_SAMPLE_ADDRESS1 = "FFFF:FFFF:FFFF:FFFF::1";
final String IPV6_SAMPLE_ADDRESS2 = "FFFF:FFFF:FFFF::1:FFFF";
final String IPV6_SAMPLE_ADDRESS3 = "FFFF:FFFF::1:FFFF:FFFF";
final String PORT_SUFFIX = ":12345";
String IPV6_SAMPLE_ADDRESS1_WITH_PORT = IPV6_SAMPLE_ADDRESS1 + PORT_SUFFIX;
String IPV6_SAMPLE_ADDRESS2_WITH_PORT = IPV6_SAMPLE_ADDRESS2 + PORT_SUFFIX;
String IPV6_SAMPLE_ADDRESS3_WITH_PORT = IPV6_SAMPLE_ADDRESS3 + PORT_SUFFIX;
assertEquals("[" + IPV6_SAMPLE_ADDRESS1 + "]" + PORT_SUFFIX , NetUtils.normalizeV6Address(IPV6_SAMPLE_ADDRESS1_WITH_PORT));
assertEquals("[" + IPV6_SAMPLE_ADDRESS2 + "]" + PORT_SUFFIX , NetUtils.normalizeV6Address(IPV6_SAMPLE_ADDRESS2_WITH_PORT));
assertEquals("[" + IPV6_SAMPLE_ADDRESS3 + "]" + PORT_SUFFIX , NetUtils.normalizeV6Address(IPV6_SAMPLE_ADDRESS3_WITH_PORT));
}

@Test
public void testCreateSocketAddressWithIPV6() throws Throwable {
final String IPV6_SAMPLE_ADDRESS1 = "FFFF:FFFF:FFFF:FFFF::1";
final String IPV6_SAMPLE_ADDRESS2 = "FFFF:FFFF:FFFF::1:FFFF";
final String IPV6_SAMPLE_ADDRESS3 = "FFFF:FFFF::1:FFFF:FFFF";
final String IPV4_SAMPLE_ADDRESS = "192.168.0.1";
final String PORT_SUFFIX = ":12345";
final int DEFAULT_PORT = 10000;

String IPV6_SAMPLE_ADDRESS1_WITH_PORT = IPV6_SAMPLE_ADDRESS1 + PORT_SUFFIX;
String IPV6_SAMPLE_ADDRESS2_WITH_PORT = IPV6_SAMPLE_ADDRESS2 + PORT_SUFFIX;
String IPV6_SAMPLE_ADDRESS3_WITH_PORT = IPV6_SAMPLE_ADDRESS3 + PORT_SUFFIX;
String IPV4_SAMPLE_ADDRESS_WITH_PORT = IPV4_SAMPLE_ADDRESS + PORT_SUFFIX;

InetSocketAddress addr1 = NetUtils.createSocketAddr(IPV6_SAMPLE_ADDRESS1_WITH_PORT, DEFAULT_PORT, "myconfig1");
InetSocketAddress addr2 = NetUtils.createSocketAddr(IPV6_SAMPLE_ADDRESS2_WITH_PORT, DEFAULT_PORT, "myconfig2");
InetSocketAddress addr3 = NetUtils.createSocketAddr(IPV6_SAMPLE_ADDRESS3_WITH_PORT, DEFAULT_PORT, "myconfig3");
InetSocketAddress addr4 = NetUtils.createSocketAddr(IPV4_SAMPLE_ADDRESS_WITH_PORT, DEFAULT_PORT, "myconfig4");
assertEquals("[" + IPV6_SAMPLE_ADDRESS1 + "]", addr1.getHostName());
assertEquals("[" + IPV6_SAMPLE_ADDRESS2 + "]", addr2.getHostName());
assertEquals("[" + IPV6_SAMPLE_ADDRESS3 + "]", addr3.getHostName());
assertEquals("[" + IPV4_SAMPLE_ADDRESS + "]", addr4.getHostName());
assertEquals(12345, addr1.getPort());
assertEquals(12345, addr2.getPort());
assertEquals(12345, addr3.getPort());
assertEquals(12345, addr4.getPort());

final String IPV6_SAMPLE_ADDRESS1_WITHSCOPE = IPV6_SAMPLE_ADDRESS1 + "%2";
final String IPV6_SAMPLE_ADDRESS2_WITHSCOPE = IPV6_SAMPLE_ADDRESS2 + "%2";
final String IPV6_SAMPLE_ADDRESS3_WITHSCOPE = IPV6_SAMPLE_ADDRESS3 + "%2";
IPV6_SAMPLE_ADDRESS1_WITH_PORT = IPV6_SAMPLE_ADDRESS1_WITHSCOPE + PORT_SUFFIX;
IPV6_SAMPLE_ADDRESS2_WITH_PORT = IPV6_SAMPLE_ADDRESS2_WITHSCOPE + PORT_SUFFIX;
IPV6_SAMPLE_ADDRESS3_WITH_PORT = IPV6_SAMPLE_ADDRESS3_WITHSCOPE + PORT_SUFFIX;
addr1 = NetUtils.createSocketAddr(IPV6_SAMPLE_ADDRESS1_WITH_PORT, DEFAULT_PORT, "myconfig1");
addr2 = NetUtils.createSocketAddr(IPV6_SAMPLE_ADDRESS2_WITH_PORT, DEFAULT_PORT, "myconfig2");
addr3 = NetUtils.createSocketAddr(IPV6_SAMPLE_ADDRESS3_WITH_PORT, DEFAULT_PORT, "myconfig3");
assertEquals("[" + IPV6_SAMPLE_ADDRESS1 + "]", addr1.getHostName());
assertEquals("[" + IPV6_SAMPLE_ADDRESS2 + "]", addr2.getHostName());
assertEquals("[" + IPV6_SAMPLE_ADDRESS3 + "]", addr3.getHostName());
assertEquals(12345, addr1.getPort());
assertEquals(12345, addr2.getPort());
assertEquals(12345, addr3.getPort());
}
}
8 changes: 8 additions & 0 deletions ...p-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineConnector.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,8 @@
import javax.net.ssl.HttpsURLConnection;
import javax.net.ssl.SSLSocketFactory;

import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler;
import org.apache.hadoop.security.authentication.server.PseudoAuthenticationHandler;
import org.slf4j.Logger;
Expand Down Expand Up @@ -197,6 +199,12 @@ private static void setTimeouts(URLConnection connection, int socketTimeout) {

public static URI constructResURI(Configuration conf, String address,
String uri) {
if (NetUtils.isIPV6Address(address)) {
// normalize the ip: port with ipv6 scheme
// For an address like ffff:ffff:ffff::0001:8080 will be transformed into
// [ffff:ffff:ffff::0001]:8080
address = NetUtils.normalizeV6Address(address);
}
return URI.create(
JOINER.join(YarnConfiguration.useHttps(conf) ? "https://" : "http://",
address, uri));
Expand Down
7 changes: 7 additions & 0 deletions ...yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
import java.util.Iterator;
import java.util.List;

import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Evolving;
import org.apache.hadoop.conf.Configuration;
Expand Down Expand Up @@ -428,6 +429,12 @@ public static String getAggregatedLogURL(String serverHttpAddress,
user == null || user.isEmpty()) {
return null;
}
String[] ipAdds = serverHttpAddress.split("//");
if (ipAdds.length >= 2 && NetUtils.isIPV6Address(ipAdds[1])) {
ipAdds[1] = NetUtils.normalizeV6Address(ipAdds[1]);
serverHttpAddress = ipAdds[0] + "//" + ipAdds[1];
}

return PATH_JOINER.join(serverHttpAddress, "applicationhistory", "logs",
allocatedNode, containerId, entity, user);
}
Expand Down