Skip to content

Commit 72d425c

Browse files
author
Sahil Takiar
committed
CDH-78803: HDFS-14304: High lock contention on hdfsHashMutex in libhdfs
This closes apache#595 Signed-off-by: Todd Lipcon <[email protected]> (cherry picked from commit 18c57cf) Conflicts: hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/native_mini_dfs.c hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/exception.c hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/jni_helper.h Change-Id: I9d6fb410cae3311e384df2008a3f2a6645355574
1 parent fc2e5e3 commit 72d425c

File tree

16 files changed

+657
-966
lines changed

16 files changed

+657
-966
lines changed

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/CMakeLists.txt

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,8 @@ include_directories(
2929

3030
add_library(native_mini_dfs
3131
native_mini_dfs.c
32-
../libhdfs/common/htable.c
3332
../libhdfs/exception.c
33+
../libhdfs/jclasses.c
3434
../libhdfs/jni_helper.c
3535
${OS_DIR}/mutexes.c
3636
${OS_DIR}/thread_local_storage.c
@@ -39,6 +39,3 @@ add_library(native_mini_dfs
3939
add_executable(test_native_mini_dfs test_native_mini_dfs.c)
4040
target_link_libraries(test_native_mini_dfs native_mini_dfs ${JAVA_JVM_LIBRARY})
4141
add_test(test_test_native_mini_dfs test_native_mini_dfs)
42-
43-
add_executable(test_htable ../libhdfs/common/htable.c test_htable.c)
44-
target_link_libraries(test_htable ${OS_LINK_LIBRARIES})

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/native_mini_dfs.c

Lines changed: 19 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
*/
1818

1919
#include "exception.h"
20+
#include "jclasses.h"
2021
#include "jni_helper.h"
2122
#include "native_mini_dfs.h"
2223
#include "platform.h"
@@ -36,9 +37,7 @@
3637

3738
#define MINIDFS_CLUSTER_BUILDER "org/apache/hadoop/hdfs/MiniDFSCluster$Builder"
3839
#define MINIDFS_CLUSTER "org/apache/hadoop/hdfs/MiniDFSCluster"
39-
#define HADOOP_CONF "org/apache/hadoop/conf/Configuration"
4040
#define HADOOP_NAMENODE "org/apache/hadoop/hdfs/server/namenode/NameNode"
41-
#define JAVA_INETSOCKETADDRESS "java/net/InetSocketAddress"
4241

4342
struct NativeMiniDfsCluster {
4443
/**
@@ -60,8 +59,7 @@ static int hdfsDisableDomainSocketSecurity(void)
6059
errno = EINTERNAL;
6160
return -1;
6261
}
63-
jthr = invokeMethod(env, NULL, STATIC, NULL,
64-
"org/apache/hadoop/net/unix/DomainSocket",
62+
jthr = invokeMethod(env, NULL, STATIC, NULL, JC_DOMAIN_SOCKET,
6563
"disableBindPathValidation", "()V");
6664
if (jthr) {
6765
errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
@@ -126,11 +124,6 @@ struct NativeMiniDfsCluster* nmdCreate(struct NativeMiniDfsConf *conf)
126124
"nmdCreate: new Configuration");
127125
goto error;
128126
}
129-
if (jthr) {
130-
printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
131-
"nmdCreate: Configuration::setBoolean");
132-
goto error;
133-
}
134127
// Disable 'minimum block size' -- it's annoying in tests.
135128
(*env)->DeleteLocalRef(env, jconfStr);
136129
jconfStr = NULL;
@@ -140,8 +133,9 @@ struct NativeMiniDfsCluster* nmdCreate(struct NativeMiniDfsConf *conf)
140133
"nmdCreate: new String");
141134
goto error;
142135
}
143-
jthr = invokeMethod(env, NULL, INSTANCE, cobj, HADOOP_CONF,
144-
"setLong", "(Ljava/lang/String;J)V", jconfStr, 0LL);
136+
jthr = invokeMethod(env, NULL, INSTANCE, cobj,
137+
JC_CONFIGURATION, "setLong", "(Ljava/lang/String;J)V", jconfStr,
138+
0LL);
145139
if (jthr) {
146140
printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
147141
"nmdCreate: Configuration::setLong");
@@ -163,7 +157,7 @@ struct NativeMiniDfsCluster* nmdCreate(struct NativeMiniDfsConf *conf)
163157
goto error;
164158
}
165159
}
166-
jthr = invokeMethod(env, &val, INSTANCE, bld, MINIDFS_CLUSTER_BUILDER,
160+
jthr = findClassAndInvokeMethod(env, &val, INSTANCE, bld, MINIDFS_CLUSTER_BUILDER,
167161
"format", "(Z)L" MINIDFS_CLUSTER_BUILDER ";", conf->doFormat);
168162
if (jthr) {
169163
printExceptionAndFree(env, jthr, PRINT_EXC_ALL, "nmdCreate: "
@@ -172,7 +166,7 @@ struct NativeMiniDfsCluster* nmdCreate(struct NativeMiniDfsConf *conf)
172166
}
173167
(*env)->DeleteLocalRef(env, val.l);
174168
if (conf->webhdfsEnabled) {
175-
jthr = invokeMethod(env, &val, INSTANCE, bld, MINIDFS_CLUSTER_BUILDER,
169+
jthr = findClassAndInvokeMethod(env, &val, INSTANCE, bld, MINIDFS_CLUSTER_BUILDER,
176170
"nameNodeHttpPort", "(I)L" MINIDFS_CLUSTER_BUILDER ";",
177171
conf->namenodeHttpPort);
178172
if (jthr) {
@@ -182,7 +176,7 @@ struct NativeMiniDfsCluster* nmdCreate(struct NativeMiniDfsConf *conf)
182176
}
183177
(*env)->DeleteLocalRef(env, val.l);
184178
}
185-
jthr = invokeMethod(env, &val, INSTANCE, bld, MINIDFS_CLUSTER_BUILDER,
179+
jthr = findClassAndInvokeMethod(env, &val, INSTANCE, bld, MINIDFS_CLUSTER_BUILDER,
186180
"build", "()L" MINIDFS_CLUSTER ";");
187181
if (jthr) {
188182
printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
@@ -232,7 +226,7 @@ int nmdShutdown(struct NativeMiniDfsCluster* cl)
232226
fprintf(stderr, "nmdShutdown: getJNIEnv failed\n");
233227
return -EIO;
234228
}
235-
jthr = invokeMethod(env, NULL, INSTANCE, cl->obj,
229+
jthr = findClassAndInvokeMethod(env, NULL, INSTANCE, cl->obj,
236230
MINIDFS_CLUSTER, "shutdown", "()V");
237231
if (jthr) {
238232
printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
@@ -250,7 +244,7 @@ int nmdWaitClusterUp(struct NativeMiniDfsCluster *cl)
250244
fprintf(stderr, "nmdWaitClusterUp: getJNIEnv failed\n");
251245
return -EIO;
252246
}
253-
jthr = invokeMethod(env, NULL, INSTANCE, cl->obj,
247+
jthr = findClassAndInvokeMethod(env, NULL, INSTANCE, cl->obj,
254248
MINIDFS_CLUSTER, "waitClusterUp", "()V");
255249
if (jthr) {
256250
printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
@@ -272,7 +266,7 @@ int nmdGetNameNodePort(const struct NativeMiniDfsCluster *cl)
272266
}
273267
// Note: this will have to be updated when HA nativeMiniDfs clusters are
274268
// supported
275-
jthr = invokeMethod(env, &jVal, INSTANCE, cl->obj,
269+
jthr = findClassAndInvokeMethod(env, &jVal, INSTANCE, cl->obj,
276270
MINIDFS_CLUSTER, "getNameNodePort", "()I");
277271
if (jthr) {
278272
printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
@@ -297,7 +291,7 @@ int nmdGetNameNodeHttpAddress(const struct NativeMiniDfsCluster *cl,
297291
return -EIO;
298292
}
299293
// First get the (first) NameNode of the cluster
300-
jthr = invokeMethod(env, &jVal, INSTANCE, cl->obj, MINIDFS_CLUSTER,
294+
jthr = findClassAndInvokeMethod(env, &jVal, INSTANCE, cl->obj, MINIDFS_CLUSTER,
301295
"getNameNode", "()L" HADOOP_NAMENODE ";");
302296
if (jthr) {
303297
printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
@@ -308,27 +302,27 @@ int nmdGetNameNodeHttpAddress(const struct NativeMiniDfsCluster *cl,
308302
jNameNode = jVal.l;
309303

310304
// Then get the http address (InetSocketAddress) of the NameNode
311-
jthr = invokeMethod(env, &jVal, INSTANCE, jNameNode, HADOOP_NAMENODE,
312-
"getHttpAddress", "()L" JAVA_INETSOCKETADDRESS ";");
305+
jthr = findClassAndInvokeMethod(env, &jVal, INSTANCE, jNameNode, HADOOP_NAMENODE,
306+
"getHttpAddress", "()L" JAVA_NET_ISA ";");
313307
if (jthr) {
314308
ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
315309
"nmdGetNameNodeHttpAddress: "
316310
"NameNode#getHttpAddress");
317311
goto error_dlr_nn;
318312
}
319313
jAddress = jVal.l;
320-
321-
jthr = invokeMethod(env, &jVal, INSTANCE, jAddress,
322-
JAVA_INETSOCKETADDRESS, "getPort", "()I");
314+
315+
jthr = findClassAndInvokeMethod(env, &jVal, INSTANCE, jAddress,
316+
JAVA_NET_ISA, "getPort", "()I");
323317
if (jthr) {
324318
ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,
325319
"nmdGetNameNodeHttpAddress: "
326320
"InetSocketAddress#getPort");
327321
goto error_dlr_addr;
328322
}
329323
*port = jVal.i;
330-
331-
jthr = invokeMethod(env, &jVal, INSTANCE, jAddress, JAVA_INETSOCKETADDRESS,
324+
325+
jthr = findClassAndInvokeMethod(env, &jVal, INSTANCE, jAddress, JAVA_NET_ISA,
332326
"getHostName", "()Ljava/lang/String;");
333327
if (jthr) {
334328
ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL,

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/test_htable.c

Lines changed: 0 additions & 100 deletions
This file was deleted.

hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/CMakeLists.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ hadoop_add_dual_library(hdfs
3535
exception.c
3636
jni_helper.c
3737
hdfs.c
38-
common/htable.c
38+
jclasses.c
3939
${OS_DIR}/mutexes.c
4040
${OS_DIR}/thread_local_storage.c
4141
)

0 commit comments

Comments
 (0)