net/
${shaded.dependency.prefix}.net.
diff --git a/hadoop-client-modules/hadoop-client/pom.xml b/hadoop-client-modules/hadoop-client/pom.xml
index 9216a2e54a397..dced359b286d9 100644
--- a/hadoop-client-modules/hadoop-client/pom.xml
+++ b/hadoop-client-modules/hadoop-client/pom.xml
@@ -18,11 +18,11 @@
org.apache.hadoop
hadoop-project-dist
- 3.3.0-SNAPSHOT
+ 3.4.0-SNAPSHOT
../../hadoop-project-dist
hadoop-client
- 3.3.0-SNAPSHOT
+ 3.4.0-SNAPSHOT
Apache Hadoop Client aggregation pom with dependencies exposed
Apache Hadoop Client Aggregator
diff --git a/hadoop-client-modules/pom.xml b/hadoop-client-modules/pom.xml
index 0895e31ca307f..fb4aedb0aeb43 100644
--- a/hadoop-client-modules/pom.xml
+++ b/hadoop-client-modules/pom.xml
@@ -18,7 +18,7 @@
org.apache.hadoop
hadoop-project
- 3.3.0-SNAPSHOT
+ 3.4.0-SNAPSHOT
../hadoop-project
hadoop-client-modules
diff --git a/hadoop-cloud-storage-project/hadoop-cloud-storage/pom.xml b/hadoop-cloud-storage-project/hadoop-cloud-storage/pom.xml
index b5e35b079f9fd..11b092674cf4f 100644
--- a/hadoop-cloud-storage-project/hadoop-cloud-storage/pom.xml
+++ b/hadoop-cloud-storage-project/hadoop-cloud-storage/pom.xml
@@ -18,11 +18,11 @@
org.apache.hadoop
hadoop-project
- 3.3.0-SNAPSHOT
+ 3.4.0-SNAPSHOT
../../hadoop-project
hadoop-cloud-storage
- 3.3.0-SNAPSHOT
+ 3.4.0-SNAPSHOT
jar
Apache Hadoop Cloud Storage
@@ -128,5 +128,10 @@
hadoop-openstack
compile
+
+ org.apache.hadoop
+ hadoop-cos
+ compile
+
diff --git a/hadoop-cloud-storage-project/hadoop-cos/dev-support/findbugs-exclude.xml b/hadoop-cloud-storage-project/hadoop-cos/dev-support/findbugs-exclude.xml
index 40d78d0cd6cec..e647e678a07a6 100644
--- a/hadoop-cloud-storage-project/hadoop-cos/dev-support/findbugs-exclude.xml
+++ b/hadoop-cloud-storage-project/hadoop-cos/dev-support/findbugs-exclude.xml
@@ -15,4 +15,9 @@
limitations under the License.
-->
+
+
+
+ h_LIB
+
diff --git a/hadoop-cloud-storage-project/hadoop-cos/pom.xml b/hadoop-cloud-storage-project/hadoop-cos/pom.xml
index 839bd04c9b643..d18b09f450408 100644
--- a/hadoop-cloud-storage-project/hadoop-cos/pom.xml
+++ b/hadoop-cloud-storage-project/hadoop-cos/pom.xml
@@ -20,7 +20,7 @@
org.apache.hadoop
hadoop-project
- 3.3.0-SNAPSHOT
+ 3.4.0-SNAPSHOT
../../hadoop-project
hadoop-cos
@@ -81,6 +81,22 @@
3600
+
+ org.apache.maven.plugins
+ maven-dependency-plugin
+
+
+ deplist
+ compile
+
+ list
+
+
+ ${project.basedir}/target/hadoop-cloud-storage-deps/${project.artifactId}.cloud-storage-optional.txt
+
+
+
+
@@ -93,8 +109,8 @@
com.qcloud
- cos_api
- 5.4.9
+ cos_api-bundle
+ 5.6.19
compile
diff --git a/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/BufferPool.java b/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/BufferPool.java
index a4ee4d5be9ac8..409c9cb42f966 100644
--- a/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/BufferPool.java
+++ b/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/BufferPool.java
@@ -63,32 +63,27 @@ private BufferPool() {
private File createDir(String dirPath) throws IOException {
File dir = new File(dirPath);
- if (null != dir) {
- if (!dir.exists()) {
- LOG.debug("Buffer dir: [{}] does not exists. create it first.",
- dirPath);
- if (dir.mkdirs()) {
- if (!dir.setWritable(true) || !dir.setReadable(true)
- || !dir.setExecutable(true)) {
- LOG.warn("Set the buffer dir: [{}]'s permission [writable,"
- + "readable, executable] failed.", dir.getAbsolutePath());
- }
- LOG.debug("Buffer dir: [{}] is created successfully.",
- dir.getAbsolutePath());
- } else {
- // Once again, check if it has been created successfully.
- // Prevent problems created by multiple processes at the same time.
- if (!dir.exists()) {
- throw new IOException("buffer dir:" + dir.getAbsolutePath()
- + " is created unsuccessfully");
- }
+ if (!dir.exists()) {
+ LOG.debug("Buffer dir: [{}] does not exists. create it first.",
+ dirPath);
+ if (dir.mkdirs()) {
+ if (!dir.setWritable(true) || !dir.setReadable(true)
+ || !dir.setExecutable(true)) {
+ LOG.warn("Set the buffer dir: [{}]'s permission [writable,"
+ + "readable, executable] failed.", dir.getAbsolutePath());
}
+ LOG.debug("Buffer dir: [{}] is created successfully.",
+ dir.getAbsolutePath());
} else {
- LOG.debug("buffer dir: {} already exists.", dirPath);
+ // Once again, check if it has been created successfully.
+ // Prevent problems created by multiple processes at the same time.
+ if (!dir.exists()) {
+ throw new IOException("buffer dir:" + dir.getAbsolutePath()
+ + " is created unsuccessfully");
+ }
}
} else {
- throw new IOException("creating buffer dir: " + dir.getAbsolutePath()
- + "unsuccessfully.");
+ LOG.debug("buffer dir: {} already exists.", dirPath);
}
return dir;
diff --git a/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/CosNFileReadTask.java b/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/CosNFileReadTask.java
index a5dcdda07120b..249e9e1ade82a 100644
--- a/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/CosNFileReadTask.java
+++ b/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/CosNFileReadTask.java
@@ -80,7 +80,6 @@ public CosNFileReadTask(
public void run() {
int retries = 0;
RetryPolicy.RetryAction retryAction;
- LOG.info(Thread.currentThread().getName() + "read ...");
try {
this.readBuffer.lock();
do {
diff --git a/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/CosNUtils.java b/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/CosNUtils.java
index 39981caba24bb..cdac15ffc619e 100644
--- a/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/CosNUtils.java
+++ b/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/CosNUtils.java
@@ -22,15 +22,16 @@
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.lang.reflect.Modifier;
+import java.net.URI;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.qcloud.cos.auth.COSCredentialsProvider;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.cosn.auth.COSCredentialProviderList;
-import org.apache.hadoop.fs.cosn.auth.EnvironmentVariableCredentialProvider;
-import org.apache.hadoop.fs.cosn.auth.SimpleCredentialProvider;
+import org.apache.hadoop.fs.cosn.auth.COSCredentialsProviderList;
+import org.apache.hadoop.fs.cosn.auth.EnvironmentVariableCredentialsProvider;
+import org.apache.hadoop.fs.cosn.auth.SimpleCredentialsProvider;
/**
* Utility methods for CosN code.
@@ -48,21 +49,23 @@ public final class CosNUtils {
private CosNUtils() {
}
- public static COSCredentialProviderList createCosCredentialsProviderSet(
+ public static COSCredentialsProviderList createCosCredentialsProviderSet(
+ URI uri,
Configuration conf) throws IOException {
- COSCredentialProviderList credentialProviderList =
- new COSCredentialProviderList();
+ COSCredentialsProviderList credentialProviderList =
+ new COSCredentialsProviderList();
Class[] cosClasses = CosNUtils.loadCosProviderClasses(
conf,
CosNConfigKeys.COSN_CREDENTIALS_PROVIDER);
if (0 == cosClasses.length) {
- credentialProviderList.add(new SimpleCredentialProvider(conf));
- credentialProviderList.add(new EnvironmentVariableCredentialProvider());
+ credentialProviderList.add(
+ new SimpleCredentialsProvider(uri, conf));
+ credentialProviderList.add(
+ new EnvironmentVariableCredentialsProvider(uri, conf));
} else {
for (Class credClass : cosClasses) {
- credentialProviderList.add(createCOSCredentialProvider(
- conf,
+ credentialProviderList.add(createCOSCredentialProvider(uri, conf,
credClass));
}
}
@@ -83,16 +86,17 @@ public static Class[] loadCosProviderClasses(
}
public static COSCredentialsProvider createCOSCredentialProvider(
+ URI uri,
Configuration conf,
Class credClass) throws IOException {
COSCredentialsProvider credentialsProvider;
if (!COSCredentialsProvider.class.isAssignableFrom(credClass)) {
- throw new IllegalArgumentException(
- "class " + credClass + " " + NOT_COS_CREDENTIAL_PROVIDER);
+ throw new IllegalArgumentException("class " + credClass + " " +
+ NOT_COS_CREDENTIAL_PROVIDER);
}
if (Modifier.isAbstract(credClass.getModifiers())) {
- throw new IllegalArgumentException(
- "class " + credClass + " " + ABSTRACT_CREDENTIAL_PROVIDER);
+ throw new IllegalArgumentException("class " + credClass + " " +
+ ABSTRACT_CREDENTIAL_PROVIDER);
}
LOG.debug("Credential Provider class: " + credClass.getName());
@@ -112,8 +116,18 @@ public static COSCredentialsProvider createCOSCredentialProvider(
return credentialsProvider;
}
- Method factory = getFactoryMethod(
- credClass, COSCredentialsProvider.class, "getInstance");
+ // new credClass(uri, conf)
+ constructor = getConstructor(credClass, URI.class,
+ Configuration.class);
+ if (null != constructor) {
+ credentialsProvider =
+ (COSCredentialsProvider) constructor.newInstance(uri,
+ conf);
+ return credentialsProvider;
+ }
+
+ Method factory = getFactoryMethod(credClass,
+ COSCredentialsProvider.class, "getInstance");
if (null != factory) {
credentialsProvider = (COSCredentialsProvider) factory.invoke(null);
return credentialsProvider;
diff --git a/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/CosNativeFileSystemStore.java b/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/CosNativeFileSystemStore.java
index 833f42d7be6e7..d2484c0e47b3c 100644
--- a/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/CosNativeFileSystemStore.java
+++ b/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/CosNativeFileSystemStore.java
@@ -34,6 +34,7 @@
import com.qcloud.cos.ClientConfig;
import com.qcloud.cos.auth.BasicCOSCredentials;
import com.qcloud.cos.auth.COSCredentials;
+import com.qcloud.cos.endpoint.SuffixEndpointBuilder;
import com.qcloud.cos.exception.CosClientException;
import com.qcloud.cos.exception.CosServiceException;
import com.qcloud.cos.http.HttpProtocol;
@@ -64,7 +65,7 @@
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.cosn.auth.COSCredentialProviderList;
+import org.apache.hadoop.fs.cosn.auth.COSCredentialsProviderList;
import org.apache.hadoop.util.VersionInfo;
import org.apache.http.HttpStatus;
@@ -89,9 +90,9 @@ class CosNativeFileSystemStore implements NativeFileSystemStore {
* @throws IOException Initialize the COS client failed,
* caused by incorrect options.
*/
- private void initCOSClient(Configuration conf) throws IOException {
- COSCredentialProviderList credentialProviderList =
- CosNUtils.createCosCredentialsProviderSet(conf);
+ private void initCOSClient(URI uri, Configuration conf) throws IOException {
+ COSCredentialsProviderList credentialProviderList =
+ CosNUtils.createCosCredentialsProviderSet(uri, conf);
String region = conf.get(CosNConfigKeys.COSN_REGION_KEY);
String endpointSuffix = conf.get(
CosNConfigKeys.COSN_ENDPOINT_SUFFIX_KEY);
@@ -113,7 +114,7 @@ private void initCOSClient(Configuration conf) throws IOException {
ClientConfig config;
if (null == region) {
config = new ClientConfig(new Region(""));
- config.setEndPointSuffix(endpointSuffix);
+ config.setEndpointBuilder(new SuffixEndpointBuilder(endpointSuffix));
} else {
config = new ClientConfig(new Region(region));
}
@@ -146,7 +147,7 @@ private void initCOSClient(Configuration conf) throws IOException {
@Override
public void initialize(URI uri, Configuration conf) throws IOException {
try {
- initCOSClient(conf);
+ initCOSClient(uri, conf);
this.bucketName = uri.getHost();
} catch (Exception e) {
handleException(e, "");
@@ -174,8 +175,8 @@ private void storeFileWithRetry(String key, InputStream inputStream,
PutObjectResult putObjectResult =
(PutObjectResult) callCOSClientWithRetry(putObjectRequest);
- LOG.debug("Store file successfully. COS key: [{}], ETag: [{}], "
- + "MD5: [{}].", key, putObjectResult.getETag(), new String(md5Hash));
+ LOG.debug("Store file successfully. COS key: [{}], ETag: [{}].",
+ key, putObjectResult.getETag());
} catch (Exception e) {
String errMsg = String.format("Store file failed. COS key: [%s], "
+ "exception: [%s]", key, e.toString());
@@ -196,8 +197,7 @@ private void storeFileWithRetry(String key, InputStream inputStream,
public void storeFile(String key, File file, byte[] md5Hash)
throws IOException {
LOG.info("Store file from local path: [{}]. file length: [{}] COS key: " +
- "[{}] MD5: [{}].", file.getCanonicalPath(), file.length(), key,
- new String(md5Hash));
+ "[{}]", file.getCanonicalPath(), file.length(), key);
storeFileWithRetry(key, new BufferedInputStream(new FileInputStream(file)),
md5Hash, file.length());
}
@@ -218,7 +218,7 @@ public void storeFile(
byte[] md5Hash,
long contentLength) throws IOException {
LOG.info("Store file from input stream. COS key: [{}], "
- + "length: [{}], MD5: [{}].", key, contentLength, md5Hash);
+ + "length: [{}].", key, contentLength);
storeFileWithRetry(key, inputStream, md5Hash, contentLength);
}
@@ -250,7 +250,11 @@ public void storeEmptyFile(String key) throws IOException {
public PartETag uploadPart(File file, String key, String uploadId,
int partNum) throws IOException {
InputStream inputStream = new FileInputStream(file);
- return uploadPart(inputStream, key, uploadId, partNum, file.length());
+ try {
+ return uploadPart(inputStream, key, uploadId, partNum, file.length());
+ } finally {
+ inputStream.close();
+ }
}
@Override
diff --git a/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/auth/AbstractCOSCredentialsProvider.java b/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/auth/AbstractCOSCredentialsProvider.java
new file mode 100644
index 0000000000000..1363a7934cba0
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/auth/AbstractCOSCredentialsProvider.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.cosn.auth;
+
+import com.qcloud.cos.auth.COSCredentialsProvider;
+import org.apache.hadoop.conf.Configuration;
+
+import javax.annotation.Nullable;
+import java.net.URI;
+
+/**
+ * The base class for COS credential providers which take a URI or
+ * configuration in their constructor.
+ */
+public abstract class AbstractCOSCredentialsProvider
+ implements COSCredentialsProvider {
+ private final URI uri;
+ private final Configuration conf;
+
+ public AbstractCOSCredentialsProvider(@Nullable URI uri,
+ Configuration conf) {
+ this.uri = uri;
+ this.conf = conf;
+ }
+
+ public URI getUri() {
+ return uri;
+ }
+
+ public Configuration getConf() {
+ return conf;
+ }
+}
\ No newline at end of file
diff --git a/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/auth/COSCredentialProviderList.java b/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/auth/COSCredentialProviderList.java
deleted file mode 100644
index e900b997e4858..0000000000000
--- a/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/auth/COSCredentialProviderList.java
+++ /dev/null
@@ -1,139 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.fs.cosn.auth;
-
-import java.io.Closeable;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import com.google.common.base.Preconditions;
-import com.qcloud.cos.auth.AnonymousCOSCredentials;
-import com.qcloud.cos.auth.COSCredentials;
-import com.qcloud.cos.auth.COSCredentialsProvider;
-import com.qcloud.cos.exception.CosClientException;
-import com.qcloud.cos.utils.StringUtils;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * a list of cos credentials provider.
- */
-public class COSCredentialProviderList implements
- COSCredentialsProvider, AutoCloseable {
- private static final Logger LOG =
- LoggerFactory.getLogger(COSCredentialProviderList.class);
-
- private static final String NO_COS_CREDENTIAL_PROVIDERS =
- "No COS Credential Providers";
- private static final String CREDENTIALS_REQUESTED_WHEN_CLOSED =
- "Credentials requested after provider list was closed";
-
- private final List providers =
- new ArrayList<>(1);
- private boolean reuseLastProvider = true;
- private COSCredentialsProvider lastProvider;
-
- private final AtomicInteger refCount = new AtomicInteger(1);
- private final AtomicBoolean isClosed = new AtomicBoolean(false);
-
- public COSCredentialProviderList() {
- }
-
- public COSCredentialProviderList(
- Collection providers) {
- this.providers.addAll(providers);
- }
-
- public void add(COSCredentialsProvider provider) {
- this.providers.add(provider);
- }
-
- public int getRefCount() {
- return this.refCount.get();
- }
-
- public void checkNotEmpty() {
- if (this.providers.isEmpty()) {
- throw new NoAuthWithCOSException(NO_COS_CREDENTIAL_PROVIDERS);
- }
- }
-
- public COSCredentialProviderList share() {
- Preconditions.checkState(!this.closed(), "Provider list is closed");
- this.refCount.incrementAndGet();
- return this;
- }
-
- public boolean closed() {
- return this.isClosed.get();
- }
-
- @Override
- public COSCredentials getCredentials() {
- if (this.closed()) {
- throw new NoAuthWithCOSException(CREDENTIALS_REQUESTED_WHEN_CLOSED);
- }
-
- this.checkNotEmpty();
-
- if (this.reuseLastProvider && this.lastProvider != null) {
- return this.lastProvider.getCredentials();
- }
-
- for (COSCredentialsProvider provider : this.providers) {
- try {
- COSCredentials credentials = provider.getCredentials();
- if (!StringUtils.isNullOrEmpty(credentials.getCOSAccessKeyId())
- && !StringUtils.isNullOrEmpty(credentials.getCOSSecretKey())
- || credentials instanceof AnonymousCOSCredentials) {
- this.lastProvider = provider;
- return credentials;
- }
- } catch (CosClientException e) {
- LOG.warn("No credentials provided by {}: {}", provider, e.toString());
- }
- }
-
- throw new NoAuthWithCOSException(
- "No COS Credentials provided by " + this.providers.toString());
- }
-
- @Override
- public void close() throws Exception {
- if (this.closed()) {
- return;
- }
-
- int remainder = this.refCount.decrementAndGet();
- if (remainder != 0) {
- return;
- }
- this.isClosed.set(true);
-
- for (COSCredentialsProvider provider : this.providers) {
- if (provider instanceof Closeable) {
- ((Closeable) provider).close();
- }
- }
- }
-
-}
diff --git a/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/auth/COSCredentialsProviderList.java b/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/auth/COSCredentialsProviderList.java
new file mode 100644
index 0000000000000..e4c59a5a27611
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/auth/COSCredentialsProviderList.java
@@ -0,0 +1,145 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.cosn.auth;
+
+import java.io.Closeable;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import com.google.common.base.Preconditions;
+import com.qcloud.cos.auth.AnonymousCOSCredentials;
+import com.qcloud.cos.auth.COSCredentials;
+import com.qcloud.cos.auth.COSCredentialsProvider;
+import com.qcloud.cos.utils.StringUtils;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * a list of cos credentials provider.
+ */
+public class COSCredentialsProviderList implements
+ COSCredentialsProvider, AutoCloseable {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(COSCredentialsProviderList.class);
+
+ private static final String NO_COS_CREDENTIAL_PROVIDERS =
+ "No COS Credential Providers";
+ private static final String CREDENTIALS_REQUESTED_WHEN_CLOSED =
+ "Credentials requested after provider list was closed";
+
+ private final List providers =
+ new ArrayList(1);
+ private boolean reuseLastProvider = true;
+ private COSCredentialsProvider lastProvider;
+
+ private final AtomicInteger refCount = new AtomicInteger(1);
+ private final AtomicBoolean isClosed = new AtomicBoolean(false);
+
+ public COSCredentialsProviderList() {
+ }
+
+ public COSCredentialsProviderList(
+ Collection providers) {
+ this.providers.addAll(providers);
+ }
+
+ public void add(COSCredentialsProvider provider) {
+ this.providers.add(provider);
+ }
+
+ public int getRefCount() {
+ return this.refCount.get();
+ }
+
+ public void checkNotEmpty() {
+ if (this.providers.isEmpty()) {
+ throw new NoAuthWithCOSException(NO_COS_CREDENTIAL_PROVIDERS);
+ }
+ }
+
+ public COSCredentialsProviderList share() {
+ Preconditions.checkState(!this.closed(), "Provider list is closed");
+ this.refCount.incrementAndGet();
+ return this;
+ }
+
+ public boolean closed() {
+ return this.isClosed.get();
+ }
+
+ @Override
+ public COSCredentials getCredentials() {
+ if (this.closed()) {
+ throw new NoAuthWithCOSException(CREDENTIALS_REQUESTED_WHEN_CLOSED);
+ }
+
+ this.checkNotEmpty();
+
+ if (this.reuseLastProvider && this.lastProvider != null) {
+ return this.lastProvider.getCredentials();
+ }
+
+ for (COSCredentialsProvider provider : this.providers) {
+ COSCredentials credentials = provider.getCredentials();
+ if (null != credentials
+ && !StringUtils.isNullOrEmpty(credentials.getCOSAccessKeyId())
+ && !StringUtils.isNullOrEmpty(credentials.getCOSSecretKey())
+ || credentials instanceof AnonymousCOSCredentials) {
+ this.lastProvider = provider;
+ return credentials;
+ }
+ }
+
+ throw new NoAuthWithCOSException(
+ "No COS Credentials provided by " + this.providers.toString());
+ }
+
+ @Override
+ public void refresh() {
+ if (this.closed()) {
+ return;
+ }
+
+ for (COSCredentialsProvider cosCredentialsProvider : this.providers) {
+ cosCredentialsProvider.refresh();
+ }
+ }
+
+ @Override
+ public void close() throws Exception {
+ if (this.closed()) {
+ return;
+ }
+
+ int remainder = this.refCount.decrementAndGet();
+ if (remainder != 0) {
+ return;
+ }
+ this.isClosed.set(true);
+
+ for (COSCredentialsProvider provider : this.providers) {
+ if (provider instanceof Closeable) {
+ ((Closeable) provider).close();
+ }
+ }
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/auth/EnvironmentVariableCredentialProvider.java b/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/auth/EnvironmentVariableCredentialProvider.java
deleted file mode 100644
index 0a7786b882f8b..0000000000000
--- a/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/auth/EnvironmentVariableCredentialProvider.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.fs.cosn.auth;
-
-import com.qcloud.cos.auth.BasicCOSCredentials;
-import com.qcloud.cos.auth.COSCredentials;
-import com.qcloud.cos.auth.COSCredentialsProvider;
-import com.qcloud.cos.exception.CosClientException;
-import com.qcloud.cos.utils.StringUtils;
-
-import org.apache.hadoop.fs.cosn.Constants;
-
-/**
- * the provider obtaining the cos credentials from the environment variables.
- */
-public class EnvironmentVariableCredentialProvider
- implements COSCredentialsProvider {
- @Override
- public COSCredentials getCredentials() {
- String secretId = System.getenv(Constants.COSN_SECRET_ID_ENV);
- String secretKey = System.getenv(Constants.COSN_SECRET_KEY_ENV);
-
- secretId = StringUtils.trim(secretId);
- secretKey = StringUtils.trim(secretKey);
-
- if (!StringUtils.isNullOrEmpty(secretId)
- && !StringUtils.isNullOrEmpty(secretKey)) {
- return new BasicCOSCredentials(secretId, secretKey);
- } else {
- throw new CosClientException(
- "Unable to load COS credentials from environment variables" +
- "(COS_SECRET_ID or COS_SECRET_KEY)");
- }
- }
-
- @Override
- public String toString() {
- return "EnvironmentVariableCredentialProvider{}";
- }
-}
diff --git a/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/auth/EnvironmentVariableCredentialsProvider.java b/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/auth/EnvironmentVariableCredentialsProvider.java
new file mode 100644
index 0000000000000..baa76908b6147
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/auth/EnvironmentVariableCredentialsProvider.java
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.cosn.auth;
+
+import com.qcloud.cos.auth.BasicCOSCredentials;
+import com.qcloud.cos.auth.COSCredentials;
+import com.qcloud.cos.auth.COSCredentialsProvider;
+import com.qcloud.cos.utils.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.cosn.Constants;
+
+import javax.annotation.Nullable;
+import java.net.URI;
+
+/**
+ * The provider obtaining the cos credentials from the environment variables.
+ */
+public class EnvironmentVariableCredentialsProvider
+ extends AbstractCOSCredentialsProvider implements COSCredentialsProvider {
+
+ public EnvironmentVariableCredentialsProvider(@Nullable URI uri,
+ Configuration conf) {
+ super(uri, conf);
+ }
+
+ @Override
+ public COSCredentials getCredentials() {
+ String secretId = System.getenv(Constants.COSN_SECRET_ID_ENV);
+ String secretKey = System.getenv(Constants.COSN_SECRET_KEY_ENV);
+
+ secretId = StringUtils.trim(secretId);
+ secretKey = StringUtils.trim(secretKey);
+
+ if (!StringUtils.isNullOrEmpty(secretId)
+ && !StringUtils.isNullOrEmpty(secretKey)) {
+ return new BasicCOSCredentials(secretId, secretKey);
+ }
+
+ return null;
+ }
+
+ @Override
+ public void refresh() {
+ }
+
+ @Override
+ public String toString() {
+ return String.format("EnvironmentVariableCredentialsProvider{%s, %s}",
+ Constants.COSN_SECRET_ID_ENV,
+ Constants.COSN_SECRET_KEY_ENV);
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/auth/SimpleCredentialProvider.java b/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/auth/SimpleCredentialProvider.java
deleted file mode 100644
index f0635fc0d00cf..0000000000000
--- a/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/auth/SimpleCredentialProvider.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.fs.cosn.auth;
-
-import com.qcloud.cos.auth.BasicCOSCredentials;
-import com.qcloud.cos.auth.COSCredentials;
-import com.qcloud.cos.auth.COSCredentialsProvider;
-import com.qcloud.cos.exception.CosClientException;
-
-import org.apache.commons.lang3.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.cosn.CosNConfigKeys;
-
-/**
- * Get the credentials from the hadoop configuration.
- */
-public class SimpleCredentialProvider implements COSCredentialsProvider {
- private String secretId;
- private String secretKey;
-
- public SimpleCredentialProvider(Configuration conf) {
- this.secretId = conf.get(
- CosNConfigKeys.COSN_SECRET_ID_KEY
- );
- this.secretKey = conf.get(
- CosNConfigKeys.COSN_SECRET_KEY_KEY
- );
- }
-
- @Override
- public COSCredentials getCredentials() {
- if (!StringUtils.isEmpty(this.secretId)
- && !StringUtils.isEmpty(this.secretKey)) {
- return new BasicCOSCredentials(this.secretId, this.secretKey);
- }
- throw new CosClientException("secret id or secret key is unset");
- }
-
-}
diff --git a/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/auth/SimpleCredentialsProvider.java b/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/auth/SimpleCredentialsProvider.java
new file mode 100644
index 0000000000000..107574a87c3aa
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-cos/src/main/java/org/apache/hadoop/fs/cosn/auth/SimpleCredentialsProvider.java
@@ -0,0 +1,60 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.cosn.auth;
+
+import com.qcloud.cos.auth.BasicCOSCredentials;
+import com.qcloud.cos.auth.COSCredentials;
+import com.qcloud.cos.auth.COSCredentialsProvider;
+import com.qcloud.cos.utils.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.cosn.CosNConfigKeys;
+
+import javax.annotation.Nullable;
+import java.net.URI;
+
+/**
+ * Get the credentials from the hadoop configuration.
+ */
+public class SimpleCredentialsProvider
+ extends AbstractCOSCredentialsProvider implements COSCredentialsProvider {
+ private String secretId;
+ private String secretKey;
+
+ public SimpleCredentialsProvider(@Nullable URI uri, Configuration conf) {
+ super(uri, conf);
+ if (null != conf) {
+ this.secretId = conf.get(
+ CosNConfigKeys.COSN_SECRET_ID_KEY);
+ this.secretKey = conf.get(
+ CosNConfigKeys.COSN_SECRET_KEY_KEY);
+ }
+ }
+
+ @Override
+ public COSCredentials getCredentials() {
+ if (!StringUtils.isNullOrEmpty(this.secretId)
+ && !StringUtils.isNullOrEmpty(this.secretKey)) {
+ return new BasicCOSCredentials(this.secretId, this.secretKey);
+ }
+ return null;
+ }
+
+ @Override
+ public void refresh() {
+ }
+}
diff --git a/hadoop-cloud-storage-project/hadoop-cos/site/markdown/cloud-storage/index.md b/hadoop-cloud-storage-project/hadoop-cos/src/site/markdown/cloud-storage/index.md
similarity index 93%
rename from hadoop-cloud-storage-project/hadoop-cos/site/markdown/cloud-storage/index.md
rename to hadoop-cloud-storage-project/hadoop-cos/src/site/markdown/cloud-storage/index.md
index 7049b3f0f013f..9c96ac3659815 100644
--- a/hadoop-cloud-storage-project/hadoop-cos/site/markdown/cloud-storage/index.md
+++ b/hadoop-cloud-storage-project/hadoop-cos/src/site/markdown/cloud-storage/index.md
@@ -130,20 +130,19 @@ Each user needs to properly configure the credentials ( User's secreteId and sec
```xml
fs.cosn.credentials.provider
- org.apache.hadoop.fs.auth.SimpleCredentialProvider
+ org.apache.hadoop.fs.auth.SimpleCredentialsProvider
This option allows the user to specify how to get the credentials.
Comma-separated class names of credential provider classes which implement
com.qcloud.cos.auth.COSCredentialsProvider:
- 1.org.apache.hadoop.fs.auth.SimpleCredentialProvider: Obtain the secret id and secret key
- from fs.cosn.userinfo.secretId and fs.cosn.userinfo.secretKey in core-site.xml
- 2.org.apache.hadoop.fs.auth.EnvironmentVariableCredentialProvider: Obtain the secret id and secret key from system environment variables named COS_SECRET_ID and COS_SECRET_KEY
+ 1.org.apache.hadoop.fs.auth.SimpleCredentialsProvider: Obtain the secret id and secret key from fs.cosn.userinfo.secretId and fs.cosn.userinfo.secretKey in core-site.xml
+ 2.org.apache.hadoop.fs.auth.EnvironmentVariableCredentialsProvider: Obtain the secret id and secret key from system environment variables named COS_SECRET_ID and COS_SECRET_KEY
If unspecified, the default order of credential providers is:
- 1. org.apache.hadoop.fs.auth.SimpleCredentialProvider
- 2. org.apache.hadoop.fs.auth.EnvironmentVariableCredentialProvider
+ 1. org.apache.hadoop.fs.auth.SimpleCredentialsProvider
+ 2. org.apache.hadoop.fs.auth.EnvironmentVariableCredentialsProvider
@@ -237,7 +236,7 @@ Hadoop-COS provides rich runtime properties to set, and most of these do not req
| properties | description | default value | required |
|:----------:|:-----------|:-------------:|:--------:|
| fs.defaultFS | Configure the default file system used by Hadoop.| None | NO |
-| fs.cosn.credentials.provider | This option allows the user to specify how to get the credentials. Comma-separated class names of credential provider classes which implement com.qcloud.cos.auth.COSCredentialsProvider:
1. org.apache.hadoop.fs.cos.auth.SimpleCredentialProvider: Obtain the secret id and secret key from `fs.cosn.userinfo.secretId` and `fs.cosn.userinfo.secretKey` in core-site.xml;
2. org.apache.hadoop.fs.auth.EnvironmentVariableCredentialProvider: Obtain the secret id and secret key from system environment variables named `COSN_SECRET_ID` and `COSN_SECRET_KEY`.
If unspecified, the default order of credential providers is:
1. org.apache.hadoop.fs.auth.SimpleCredentialProvider;
2. org.apache.hadoop.fs.auth.EnvironmentVariableCredentialProvider. | None | NO |
+| fs.cosn.credentials.provider | This option allows the user to specify how to get the credentials. Comma-separated class names of credential provider classes which implement com.qcloud.cos.auth.COSCredentialsProvider:
1. org.apache.hadoop.fs.cos.auth.SimpleCredentialsProvider: Obtain the secret id and secret key from `fs.cosn.userinfo.secretId` and `fs.cosn.userinfo.secretKey` in core-site.xml;
2. org.apache.hadoop.fs.auth.EnvironmentVariableCredentialsProvider: Obtain the secret id and secret key from system environment variables named `COSN_SECRET_ID` and `COSN_SECRET_KEY`.
If unspecified, the default order of credential providers is:
1. org.apache.hadoop.fs.auth.SimpleCredentialsProvider;
2. org.apache.hadoop.fs.auth.EnvironmentVariableCredentialsProvider. | None | NO |
| fs.cosn.userinfo.secretId/secretKey | The API key information of your account | None | YES |
| fs.cosn.bucket.region | The region where the bucket is located. | None | YES |
| fs.cosn.impl | The implementation class of the CosN filesystem. | None | YES |
diff --git a/hadoop-cloud-storage-project/hadoop-cos/site/resources/css/site.css b/hadoop-cloud-storage-project/hadoop-cos/src/site/resources/css/site.css
similarity index 100%
rename from hadoop-cloud-storage-project/hadoop-cos/site/resources/css/site.css
rename to hadoop-cloud-storage-project/hadoop-cos/src/site/resources/css/site.css
diff --git a/hadoop-cloud-storage-project/hadoop-cos/src/test/java/org/apache/hadoop/fs/cosn/TestCosCredentials.java b/hadoop-cloud-storage-project/hadoop-cos/src/test/java/org/apache/hadoop/fs/cosn/TestCosCredentials.java
new file mode 100644
index 0000000000000..8b74f3639ddbd
--- /dev/null
+++ b/hadoop-cloud-storage-project/hadoop-cos/src/test/java/org/apache/hadoop/fs/cosn/TestCosCredentials.java
@@ -0,0 +1,134 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.cosn;
+
+import com.qcloud.cos.auth.COSCredentials;
+import com.qcloud.cos.auth.COSCredentialsProvider;
+import org.apache.hadoop.conf.Configuration;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.fail;
+
+public class TestCosCredentials {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TestCosCredentials.class);
+
+ private final URI fsUri;
+
+ private final String testCosNSecretId = "secretId";
+ private final String testCosNSecretKey = "secretKey";
+ private final String testCosNEnvSecretId = "env_secretId";
+ private final String testCosNEnvSecretKey = "env_secretKey";
+
+ public TestCosCredentials() throws URISyntaxException {
+ // A fake uri for tests.
+ this.fsUri = new URI("cosn://test-bucket-1250000000");
+ }
+
+ @Test
+ public void testSimpleCredentialsProvider() throws Throwable {
+ Configuration configuration = new Configuration();
+ configuration.set(CosNConfigKeys.COSN_SECRET_ID_KEY,
+ testCosNSecretId);
+ configuration.set(CosNConfigKeys.COSN_SECRET_KEY_KEY,
+ testCosNSecretKey);
+ validateCredentials(this.fsUri, configuration);
+ }
+
+ @Test
+ public void testEnvironmentCredentialsProvider() throws Throwable {
+ Configuration configuration = new Configuration();
+ // Set EnvironmentVariableCredentialsProvider as the CosCredentials
+ // Provider.
+ configuration.set(CosNConfigKeys.COSN_CREDENTIALS_PROVIDER,
+ "org.apache.hadoop.fs.cosn.EnvironmentVariableCredentialsProvider");
+ // Set the environment variables storing the secret id and secret key.
+ System.setProperty(Constants.COSN_SECRET_ID_ENV, testCosNEnvSecretId);
+ System.setProperty(Constants.COSN_SECRET_KEY_ENV, testCosNEnvSecretKey);
+ validateCredentials(this.fsUri, configuration);
+ }
+
+ private void validateCredentials(URI uri, Configuration configuration)
+ throws IOException {
+ if (null != configuration) {
+ COSCredentialsProvider credentialsProvider =
+ CosNUtils.createCosCredentialsProviderSet(uri, configuration);
+ COSCredentials cosCredentials = credentialsProvider.getCredentials();
+ assertNotNull("The cos credentials obtained is null.", cosCredentials);
+ if (configuration.get(
+ CosNConfigKeys.COSN_CREDENTIALS_PROVIDER).compareToIgnoreCase(
+ "org.apache.hadoop.fs.cosn.EnvironmentVariableCredentialsProvider")
+ == 0) {
+ if (null == cosCredentials.getCOSAccessKeyId()
+ || cosCredentials.getCOSAccessKeyId().isEmpty()
+ || null == cosCredentials.getCOSSecretKey()
+ || cosCredentials.getCOSSecretKey().isEmpty()) {
+ String failMessage = String.format(
+ "Test EnvironmentVariableCredentialsProvider failed. The " +
+ "expected is [secretId: %s, secretKey: %s], but got null or" +
+ " empty.", testCosNEnvSecretId, testCosNEnvSecretKey);
+ fail(failMessage);
+ }
+
+ if (cosCredentials.getCOSAccessKeyId()
+ .compareTo(testCosNEnvSecretId) != 0
+ || cosCredentials.getCOSSecretKey()
+ .compareTo(testCosNEnvSecretKey) != 0) {
+ String failMessage = String.format("Test " +
+ "EnvironmentVariableCredentialsProvider failed. " +
+ "The expected is [secretId: %s, secretKey: %s], but got is " +
+ "[secretId:%s, secretKey:%s].", testCosNEnvSecretId,
+ testCosNEnvSecretKey, cosCredentials.getCOSAccessKeyId(),
+ cosCredentials.getCOSSecretKey());
+ }
+ // expected
+ } else {
+ if (null == cosCredentials.getCOSAccessKeyId()
+ || cosCredentials.getCOSAccessKeyId().isEmpty()
+ || null == cosCredentials.getCOSSecretKey()
+ || cosCredentials.getCOSSecretKey().isEmpty()) {
+ String failMessage = String.format(
+ "Test COSCredentials failed. The " +
+ "expected is [secretId: %s, secretKey: %s], but got null or" +
+ " empty.", testCosNSecretId, testCosNSecretKey);
+ fail(failMessage);
+ }
+ if (cosCredentials.getCOSAccessKeyId()
+ .compareTo(testCosNSecretId) != 0
+ || cosCredentials.getCOSSecretKey()
+ .compareTo(testCosNSecretKey) != 0) {
+ String failMessage = String.format("Test " +
+ "EnvironmentVariableCredentialsProvider failed. " +
+ "The expected is [secretId: %s, secretKey: %s], but got is " +
+ "[secretId:%s, secretKey:%s].", testCosNSecretId,
+ testCosNSecretKey, cosCredentials.getCOSAccessKeyId(),
+ cosCredentials.getCOSSecretKey());
+ fail(failMessage);
+ }
+ // expected
+ }
+ }
+ }
+}
diff --git a/hadoop-cloud-storage-project/pom.xml b/hadoop-cloud-storage-project/pom.xml
index f39e8c3aaf9f8..da0d88a8117b8 100644
--- a/hadoop-cloud-storage-project/pom.xml
+++ b/hadoop-cloud-storage-project/pom.xml
@@ -20,11 +20,11 @@
org.apache.hadoop
hadoop-project
- 3.3.0-SNAPSHOT
+ 3.4.0-SNAPSHOT
../hadoop-project
hadoop-cloud-storage-project
- 3.3.0-SNAPSHOT
+ 3.4.0-SNAPSHOT
Apache Hadoop Cloud Storage Project
Apache Hadoop Cloud Storage Project
pom
diff --git a/hadoop-common-project/hadoop-annotations/pom.xml b/hadoop-common-project/hadoop-annotations/pom.xml
index 738f0ada8f1e9..a262d55b0426c 100644
--- a/hadoop-common-project/hadoop-annotations/pom.xml
+++ b/hadoop-common-project/hadoop-annotations/pom.xml
@@ -20,11 +20,11 @@
org.apache.hadoop
hadoop-project
- 3.3.0-SNAPSHOT
+ 3.4.0-SNAPSHOT
../../hadoop-project
hadoop-annotations
- 3.3.0-SNAPSHOT
+ 3.4.0-SNAPSHOT
Apache Hadoop Annotations
Apache Hadoop Annotations
jar
diff --git a/hadoop-common-project/hadoop-auth-examples/pom.xml b/hadoop-common-project/hadoop-auth-examples/pom.xml
index fb904912999b8..4deda432797e0 100644
--- a/hadoop-common-project/hadoop-auth-examples/pom.xml
+++ b/hadoop-common-project/hadoop-auth-examples/pom.xml
@@ -20,11 +20,11 @@
org.apache.hadoop
hadoop-project
- 3.3.0-SNAPSHOT
+ 3.4.0-SNAPSHOT
../../hadoop-project
hadoop-auth-examples
- 3.3.0-SNAPSHOT
+ 3.4.0-SNAPSHOT
war
Apache Hadoop Auth Examples
diff --git a/hadoop-common-project/hadoop-auth/pom.xml b/hadoop-common-project/hadoop-auth/pom.xml
index 20a3e7059b154..4ff3bc14927fe 100644
--- a/hadoop-common-project/hadoop-auth/pom.xml
+++ b/hadoop-common-project/hadoop-auth/pom.xml
@@ -20,11 +20,11 @@
org.apache.hadoop
hadoop-project
- 3.3.0-SNAPSHOT
+ 3.4.0-SNAPSHOT
../../hadoop-project
hadoop-auth
- 3.3.0-SNAPSHOT
+ 3.4.0-SNAPSHOT
jar
Apache Hadoop Auth
diff --git a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/AuthenticatedURL.java b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/AuthenticatedURL.java
index 1093d8a2539eb..488400647cf06 100644
--- a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/AuthenticatedURL.java
+++ b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/AuthenticatedURL.java
@@ -99,7 +99,10 @@ public void put(URI uri, Map> responseHeaders) {
cookies = HttpCookie.parse(header);
} catch (IllegalArgumentException iae) {
// don't care. just skip malformed cookie headers.
- LOG.debug("Cannot parse cookie header: " + header, iae);
+ // When header is empty - "Cannot parse cookie header, header = ,
+ // reason = Empty cookie header string"
+ LOG.debug("Cannot parse cookie header, header = {}, reason = {} ",
+ header, iae.getMessage());
continue;
}
for (HttpCookie cookie : cookies) {
diff --git a/hadoop-common-project/hadoop-common/dev-support/findbugsExcludeFile.xml b/hadoop-common-project/hadoop-common/dev-support/findbugsExcludeFile.xml
index 802197e33cbcd..cf5c3874d1063 100644
--- a/hadoop-common-project/hadoop-common/dev-support/findbugsExcludeFile.xml
+++ b/hadoop-common-project/hadoop-common/dev-support/findbugsExcludeFile.xml
@@ -460,4 +460,10 @@
+
+
+
+
+
+
diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml
index 84d3ae5b5addc..dd058812fc774 100644
--- a/hadoop-common-project/hadoop-common/pom.xml
+++ b/hadoop-common-project/hadoop-common/pom.xml
@@ -20,11 +20,11 @@
org.apache.hadoop
hadoop-project-dist
- 3.3.0-SNAPSHOT
+ 3.4.0-SNAPSHOT
../../hadoop-project-dist
hadoop-common
- 3.3.0-SNAPSHOT
+ 3.4.0-SNAPSHOT
Apache Hadoop Common
Apache Hadoop Common
jar
@@ -39,6 +39,10 @@
+
+ org.apache.hadoop.thirdparty
+ hadoop-shaded-protobuf_3_7
+
org.apache.hadoop
hadoop-annotations
@@ -90,8 +94,8 @@
compile
- javax.activation
- javax.activation-api
+ jakarta.activation
+ jakarta.activation-api
runtime
@@ -272,6 +276,11 @@
sshd-core
test
+
+ org.apache.ftpserver
+ ftpserver-core
+ test
+
org.apache.htrace
@@ -346,12 +355,12 @@
org.wildfly.openssl
wildfly-openssl
- provided
+ test
- org.assertj
- assertj-core
- test
+ org.wildfly.openssl
+ wildfly-openssl-java
+ provided
@@ -394,6 +403,36 @@
+
+ com.google.code.maven-replacer-plugin
+ replacer
+
+
+ replace-generated-sources
+
+ false
+
+
+
+ replace-generated-test-sources
+
+ false
+
+
+
+ replace-sources
+
+ false
+
+
+
+ replace-test-sources
+
+ false
+
+
+
+
org.apache.hadoop
hadoop-maven-plugins
@@ -842,9 +881,13 @@
parallel-tests-createdir
+ process-test-resources
parallel-tests-createdir
+
+ ${test.build.data}
+
@@ -852,6 +895,7 @@
org.apache.maven.plugins
maven-surefire-plugin
+ ${ignoreTestFailure}
${testsThreadCount}
false
${maven-surefire-plugin.argLine} -DminiClusterDedicatedDirs=true
diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
index 484fe2302f9ba..4be554aef6c25 100755
--- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
+++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
@@ -213,7 +213,7 @@ function hadoop_privilege_check
[[ "${EUID}" = 0 ]]
}
-## @description Execute a command via su when running as root
+## @description Execute a command via sudo when running as root
## @description if the given user is found or exit with
## @description failure if not.
## @description otherwise just run it. (This is intended to
@@ -224,14 +224,14 @@ function hadoop_privilege_check
## @param user
## @param commandstring
## @return exitstatus
-function hadoop_su
+function hadoop_sudo
{
declare user=$1
shift
if hadoop_privilege_check; then
if hadoop_verify_user_resolves user; then
- su -l "${user}" -- "$@"
+ sudo -u "${user}" -- "$@"
else
hadoop_error "ERROR: Refusing to run as root: ${user} account is not found. Aborting."
return 1
@@ -241,7 +241,7 @@ function hadoop_su
fi
}
-## @description Execute a command via su when running as root
+## @description Execute a command via sudo when running as root
## @description with extra support for commands that might
## @description legitimately start as root (e.g., datanode)
## @description (This is intended to
@@ -259,7 +259,7 @@ function hadoop_uservar_su
#
# if $EUID != 0, then exec
# if $EUID =0 then
- # if hdfs_subcmd_user is defined, call hadoop_su to exec
+ # if hdfs_subcmd_user is defined, call hadoop_sudo to exec
# if hdfs_subcmd_user is not defined, error
#
# For secure daemons, this means both the secure and insecure env vars need to be
@@ -283,7 +283,7 @@ function hadoop_uservar_su
svar=$(hadoop_build_custom_subcmd_var "${program}" "${command}" SECURE_USER)
if [[ -n "${!uvar}" ]]; then
- hadoop_su "${!uvar}" "$@"
+ hadoop_sudo "${!uvar}" "$@"
elif [[ -n "${!svar}" ]]; then
## if we are here, then SECURE_USER with no USER defined
## we are already privileged, so just run the command and hope
@@ -596,11 +596,6 @@ function hadoop_bootstrap
YARN_LIB_JARS_DIR=${YARN_LIB_JARS_DIR:-"share/hadoop/yarn/lib"}
MAPRED_DIR=${MAPRED_DIR:-"share/hadoop/mapreduce"}
MAPRED_LIB_JARS_DIR=${MAPRED_LIB_JARS_DIR:-"share/hadoop/mapreduce/lib"}
- HDDS_DIR=${HDDS_DIR:-"share/hadoop/hdds"}
- HDDS_LIB_JARS_DIR=${HDDS_LIB_JARS_DIR:-"share/hadoop/hdds/lib"}
- OZONE_DIR=${OZONE_DIR:-"share/hadoop/ozone"}
- OZONE_LIB_JARS_DIR=${OZONE_LIB_JARS_DIR:-"share/hadoop/ozone/lib"}
- OZONEFS_DIR=${OZONEFS_DIR:-"share/hadoop/ozonefs"}
HADOOP_TOOLS_HOME=${HADOOP_TOOLS_HOME:-${HADOOP_HOME}}
HADOOP_TOOLS_DIR=${HADOOP_TOOLS_DIR:-"share/hadoop/tools"}
@@ -1342,7 +1337,7 @@ function hadoop_add_to_classpath_tools
# shellcheck disable=SC1090
. "${HADOOP_LIBEXEC_DIR}/tools/${module}.sh"
else
- hadoop_error "ERROR: Tools helper ${HADOOP_LIBEXEC_DIR}/tools/${module}.sh was not found."
+ hadoop_debug "Tools helper ${HADOOP_LIBEXEC_DIR}/tools/${module}.sh was not found."
fi
if declare -f hadoop_classpath_tools_${module} >/dev/null 2>&1; then
@@ -1921,6 +1916,22 @@ function hadoop_start_secure_daemon
exit 1
fi
+ if [[ -z "${HADOOP_DAEMON_JSVC_EXTRA_OPTS}" ]]; then
+ # If HADOOP_DAEMON_JSVC_EXTRA_OPTS is not set
+ if ${jsvc} -help | grep -q "\-cwd"; then
+ # Check if jsvc -help has entry for option -cwd
+ hadoop_debug "Your jsvc supports -cwd option." \
+ "Adding option '-cwd .'. See HADOOP-16276 for details."
+ HADOOP_DAEMON_JSVC_EXTRA_OPTS="-cwd ."
+ else
+ hadoop_debug "Your jsvc doesn't support -cwd option." \
+ "No need to add option '-cwd .'. See HADOOP-16276 for details."
+ fi
+ else
+ hadoop_debug "HADOOP_DAEMON_JSVC_EXTRA_OPTS is set." \
+ "Ignoring jsvc -cwd option detection and addition."
+ fi
+
# note that shellcheck will throw a
# bogus for-our-use-case 2086 here.
# it doesn't properly support multi-line situations
@@ -2035,7 +2046,8 @@ function hadoop_start_secure_daemon_wrapper
hadoop_error "ERROR: Cannot disconnect ${daemonname} process $!"
fi
# capture the ulimit output
- su "${HADOOP_SECURE_USER}" -c 'bash -c "ulimit -a"' >> "${jsvcoutfile}" 2>&1
+ #shellcheck disable=SC2024
+ sudo -u "${HADOOP_SECURE_USER}" bash -c "ulimit -a" >> "${jsvcoutfile}" 2>&1
#shellcheck disable=SC2086
if ! ps -p $! >/dev/null 2>&1; then
return 1
diff --git a/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh b/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh
index e43cd95b047ee..f4625f5999b1c 100644
--- a/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh
+++ b/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh
@@ -390,15 +390,6 @@ export HADOOP_OS_TYPE=${HADOOP_OS_TYPE:-$(uname -s)}
#
# export HDFS_DFSROUTER_OPTS=""
-###
-# Ozone Manager specific parameters
-###
-# Specify the JVM options to be used when starting the Ozone Manager.
-# These options will be appended to the options specified as HADOOP_OPTS
-# and therefore may override any similar flags set in HADOOP_OPTS
-#
-# export HDFS_OM_OPTS=""
-
###
# HDFS StorageContainerManager specific parameters
###
diff --git a/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties b/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties
index 7f9ea462679b3..52d2c1ff038e6 100644
--- a/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties
+++ b/hadoop-common-project/hadoop-common/src/main/conf/log4j.properties
@@ -282,13 +282,6 @@ log4j.appender.NMAUDIT.MaxBackupIndex=${nm.audit.log.maxbackupindex}
#log4j.appender.nodemanagerrequestlog.Filename=${hadoop.log.dir}/jetty-nodemanager-yyyy_mm_dd.log
#log4j.appender.nodemanagerrequestlog.RetainDays=3
-#Http Server request logs for Ozone S3Gateway
-log4j.logger.http.requests.s3gateway=INFO,s3gatewayrequestlog
-log4j.appender.s3gatewayrequestlog=org.apache.hadoop.http.HttpRequestLogAppender
-log4j.appender.s3gatewayrequestlog.Filename=${hadoop.log.dir}/jetty-s3gateway-yyyy_mm_dd.log
-log4j.appender.s3gatewayrequestlog.RetainDays=3
-
-
# WebHdfs request log on datanodes
# Specify -Ddatanode.webhdfs.logger=INFO,HTTPDRFA on datanode startup to
# direct the log to a separate file.
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
index 180bde26574ca..9751a9b66945c 100755
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
@@ -51,7 +51,6 @@
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
-import java.util.LinkedList;
import java.util.List;
import java.util.ListIterator;
import java.util.Map;
@@ -1691,7 +1690,11 @@ public boolean getBoolean(String name, boolean defaultValue) {
return true;
else if (StringUtils.equalsIgnoreCase("false", valueString))
return false;
- else return defaultValue;
+ else {
+ LOG.warn("Invalid value for boolean: " + valueString +
+ ", choose default value: " + defaultValue + " for " + name);
+ return defaultValue;
+ }
}
/**
@@ -3350,6 +3353,7 @@ void parseNext() throws IOException, XMLStreamException {
handleStartElement();
break;
case XMLStreamConstants.CHARACTERS:
+ case XMLStreamConstants.CDATA:
if (parseToken) {
char[] text = reader.getTextCharacters();
token.append(text, reader.getTextStart(), reader.getTextLength());
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java
index 0453ca14537c3..1df68b647c99a 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java
@@ -31,7 +31,6 @@
import java.util.List;
import java.util.Map;
import java.util.NoSuchElementException;
-import java.util.Set;
import java.util.StringTokenizer;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
@@ -45,6 +44,7 @@
import org.apache.hadoop.fs.Options.CreateOpts;
import org.apache.hadoop.fs.Options.Rename;
import org.apache.hadoop.fs.impl.AbstractFSBuilderImpl;
+import org.apache.hadoop.fs.impl.OpenFileParameters;
import org.apache.hadoop.fs.permission.AclEntry;
import org.apache.hadoop.fs.permission.AclStatus;
import org.apache.hadoop.fs.permission.FsAction;
@@ -1355,22 +1355,20 @@ public boolean equals(Object other) {
* setting up the expectation that the {@code get()} call
* is needed to evaluate the result.
* @param path path to the file
- * @param mandatoryKeys set of options declared as mandatory.
- * @param options options set during the build sequence.
- * @param bufferSize buffer size
+ * @param parameters open file parameters from the builder.
* @return a future which will evaluate to the opened file.
* @throws IOException failure to resolve the link.
* @throws IllegalArgumentException unknown mandatory key
*/
public CompletableFuture openFileWithOptions(Path path,
- Set mandatoryKeys,
- Configuration options,
- int bufferSize) throws IOException {
- AbstractFSBuilderImpl.rejectUnknownMandatoryKeys(mandatoryKeys,
+ final OpenFileParameters parameters) throws IOException {
+ AbstractFSBuilderImpl.rejectUnknownMandatoryKeys(
+ parameters.getMandatoryKeys(),
Collections.emptySet(),
"for " + path);
return LambdaUtils.eval(
- new CompletableFuture<>(), () -> open(path, bufferSize));
+ new CompletableFuture<>(), () ->
+ open(path, parameters.getBufferSize()));
}
public boolean hasPathCapability(final Path path,
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/BatchListingOperations.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/BatchListingOperations.java
new file mode 100644
index 0000000000000..f72b1e288eb49
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/BatchListingOperations.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * Interface filesystems MAY implement to offer a batched list.
+ * If implemented, filesystems SHOULD declare
+ * {@link CommonPathCapabilities#FS_EXPERIMENTAL_BATCH_LISTING} to be a supported
+ * path capability.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+public interface BatchListingOperations {
+
+ /**
+ * Batched listing API that returns {@link PartialListing}s for the
+ * passed Paths.
+ *
+ * @param paths List of paths to list.
+ * @return RemoteIterator that returns corresponding PartialListings.
+ * @throws IOException failure
+ */
+ RemoteIterator> batchedListStatusIterator(
+ List paths) throws IOException;
+
+ /**
+ * Batched listing API that returns {@link PartialListing}s for the passed
+ * Paths. The PartialListing will contain {@link LocatedFileStatus} entries
+ * with locations.
+ *
+ * @param paths List of paths to list.
+ * @return RemoteIterator that returns corresponding PartialListings.
+ * @throws IOException failure
+ */
+ RemoteIterator>
+ batchedListLocatedStatusIterator(
+ List paths) throws IOException;
+
+}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CachingGetSpaceUsed.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CachingGetSpaceUsed.java
index 92476d77ddb44..58dc82d2efb2d 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CachingGetSpaceUsed.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CachingGetSpaceUsed.java
@@ -47,6 +47,7 @@ public abstract class CachingGetSpaceUsed implements Closeable, GetSpaceUsed {
private final long jitter;
private final String dirPath;
private Thread refreshUsed;
+ private boolean shouldFirstRefresh;
/**
* This is the constructor used by the builder.
@@ -79,16 +80,30 @@ public CachingGetSpaceUsed(CachingGetSpaceUsed.Builder builder)
this.refreshInterval = interval;
this.jitter = jitter;
this.used.set(initialUsed);
+ this.shouldFirstRefresh = true;
}
void init() {
if (used.get() < 0) {
used.set(0);
+ if (!shouldFirstRefresh) {
+ // Skip initial refresh operation, so we need to do first refresh
+ // operation immediately in refresh thread.
+ initRefeshThread(true);
+ return;
+ }
refresh();
}
+ initRefeshThread(false);
+ }
+ /**
+ * RunImmediately should set true, if we skip the first refresh.
+ * @param runImmediately The param default should be false.
+ */
+ private void initRefeshThread (boolean runImmediately) {
if (refreshInterval > 0) {
- refreshUsed = new Thread(new RefreshThread(this),
+ refreshUsed = new Thread(new RefreshThread(this, runImmediately),
"refreshUsed-" + dirPath);
refreshUsed.setDaemon(true);
refreshUsed.start();
@@ -100,6 +115,14 @@ void init() {
protected abstract void refresh();
+ /**
+ * Reset that if we need to do the first refresh.
+ * @param shouldFirstRefresh The flag value to set.
+ */
+ protected void setShouldFirstRefresh(boolean shouldFirstRefresh) {
+ this.shouldFirstRefresh = shouldFirstRefresh;
+ }
+
/**
* @return an estimate of space used in the directory path.
*/
@@ -156,9 +179,11 @@ public void close() throws IOException {
private static final class RefreshThread implements Runnable {
final CachingGetSpaceUsed spaceUsed;
+ private boolean runImmediately;
- RefreshThread(CachingGetSpaceUsed spaceUsed) {
+ RefreshThread(CachingGetSpaceUsed spaceUsed, boolean runImmediately) {
this.spaceUsed = spaceUsed;
+ this.runImmediately = runImmediately;
}
@Override
@@ -176,7 +201,10 @@ public void run() {
}
// Make sure that after the jitter we didn't end up at 0.
refreshInterval = Math.max(refreshInterval, 1);
- Thread.sleep(refreshInterval);
+ if (!runImmediately) {
+ Thread.sleep(refreshInterval);
+ }
+ runImmediately = false;
// update the used variable
spaceUsed.refresh();
} catch (InterruptedException e) {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java
index 5e5d29a28bfce..cc9c284c9fa55 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java
@@ -27,8 +27,6 @@
import java.util.Collections;
import java.util.EnumSet;
import java.util.List;
-import java.util.Locale;
-import java.util.Set;
import java.util.concurrent.CompletableFuture;
import com.google.common.base.Preconditions;
@@ -37,6 +35,7 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.impl.AbstractFSBuilderImpl;
import org.apache.hadoop.fs.impl.FutureDataInputStreamBuilderImpl;
+import org.apache.hadoop.fs.impl.OpenFileParameters;
import org.apache.hadoop.fs.permission.AclEntry;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.util.DataChecksum;
@@ -845,14 +844,14 @@ public FutureDataInputStreamBuilder openFile(final Path path)
@Override
protected CompletableFuture openFileWithOptions(
final Path path,
- final Set mandatoryKeys,
- final Configuration options,
- final int bufferSize) throws IOException {
- AbstractFSBuilderImpl.rejectUnknownMandatoryKeys(mandatoryKeys,
+ final OpenFileParameters parameters) throws IOException {
+ AbstractFSBuilderImpl.rejectUnknownMandatoryKeys(
+ parameters.getMandatoryKeys(),
Collections.emptySet(),
"for " + path);
return LambdaUtils.eval(
- new CompletableFuture<>(), () -> open(path, bufferSize));
+ new CompletableFuture<>(),
+ () -> open(path, parameters.getBufferSize()));
}
/**
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ClusterStorageCapacityExceededException.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ClusterStorageCapacityExceededException.java
index bbbf073cc241c..908d67723e7be 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ClusterStorageCapacityExceededException.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ClusterStorageCapacityExceededException.java
@@ -27,7 +27,7 @@
* cluster filesystem is exceeded. See also
* https://issues.apache.org/jira/browse/MAPREDUCE-7148.
*/
-@InterfaceAudience.LimitedPrivate({ "HDFS", "MapReduce" })
+@InterfaceAudience.LimitedPrivate({ "HDFS", "MapReduce", "Tez" })
@InterfaceStability.Evolving
public class ClusterStorageCapacityExceededException extends IOException {
private static final long serialVersionUID = 1L;
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
index 8c09db1284cff..c08af395ad2f9 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
@@ -114,6 +114,9 @@ public class CommonConfigurationKeys extends CommonConfigurationKeysPublic {
"callqueue.overflow.trigger.failover";
public static final boolean IPC_CALLQUEUE_SERVER_FAILOVER_ENABLE_DEFAULT =
false;
+ /** Callqueue subqueue capacity weights. */
+ public static final String IPC_CALLQUEUE_CAPACITY_WEIGHTS_KEY =
+ "callqueue.capacity.weights";
/**
* IPC scheduler priority levels.
@@ -426,4 +429,13 @@ public class CommonConfigurationKeys extends CommonConfigurationKeysPublic {
"dfs.client.ignore.namenode.default.kms.uri";
public static final boolean
DFS_CLIENT_IGNORE_NAMENODE_DEFAULT_KMS_URI_DEFAULT = false;
+
+ /**
+ * Whether or not ThreadMXBean is used for getting thread info in JvmMetrics,
+ * ThreadGroup approach is preferred for better performance.
+ */
+ public static final String HADOOP_METRICS_JVM_USE_THREAD_MXBEAN =
+ "hadoop.metrics.jvm.use-thread-mxbean";
+ public static final boolean HADOOP_METRICS_JVM_USE_THREAD_MXBEAN_DEFAULT =
+ false;
}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
index a68012b06d2bc..ce132f9a37891 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
@@ -988,5 +988,14 @@ public class CommonConfigurationKeysPublic {
public static final String HADOOP_PROMETHEUS_ENABLED =
"hadoop.prometheus.endpoint.enabled";
public static final boolean HADOOP_PROMETHEUS_ENABLED_DEFAULT = false;
+
+ /**
+ * @see
+ *
+ * core-default.xml
+ */
+ public static final String HADOOP_HTTP_IDLE_TIMEOUT_MS_KEY =
+ "hadoop.http.idle_timeout.ms";
+ public static final int HADOOP_HTTP_IDLE_TIMEOUT_MS_DEFAULT = 1000;
}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonPathCapabilities.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonPathCapabilities.java
index 31e6bac0ccee5..fb46ef81e36fa 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonPathCapabilities.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonPathCapabilities.java
@@ -18,6 +18,8 @@
package org.apache.hadoop.fs;
+import org.apache.hadoop.classification.InterfaceStability;
+
/**
* Common path capabilities.
*/
@@ -123,4 +125,10 @@ private CommonPathCapabilities() {
*/
public static final String FS_XATTRS = "fs.capability.paths.xattrs";
+ /**
+ * Probe for support for {@link BatchListingOperations}.
+ */
+ @InterfaceStability.Unstable
+ public static final String FS_EXPERIMENTAL_BATCH_LISTING =
+ "fs.capability.batch.listing";
}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ContentSummary.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ContentSummary.java
index cdbd10f636dd3..20e205a8b32cf 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ContentSummary.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ContentSummary.java
@@ -281,6 +281,21 @@ public int hashCode() {
private static final String ALL_HEADER = QUOTA_HEADER + SUMMARY_HEADER;
+ /**
+ * Output format:<-------18-------> <----------24---------->
+ * <----------24---------->. <-------------28------------> SNAPSHOT_LENGTH
+ * SNAPSHOT_FILE_COUNT SNAPSHOT_DIR_COUNT SNAPSHOT_SPACE_CONSUMED
+ */
+ private static final String SNAPSHOT_FORMAT = "%18s %24s %24s %28s ";
+
+ private static final String[] SNAPSHOT_HEADER_FIELDS =
+ new String[] {"SNAPSHOT_LENGTH", "SNAPSHOT_FILE_COUNT",
+ "SNAPSHOT_DIR_COUNT", "SNAPSHOT_SPACE_CONSUMED"};
+
+ /** The header string. */
+ private static final String SNAPSHOT_HEADER =
+ String.format(SNAPSHOT_FORMAT, (Object[]) SNAPSHOT_HEADER_FIELDS);
+
/** Return the header of the output.
* if qOption is false, output directory count, file count, and content size;
@@ -293,7 +308,9 @@ public static String getHeader(boolean qOption) {
return qOption ? ALL_HEADER : SUMMARY_HEADER;
}
-
+ public static String getSnapshotHeader() {
+ return SNAPSHOT_HEADER;
+ }
/**
* Returns the names of the fields from the summary header.
@@ -416,7 +433,7 @@ public String toString(boolean qOption, boolean hOption, boolean tOption,
}
/**
- * Formats a size to be human readable or in bytes
+ * Formats a size to be human readable or in bytes.
* @param size value to be formatted
* @param humanReadable flag indicating human readable or not
* @return String representation of the size
@@ -426,4 +443,17 @@ private String formatSize(long size, boolean humanReadable) {
? StringUtils.TraditionalBinaryPrefix.long2String(size, "", 1)
: String.valueOf(size);
}
+
+ /**
+ * Return the string representation of the snapshot counts in the output
+ * format.
+ * @param hOption flag indicating human readable or not
+ * @return String representation of the snapshot counts
+ */
+ public String toSnapshot(boolean hOption) {
+ return String.format(SNAPSHOT_FORMAT, formatSize(snapshotLength, hOption),
+ formatSize(snapshotFileCount, hOption),
+ formatSize(snapshotDirectoryCount, hOption),
+ formatSize(snapshotSpaceConsumed, hOption));
+ }
}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CreateFlag.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CreateFlag.java
index 58b5f704bb831..71993713ad2eb 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CreateFlag.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CreateFlag.java
@@ -123,7 +123,13 @@ public enum CreateFlag {
* locality. The first block replica should be placed randomly within the
* cluster. Subsequent block replicas should follow DataNode locality rules.
*/
- IGNORE_CLIENT_LOCALITY((short) 0x100);
+ IGNORE_CLIENT_LOCALITY((short) 0x100),
+
+ /**
+ * Advise that a block replica NOT be written to the local rack DataNode where
+ * 'local' means the same rack as the client is being run on.
+ */
+ NO_LOCAL_RACK((short) 0x120);
private final short mode;
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegateToFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegateToFileSystem.java
index a8f294f379158..3a139781e0372 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegateToFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/DelegateToFileSystem.java
@@ -24,13 +24,13 @@
import java.util.Arrays;
import java.util.EnumSet;
import java.util.List;
-import java.util.Set;
import java.util.concurrent.CompletableFuture;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Options.ChecksumOpt;
+import org.apache.hadoop.fs.impl.OpenFileParameters;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.util.Progressable;
@@ -266,20 +266,17 @@ public List> getDelegationTokens(String renewer) throws IOException {
/**
* Open a file by delegating to
- * {@link FileSystem#openFileWithOptions(Path, Set, Configuration, int)}.
+ * {@link FileSystem#openFileWithOptions(Path, org.apache.hadoop.fs.impl.OpenFileParameters)}.
* @param path path to the file
- * @param mandatoryKeys set of options declared as mandatory.
- * @param options options set during the build sequence.
- * @param bufferSize buffer size
- * @return a future which will evaluate to the opened file.
+ * @param parameters open file parameters from the builder.
+ *
+ * @return a future which will evaluate to the opened file.ControlAlpha
* @throws IOException failure to resolve the link.
* @throws IllegalArgumentException unknown mandatory key
*/
public CompletableFuture openFileWithOptions(Path path,
- Set mandatoryKeys,
- Configuration options,
- int bufferSize) throws IOException {
- return fsImpl.openFileWithOptions(path, mandatoryKeys, options, bufferSize);
+ final OpenFileParameters parameters) throws IOException {
+ return fsImpl.openFileWithOptions(path, parameters);
}
@Override
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
index b2c1369a9c1fe..ba0064f0813d3 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
@@ -47,7 +47,7 @@
import org.apache.hadoop.fs.Options.CreateOpts;
import org.apache.hadoop.fs.impl.FutureDataInputStreamBuilderImpl;
import org.apache.hadoop.fs.impl.FsLinkResolution;
-import org.apache.hadoop.fs.impl.PathCapabilitiesSupport;
+import org.apache.hadoop.fs.impl.OpenFileParameters;
import org.apache.hadoop.fs.permission.AclEntry;
import org.apache.hadoop.fs.permission.AclStatus;
import org.apache.hadoop.fs.permission.FsAction;
@@ -483,7 +483,7 @@ public static FileContext getFileContext(final URI defaultFsUri,
*/
public static FileContext getFileContext(final Configuration aConf)
throws UnsupportedFileSystemException {
- final URI defaultFsUri = URI.create(aConf.get(FS_DEFAULT_NAME_KEY,
+ final URI defaultFsUri = URI.create(aConf.getTrimmed(FS_DEFAULT_NAME_KEY,
FS_DEFAULT_NAME_DEFAULT));
if ( defaultFsUri.getScheme() != null
&& !defaultFsUri.getScheme().trim().isEmpty()) {
@@ -2924,16 +2924,18 @@ protected FSDataInputStreamBuilder(
@Override
public CompletableFuture build() throws IOException {
final Path absF = fixRelativePart(getPath());
+ OpenFileParameters parameters = new OpenFileParameters()
+ .withMandatoryKeys(getMandatoryKeys())
+ .withOptions(getOptions())
+ .withBufferSize(getBufferSize())
+ .withStatus(getStatus());
return new FSLinkResolver>() {
@Override
public CompletableFuture next(
final AbstractFileSystem fs,
final Path p)
throws IOException {
- return fs.openFileWithOptions(p,
- getMandatoryKeys(),
- getOptions(),
- getBufferSize());
+ return fs.openFileWithOptions(p, parameters);
}
}.resolve(FileContext.this, absF);
}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
index 2376c051c99f9..abb31ed869591 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
@@ -58,11 +58,13 @@
import org.apache.hadoop.fs.Options.Rename;
import org.apache.hadoop.fs.impl.AbstractFSBuilderImpl;
import org.apache.hadoop.fs.impl.FutureDataInputStreamBuilderImpl;
+import org.apache.hadoop.fs.impl.OpenFileParameters;
import org.apache.hadoop.fs.permission.AclEntry;
import org.apache.hadoop.fs.permission.AclStatus;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsCreateModes;
import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.MultipleIOException;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.AccessControlException;
@@ -130,6 +132,25 @@
* New methods may be marked as Unstable or Evolving for their initial release,
* as a warning that they are new and may change based on the
* experience of use in applications.
+ * Important note for developers
+ *
+ * If you're making changes here to the public API or protected methods,
+ * you must review the following subclasses and make sure that
+ * they are filtering/passing through new methods as appropriate.
+ *
+ * {@link FilterFileSystem}: methods are passed through.
+ * {@link ChecksumFileSystem}: checksums are created and
+ * verified.
+ * {@code TestHarFileSystem} will need its {@code MustNotImplement}
+ * interface updated.
+ *
+ * There are some external places your changes will break things.
+ * Do co-ordinate changes here.
+ *
+ * HBase: HBoss
+ * Hive: HiveShim23
+ * {@code shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java}
+ *
*****************************************************************/
@SuppressWarnings("DeprecatedIsStillUsed")
@InterfaceAudience.Public
@@ -257,7 +278,8 @@ public static FileSystem get(Configuration conf) throws IOException {
* @return the uri of the default filesystem
*/
public static URI getDefaultUri(Configuration conf) {
- URI uri = URI.create(fixName(conf.get(FS_DEFAULT_NAME_KEY, DEFAULT_FS)));
+ URI uri =
+ URI.create(fixName(conf.getTrimmed(FS_DEFAULT_NAME_KEY, DEFAULT_FS)));
if (uri.getScheme() == null) {
throw new IllegalArgumentException("No scheme in default FS: " + uri);
}
@@ -2159,24 +2181,19 @@ protected class DirListingIterator implements
private DirectoryEntries entries;
private int i = 0;
- DirListingIterator(Path path) {
+ DirListingIterator(Path path) throws IOException {
this.path = path;
+ this.entries = listStatusBatch(path, null);
}
@Override
public boolean hasNext() throws IOException {
- if (entries == null) {
- fetchMore();
- }
return i < entries.getEntries().length ||
entries.hasMore();
}
private void fetchMore() throws IOException {
- byte[] token = null;
- if (entries != null) {
- token = entries.getToken();
- }
+ byte[] token = entries.getToken();
entries = listStatusBatch(path, token);
i = 0;
}
@@ -3391,9 +3408,22 @@ private static FileSystem createFileSystem(URI uri, Configuration conf)
Tracer tracer = FsTracer.get(conf);
try(TraceScope scope = tracer.newScope("FileSystem#createFileSystem")) {
scope.addKVAnnotation("scheme", uri.getScheme());
- Class clazz = getFileSystemClass(uri.getScheme(), conf);
- FileSystem fs = (FileSystem)ReflectionUtils.newInstance(clazz, conf);
- fs.initialize(uri, conf);
+ Class clazz =
+ getFileSystemClass(uri.getScheme(), conf);
+ FileSystem fs = ReflectionUtils.newInstance(clazz, conf);
+ try {
+ fs.initialize(uri, conf);
+ } catch (IOException | RuntimeException e) {
+ // exception raised during initialization.
+ // log summary at warn and full stack at debug
+ LOGGER.warn("Failed to initialize fileystem {}: {}",
+ uri, e.toString());
+ LOGGER.debug("Failed to initialize fileystem", e);
+ // then (robustly) close the FS, so as to invoke any
+ // cleanup code.
+ IOUtils.cleanupWithLogger(LOGGER, fs);
+ throw e;
+ }
return fs;
}
}
@@ -4449,43 +4479,39 @@ public FutureDataInputStreamBuilder openFile(PathHandle pathHandle)
* the action of opening the file should begin.
*
* The base implementation performs a blocking
- * call to {@link #open(Path, int)}in this call;
+ * call to {@link #open(Path, int)} in this call;
* the actual outcome is in the returned {@code CompletableFuture}.
* This avoids having to create some thread pool, while still
* setting up the expectation that the {@code get()} call
* is needed to evaluate the result.
* @param path path to the file
- * @param mandatoryKeys set of options declared as mandatory.
- * @param options options set during the build sequence.
- * @param bufferSize buffer size
+ * @param parameters open file parameters from the builder.
* @return a future which will evaluate to the opened file.
* @throws IOException failure to resolve the link.
* @throws IllegalArgumentException unknown mandatory key
*/
protected CompletableFuture openFileWithOptions(
final Path path,
- final Set mandatoryKeys,
- final Configuration options,
- final int bufferSize) throws IOException {
- AbstractFSBuilderImpl.rejectUnknownMandatoryKeys(mandatoryKeys,
+ final OpenFileParameters parameters) throws IOException {
+ AbstractFSBuilderImpl.rejectUnknownMandatoryKeys(
+ parameters.getMandatoryKeys(),
Collections.emptySet(),
"for " + path);
return LambdaUtils.eval(
- new CompletableFuture<>(), () -> open(path, bufferSize));
+ new CompletableFuture<>(), () ->
+ open(path, parameters.getBufferSize()));
}
/**
* Execute the actual open file operation.
* The base implementation performs a blocking
- * call to {@link #open(Path, int)}in this call;
+ * call to {@link #open(Path, int)} in this call;
* the actual outcome is in the returned {@code CompletableFuture}.
* This avoids having to create some thread pool, while still
* setting up the expectation that the {@code get()} call
* is needed to evaluate the result.
* @param pathHandle path to the file
- * @param mandatoryKeys set of options declared as mandatory.
- * @param options options set during the build sequence.
- * @param bufferSize buffer size
+ * @param parameters open file parameters from the builder.
* @return a future which will evaluate to the opened file.
* @throws IOException failure to resolve the link.
* @throws IllegalArgumentException unknown mandatory key
@@ -4494,14 +4520,13 @@ protected CompletableFuture openFileWithOptions(
*/
protected CompletableFuture openFileWithOptions(
final PathHandle pathHandle,
- final Set mandatoryKeys,
- final Configuration options,
- final int bufferSize) throws IOException {
- AbstractFSBuilderImpl.rejectUnknownMandatoryKeys(mandatoryKeys,
+ final OpenFileParameters parameters) throws IOException {
+ AbstractFSBuilderImpl.rejectUnknownMandatoryKeys(
+ parameters.getMandatoryKeys(),
Collections.emptySet(), "");
CompletableFuture result = new CompletableFuture<>();
try {
- result.complete(open(pathHandle, bufferSize));
+ result.complete(open(pathHandle, parameters.getBufferSize()));
} catch (UnsupportedOperationException tx) {
// fail fast here
throw tx;
@@ -4603,12 +4628,17 @@ protected FSDataInputStreamBuilder(
@Override
public CompletableFuture build() throws IOException {
Optional optionalPath = getOptionalPath();
+ OpenFileParameters parameters = new OpenFileParameters()
+ .withMandatoryKeys(getMandatoryKeys())
+ .withOptions(getOptions())
+ .withBufferSize(getBufferSize())
+ .withStatus(super.getStatus()); // explicit to avoid IDE warnings
if(optionalPath.isPresent()) {
return getFS().openFileWithOptions(optionalPath.get(),
- getMandatoryKeys(), getOptions(), getBufferSize());
+ parameters);
} else {
return getFS().openFileWithOptions(getPathHandle(),
- getMandatoryKeys(), getOptions(), getBufferSize());
+ parameters);
}
}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
index 4566686a126fe..7bc93f9bf5db8 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
@@ -21,16 +21,20 @@
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.BufferedReader;
+import java.io.BufferedWriter;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
+import java.io.OutputStreamWriter;
import java.net.InetAddress;
import java.net.URI;
import java.net.UnknownHostException;
import java.nio.charset.Charset;
+import java.nio.charset.CharsetEncoder;
+import java.nio.charset.StandardCharsets;
import java.nio.file.AccessDeniedException;
import java.nio.file.FileSystems;
import java.nio.file.Files;
@@ -38,6 +42,7 @@
import java.util.Enumeration;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
@@ -1633,4 +1638,235 @@ public static boolean compareFs(FileSystem srcFs, FileSystem destFs) {
// check for ports
return srcUri.getPort()==dstUri.getPort();
}
+
+ /**
+ * Writes bytes to a file. This utility method opens the file for writing,
+ * creating the file if it does not exist, or overwrites an existing file. All
+ * bytes in the byte array are written to the file.
+ *
+ * @param fs the file system with which to create the file
+ * @param path the path to the file
+ * @param bytes the byte array with the bytes to write
+ *
+ * @return the file system
+ *
+ * @throws NullPointerException if any of the arguments are {@code null}
+ * @throws IOException if an I/O error occurs creating or writing to the file
+ */
+ public static FileSystem write(final FileSystem fs, final Path path,
+ final byte[] bytes) throws IOException {
+
+ Objects.requireNonNull(path);
+ Objects.requireNonNull(bytes);
+
+ try (FSDataOutputStream out = fs.createFile(path).overwrite(true).build()) {
+ out.write(bytes);
+ }
+
+ return fs;
+ }
+
+ /**
+ * Writes bytes to a file. This utility method opens the file for writing,
+ * creating the file if it does not exist, or overwrites an existing file. All
+ * bytes in the byte array are written to the file.
+ *
+ * @param fileContext the file context with which to create the file
+ * @param path the path to the file
+ * @param bytes the byte array with the bytes to write
+ *
+ * @return the file context
+ *
+ * @throws NullPointerException if any of the arguments are {@code null}
+ * @throws IOException if an I/O error occurs creating or writing to the file
+ */
+ public static FileContext write(final FileContext fileContext,
+ final Path path, final byte[] bytes) throws IOException {
+
+ Objects.requireNonNull(path);
+ Objects.requireNonNull(bytes);
+
+ try (FSDataOutputStream out =
+ fileContext.create(path).overwrite(true).build()) {
+ out.write(bytes);
+ }
+
+ return fileContext;
+ }
+
+ /**
+ * Write lines of text to a file. Each line is a char sequence and is written
+ * to the file in sequence with each line terminated by the platform's line
+ * separator, as defined by the system property {@code
+ * line.separator}. Characters are encoded into bytes using the specified
+ * charset. This utility method opens the file for writing, creating the file
+ * if it does not exist, or overwrites an existing file.
+ *
+ * @param fs the file system with which to create the file
+ * @param path the path to the file
+ * @param lines a Collection to iterate over the char sequences
+ * @param cs the charset to use for encoding
+ *
+ * @return the file system
+ *
+ * @throws NullPointerException if any of the arguments are {@code null}
+ * @throws IOException if an I/O error occurs creating or writing to the file
+ */
+ public static FileSystem write(final FileSystem fs, final Path path,
+ final Iterable lines, final Charset cs)
+ throws IOException {
+
+ Objects.requireNonNull(path);
+ Objects.requireNonNull(lines);
+ Objects.requireNonNull(cs);
+
+ CharsetEncoder encoder = cs.newEncoder();
+ try (FSDataOutputStream out = fs.createFile(path).overwrite(true).build();
+ BufferedWriter writer =
+ new BufferedWriter(new OutputStreamWriter(out, encoder))) {
+ for (CharSequence line : lines) {
+ writer.append(line);
+ writer.newLine();
+ }
+ }
+ return fs;
+ }
+
+ /**
+ * Write lines of text to a file. Each line is a char sequence and is written
+ * to the file in sequence with each line terminated by the platform's line
+ * separator, as defined by the system property {@code
+ * line.separator}. Characters are encoded into bytes using the specified
+ * charset. This utility method opens the file for writing, creating the file
+ * if it does not exist, or overwrites an existing file.
+ *
+ * @param fileContext the file context with which to create the file
+ * @param path the path to the file
+ * @param lines a Collection to iterate over the char sequences
+ * @param cs the charset to use for encoding
+ *
+ * @return the file context
+ *
+ * @throws NullPointerException if any of the arguments are {@code null}
+ * @throws IOException if an I/O error occurs creating or writing to the file
+ */
+ public static FileContext write(final FileContext fileContext,
+ final Path path, final Iterable lines,
+ final Charset cs) throws IOException {
+
+ Objects.requireNonNull(path);
+ Objects.requireNonNull(lines);
+ Objects.requireNonNull(cs);
+
+ CharsetEncoder encoder = cs.newEncoder();
+ try (FSDataOutputStream out = fileContext.create(path).overwrite(true).build();
+ BufferedWriter writer =
+ new BufferedWriter(new OutputStreamWriter(out, encoder))) {
+ for (CharSequence line : lines) {
+ writer.append(line);
+ writer.newLine();
+ }
+ }
+ return fileContext;
+ }
+
+ /**
+ * Write a line of text to a file. Characters are encoded into bytes using the
+ * specified charset. This utility method opens the file for writing, creating
+ * the file if it does not exist, or overwrites an existing file.
+ *
+ * @param fs the file system with which to create the file
+ * @param path the path to the file
+ * @param charseq the char sequence to write to the file
+ * @param cs the charset to use for encoding
+ *
+ * @return the file system
+ *
+ * @throws NullPointerException if any of the arguments are {@code null}
+ * @throws IOException if an I/O error occurs creating or writing to the file
+ */
+ public static FileSystem write(final FileSystem fs, final Path path,
+ final CharSequence charseq, final Charset cs) throws IOException {
+
+ Objects.requireNonNull(path);
+ Objects.requireNonNull(charseq);
+ Objects.requireNonNull(cs);
+
+ CharsetEncoder encoder = cs.newEncoder();
+ try (FSDataOutputStream out = fs.createFile(path).overwrite(true).build();
+ BufferedWriter writer =
+ new BufferedWriter(new OutputStreamWriter(out, encoder))) {
+ writer.append(charseq);
+ }
+ return fs;
+ }
+
+ /**
+ * Write a line of text to a file. Characters are encoded into bytes using the
+ * specified charset. This utility method opens the file for writing, creating
+ * the file if it does not exist, or overwrites an existing file.
+ *
+ * @param FileContext the file context with which to create the file
+ * @param path the path to the file
+ * @param charseq the char sequence to write to the file
+ * @param cs the charset to use for encoding
+ *
+ * @return the file context
+ *
+ * @throws NullPointerException if any of the arguments are {@code null}
+ * @throws IOException if an I/O error occurs creating or writing to the file
+ */
+ public static FileContext write(final FileContext fs, final Path path,
+ final CharSequence charseq, final Charset cs) throws IOException {
+
+ Objects.requireNonNull(path);
+ Objects.requireNonNull(charseq);
+ Objects.requireNonNull(cs);
+
+ CharsetEncoder encoder = cs.newEncoder();
+ try (FSDataOutputStream out = fs.create(path).overwrite(true).build();
+ BufferedWriter writer =
+ new BufferedWriter(new OutputStreamWriter(out, encoder))) {
+ writer.append(charseq);
+ }
+ return fs;
+ }
+
+ /**
+ * Write a line of text to a file. Characters are encoded into bytes using
+ * UTF-8. This utility method opens the file for writing, creating the file if
+ * it does not exist, or overwrites an existing file.
+ *
+ * @param fs the files system with which to create the file
+ * @param path the path to the file
+ * @param charseq the char sequence to write to the file
+ *
+ * @return the file system
+ *
+ * @throws NullPointerException if any of the arguments are {@code null}
+ * @throws IOException if an I/O error occurs creating or writing to the file
+ */
+ public static FileSystem write(final FileSystem fs, final Path path,
+ final CharSequence charseq) throws IOException {
+ return write(fs, path, charseq, StandardCharsets.UTF_8);
+ }
+
+ /**
+ * Write a line of text to a file. Characters are encoded into bytes using
+ * UTF-8. This utility method opens the file for writing, creating the file if
+ * it does not exist, or overwrites an existing file.
+ *
+ * @param fileContext the files system with which to create the file
+ * @param path the path to the file
+ * @param charseq the char sequence to write to the file
+ *
+ * @return the file context
+ *
+ * @throws NullPointerException if any of the arguments are {@code null}
+ * @throws IOException if an I/O error occurs creating or writing to the file
+ */
+ public static FileContext write(final FileContext fileContext,
+ final Path path, final CharSequence charseq) throws IOException {
+ return write(fileContext, path, charseq, StandardCharsets.UTF_8);
+ }
}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java
index 3bc3cb2e9b07a..cf12ea3898a7f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java
@@ -25,12 +25,12 @@
import java.util.EnumSet;
import java.util.List;
import java.util.Map;
-import java.util.Set;
import java.util.concurrent.CompletableFuture;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.impl.OpenFileParameters;
import org.apache.hadoop.fs.permission.AclEntry;
import org.apache.hadoop.fs.permission.AclStatus;
import org.apache.hadoop.fs.permission.FsAction;
@@ -714,20 +714,15 @@ public FutureDataInputStreamBuilder openFile(final PathHandle pathHandle)
@Override
protected CompletableFuture openFileWithOptions(
final Path path,
- final Set mandatoryKeys,
- final Configuration options,
- final int bufferSize) throws IOException {
- return fs.openFileWithOptions(path, mandatoryKeys, options, bufferSize);
+ final OpenFileParameters parameters) throws IOException {
+ return fs.openFileWithOptions(path, parameters);
}
@Override
protected CompletableFuture openFileWithOptions(
final PathHandle pathHandle,
- final Set mandatoryKeys,
- final Configuration options,
- final int bufferSize) throws IOException {
- return fs.openFileWithOptions(pathHandle, mandatoryKeys, options,
- bufferSize);
+ final OpenFileParameters parameters) throws IOException {
+ return fs.openFileWithOptions(pathHandle, parameters);
}
@Override
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java
index 731a52a7b4137..e197506edc88b 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFs.java
@@ -26,13 +26,12 @@
import java.util.EnumSet;
import java.util.List;
import java.util.Map;
-import java.util.Set;
import java.util.concurrent.CompletableFuture;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem.Statistics;
+import org.apache.hadoop.fs.impl.OpenFileParameters;
import org.apache.hadoop.fs.permission.AclEntry;
import org.apache.hadoop.fs.permission.AclStatus;
import org.apache.hadoop.fs.permission.FsAction;
@@ -440,10 +439,8 @@ public Collection getAllStoragePolicies()
@Override
public CompletableFuture openFileWithOptions(
final Path path,
- final Set mandatoryKeys,
- final Configuration options,
- final int bufferSize) throws IOException {
- return myFs.openFileWithOptions(path, mandatoryKeys, options, bufferSize);
+ final OpenFileParameters parameters) throws IOException {
+ return myFs.openFileWithOptions(path, parameters);
}
public boolean hasPathCapability(final Path path,
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsConstants.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsConstants.java
index cfef1c3827917..07c16b22358c1 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsConstants.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsConstants.java
@@ -42,4 +42,6 @@ public interface FsConstants {
*/
public static final URI VIEWFS_URI = URI.create("viewfs:///");
public static final String VIEWFS_SCHEME = "viewfs";
+ String FS_VIEWFS_OVERLOAD_SCHEME_TARGET_FS_IMPL_PATTERN =
+ "fs.viewfs.overload.scheme.target.%s.impl";
}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FutureDataInputStreamBuilder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FutureDataInputStreamBuilder.java
index 774d30927df2c..27a522e593001 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FutureDataInputStreamBuilder.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FutureDataInputStreamBuilder.java
@@ -47,4 +47,15 @@ public interface FutureDataInputStreamBuilder
CompletableFuture build()
throws IllegalArgumentException, UnsupportedOperationException,
IOException;
+
+ /**
+ * A FileStatus may be provided to the open request.
+ * It is up to the implementation whether to use this or not.
+ * @param status status.
+ * @return the builder.
+ */
+ default FutureDataInputStreamBuilder withFileStatus(FileStatus status) {
+ return this;
+ }
+
}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalFileSystemPathHandle.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalFileSystemPathHandle.java
index a6b37b32bb564..1f0a06d7dd98f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalFileSystemPathHandle.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalFileSystemPathHandle.java
@@ -16,7 +16,7 @@
*/
package org.apache.hadoop.fs;
-import com.google.protobuf.ByteString;
+import org.apache.hadoop.thirdparty.protobuf.ByteString;
import org.apache.hadoop.fs.FSProtos.LocalFileSystemPathHandleProto;
import java.io.IOException;
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PartialListing.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PartialListing.java
new file mode 100644
index 0000000000000..043f84612dc8b
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PartialListing.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs;
+
+import com.google.common.base.Preconditions;
+import org.apache.commons.lang3.builder.ToStringBuilder;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.ipc.RemoteException;
+
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * A partial listing of the children of a parent directory. Since it is a
+ * partial listing, multiple PartialListing may need to be combined to obtain
+ * the full listing of a parent directory.
+ *
+ * ListingBatch behaves similar to a Future, in that getting the result via
+ * {@link #get()} will throw an Exception if there was a failure.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+public class PartialListing {
+ private final Path listedPath;
+ private final List partialListing;
+ private final RemoteException exception;
+
+ public PartialListing(Path listedPath, List partialListing) {
+ this(listedPath, partialListing, null);
+ }
+
+ public PartialListing(Path listedPath, RemoteException exception) {
+ this(listedPath, null, exception);
+ }
+
+ private PartialListing(Path listedPath, List partialListing,
+ RemoteException exception) {
+ Preconditions.checkArgument(partialListing == null ^ exception == null);
+ this.partialListing = partialListing;
+ this.listedPath = listedPath;
+ this.exception = exception;
+ }
+
+ /**
+ * Partial listing of the path being listed. In the case where the path is
+ * a file. The list will be a singleton with the file itself.
+ *
+ * @return Partial listing of the path being listed.
+ * @throws IOException if there was an exception getting the listing.
+ */
+ public List get() throws IOException {
+ if (exception != null) {
+ throw exception.unwrapRemoteException();
+ }
+ return partialListing;
+ }
+
+ /**
+ * Path being listed.
+ *
+ * @return the path being listed.
+ */
+ public Path getListedPath() {
+ return listedPath;
+ }
+
+ @Override
+ public String toString() {
+ return new ToStringBuilder(this)
+ .append("listedPath", listedPath)
+ .append("partialListing", partialListing)
+ .append("exception", exception)
+ .toString();
+ }
+}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/QuotaUsage.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/QuotaUsage.java
index 3472362dc4792..11cc93401748e 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/QuotaUsage.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/QuotaUsage.java
@@ -40,14 +40,13 @@ public class QuotaUsage {
/** Builder class for QuotaUsage. */
public static class Builder {
public Builder() {
- this.quota = -1;
- this.spaceQuota = -1;
+ this.quota = -1L;
+ this.spaceQuota = -1L;
typeConsumed = new long[StorageType.values().length];
typeQuota = new long[StorageType.values().length];
- for (int i = 0; i < typeQuota.length; i++) {
- typeQuota[i] = -1;
- }
+
+ Arrays.fill(typeQuota, -1L);
}
public Builder fileAndDirectoryCount(long count) {
@@ -71,9 +70,8 @@ public Builder spaceQuota(long spaceQuota) {
}
public Builder typeConsumed(long[] typeConsumed) {
- for (int i = 0; i < typeConsumed.length; i++) {
- this.typeConsumed[i] = typeConsumed[i];
- }
+ System.arraycopy(typeConsumed, 0, this.typeConsumed, 0,
+ typeConsumed.length);
return this;
}
@@ -88,9 +86,7 @@ public Builder typeConsumed(StorageType type, long consumed) {
}
public Builder typeQuota(long[] typeQuota) {
- for (int i = 0; i < typeQuota.length; i++) {
- this.typeQuota[i] = typeQuota[i];
- }
+ System.arraycopy(typeQuota, 0, this.typeQuota, 0, typeQuota.length);
return this;
}
@@ -153,32 +149,21 @@ public long getSpaceQuota() {
/** Return storage type quota. */
public long getTypeQuota(StorageType type) {
- return (typeQuota != null) ? typeQuota[type.ordinal()] : -1;
+ return (typeQuota != null) ? typeQuota[type.ordinal()] : -1L;
}
/** Return storage type consumed. */
public long getTypeConsumed(StorageType type) {
- return (typeConsumed != null) ? typeConsumed[type.ordinal()] : 0;
- }
-
- /** Return storage type quota. */
- private long[] getTypesQuota() {
- return typeQuota;
- }
-
- /** Return storage type quota. */
- private long[] getTypesConsumed() {
- return typeConsumed;
+ return (typeConsumed != null) ? typeConsumed[type.ordinal()] : 0L;
}
/** Return true if any storage type quota has been set. */
public boolean isTypeQuotaSet() {
- if (typeQuota == null) {
- return false;
- }
- for (StorageType t : StorageType.getTypesSupportingQuota()) {
- if (typeQuota[t.ordinal()] > 0) {
- return true;
+ if (typeQuota != null) {
+ for (StorageType t : StorageType.getTypesSupportingQuota()) {
+ if (typeQuota[t.ordinal()] > 0L) {
+ return true;
+ }
}
}
return false;
@@ -186,45 +171,58 @@ public boolean isTypeQuotaSet() {
/** Return true if any storage type consumption information is available. */
public boolean isTypeConsumedAvailable() {
- if (typeConsumed == null) {
- return false;
- }
- for (StorageType t : StorageType.getTypesSupportingQuota()) {
- if (typeConsumed[t.ordinal()] > 0) {
- return true;
+ if (typeConsumed != null) {
+ for (StorageType t : StorageType.getTypesSupportingQuota()) {
+ if (typeConsumed[t.ordinal()] > 0L) {
+ return true;
+ }
}
}
return false;
}
@Override
- public boolean equals(Object to) {
- return (this == to || (to instanceof QuotaUsage &&
- getFileAndDirectoryCount() ==
- ((QuotaUsage) to).getFileAndDirectoryCount() &&
- getQuota() == ((QuotaUsage) to).getQuota() &&
- getSpaceConsumed() == ((QuotaUsage) to).getSpaceConsumed() &&
- getSpaceQuota() == ((QuotaUsage) to).getSpaceQuota() &&
- Arrays.equals(getTypesQuota(), ((QuotaUsage) to).getTypesQuota()) &&
- Arrays.equals(getTypesConsumed(),
- ((QuotaUsage) to).getTypesConsumed())));
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result
+ + (int) (fileAndDirectoryCount ^ (fileAndDirectoryCount >>> 32));
+ result = prime * result + (int) (quota ^ (quota >>> 32));
+ result = prime * result + (int) (spaceConsumed ^ (spaceConsumed >>> 32));
+ result = prime * result + (int) (spaceQuota ^ (spaceQuota >>> 32));
+ result = prime * result + Arrays.hashCode(typeConsumed);
+ result = prime * result + Arrays.hashCode(typeQuota);
+ return result;
}
@Override
- public int hashCode() {
- long result = (getFileAndDirectoryCount() ^ getQuota() ^
- getSpaceConsumed() ^ getSpaceQuota());
- if (getTypesQuota() != null) {
- for (long quota : getTypesQuota()) {
- result ^= quota;
- }
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
}
- if (getTypesConsumed() != null) {
- for (long consumed : getTypesConsumed()) {
- result ^= consumed;
- }
+ if (!(obj instanceof QuotaUsage)) {
+ return false;
+ }
+ QuotaUsage other = (QuotaUsage) obj;
+ if (fileAndDirectoryCount != other.fileAndDirectoryCount) {
+ return false;
+ }
+ if (quota != other.quota) {
+ return false;
+ }
+ if (spaceConsumed != other.spaceConsumed) {
+ return false;
+ }
+ if (spaceQuota != other.spaceQuota) {
+ return false;
+ }
+ if (!Arrays.equals(typeConsumed, other.typeConsumed)) {
+ return false;
+ }
+ if (!Arrays.equals(typeQuota, other.typeQuota)) {
+ return false;
}
- return (int)result;
+ return true;
}
/**
@@ -292,11 +290,11 @@ protected String getQuotaUsage(boolean hOption) {
String spaceQuotaStr = QUOTA_NONE;
String spaceQuotaRem = QUOTA_INF;
- if (quota > 0) {
+ if (quota > 0L) {
quotaStr = formatSize(quota, hOption);
quotaRem = formatSize(quota-fileAndDirectoryCount, hOption);
}
- if (spaceQuota >= 0) {
+ if (spaceQuota >= 0L) {
spaceQuotaStr = formatSize(spaceQuota, hOption);
spaceQuotaRem = formatSize(spaceQuota - spaceConsumed, hOption);
}
@@ -307,20 +305,20 @@ protected String getQuotaUsage(boolean hOption) {
protected String getTypesQuotaUsage(boolean hOption,
List types) {
- StringBuffer content = new StringBuffer();
+ StringBuilder content = new StringBuilder();
for (StorageType st : types) {
long typeQuota = getTypeQuota(st);
long typeConsumed = getTypeConsumed(st);
String quotaStr = QUOTA_NONE;
String quotaRem = QUOTA_INF;
- if (typeQuota >= 0) {
+ if (typeQuota >= 0L) {
quotaStr = formatSize(typeQuota, hOption);
quotaRem = formatSize(typeQuota - typeConsumed, hOption);
}
- content.append(String.format(STORAGE_TYPE_SUMMARY_FORMAT,
- quotaStr, quotaRem));
+ content.append(
+ String.format(STORAGE_TYPE_SUMMARY_FORMAT, quotaStr, quotaRem));
}
return content.toString();
}
@@ -332,7 +330,7 @@ protected String getTypesQuotaUsage(boolean hOption,
* @return storage header string
*/
public static String getStorageTypeHeader(List storageTypes) {
- StringBuffer header = new StringBuffer();
+ StringBuilder header = new StringBuilder();
for (StorageType st : storageTypes) {
/* the field length is 13/17 for quota and remain quota
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java
index 4b144bfddf6c6..28db2c9a1a227 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ftp/FTPFileSystem.java
@@ -20,6 +20,7 @@
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
+import java.io.OutputStream;
import java.net.ConnectException;
import java.net.URI;
@@ -41,6 +42,7 @@
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.util.Progressable;
import org.slf4j.Logger;
@@ -110,7 +112,9 @@ public void initialize(URI uri, Configuration conf) throws IOException { // get
// get port information from uri, (overrides info in conf)
int port = uri.getPort();
- port = (port == -1) ? FTP.DEFAULT_PORT : port;
+ if(port == -1){
+ port = conf.getInt(FS_FTP_HOST_PORT, FTP.DEFAULT_PORT);
+ }
conf.setInt(FS_FTP_HOST_PORT, port);
// get user/password information from URI (overrides info in conf)
@@ -340,8 +344,19 @@ public FSDataOutputStream create(Path file, FsPermission permission,
// file. The FTP client connection is closed when close() is called on the
// FSDataOutputStream.
client.changeWorkingDirectory(parent.toUri().getPath());
- FSDataOutputStream fos = new FSDataOutputStream(client.storeFileStream(file
- .getName()), statistics) {
+ OutputStream outputStream = client.storeFileStream(file.getName());
+
+ if (!FTPReply.isPositivePreliminary(client.getReplyCode())) {
+ // The ftpClient is an inconsistent state. Must close the stream
+ // which in turn will logout and disconnect from FTP server
+ if (outputStream != null) {
+ IOUtils.closeStream(outputStream);
+ }
+ disconnect(client);
+ throw new IOException("Unable to create file: " + file + ", Aborting");
+ }
+
+ FSDataOutputStream fos = new FSDataOutputStream(outputStream, statistics) {
@Override
public void close() throws IOException {
super.close();
@@ -356,12 +371,6 @@ public void close() throws IOException {
}
}
};
- if (!FTPReply.isPositivePreliminary(client.getReplyCode())) {
- // The ftpClient is an inconsistent state. Must close the stream
- // which in turn will logout and disconnect from FTP server
- fos.close();
- throw new IOException("Unable to create file: " + file + ", Aborting");
- }
return fos;
}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/FutureDataInputStreamBuilderImpl.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/FutureDataInputStreamBuilderImpl.java
index 2aa4a5d95fcc7..24a8d49747fe6 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/FutureDataInputStreamBuilderImpl.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/FutureDataInputStreamBuilderImpl.java
@@ -26,12 +26,13 @@
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileContext;
+import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FutureDataInputStreamBuilder;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathHandle;
-import static com.google.common.base.Preconditions.checkNotNull;
+import static java.util.Objects.requireNonNull;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_DEFAULT;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_KEY;
@@ -60,6 +61,12 @@ public abstract class FutureDataInputStreamBuilderImpl
private int bufferSize;
+ /**
+ * File status passed in through a {@link #withFileStatus(FileStatus)}
+ * call; null otherwise.
+ */
+ private FileStatus status;
+
/**
* Construct from a {@link FileContext}.
*
@@ -69,8 +76,8 @@ public abstract class FutureDataInputStreamBuilderImpl
*/
protected FutureDataInputStreamBuilderImpl(@Nonnull FileContext fc,
@Nonnull Path path) throws IOException {
- super(checkNotNull(path));
- checkNotNull(fc);
+ super(requireNonNull(path, "path"));
+ requireNonNull(fc, "file context");
this.fileSystem = null;
bufferSize = IO_FILE_BUFFER_SIZE_DEFAULT;
}
@@ -82,8 +89,8 @@ protected FutureDataInputStreamBuilderImpl(@Nonnull FileContext fc,
*/
protected FutureDataInputStreamBuilderImpl(@Nonnull FileSystem fileSystem,
@Nonnull Path path) {
- super(checkNotNull(path));
- this.fileSystem = checkNotNull(fileSystem);
+ super(requireNonNull(path, "path"));
+ this.fileSystem = requireNonNull(fileSystem, "fileSystem");
initFromFS();
}
@@ -108,7 +115,7 @@ private void initFromFS() {
}
protected FileSystem getFS() {
- checkNotNull(fileSystem);
+ requireNonNull(fileSystem, "fileSystem");
return fileSystem;
}
@@ -138,4 +145,18 @@ public FutureDataInputStreamBuilder builder() {
public FutureDataInputStreamBuilder getThisBuilder() {
return this;
}
+
+ @Override
+ public FutureDataInputStreamBuilder withFileStatus(FileStatus st) {
+ this.status = requireNonNull(st, "status");
+ return this;
+ }
+
+ /**
+ * Get any status set in {@link #withFileStatus(FileStatus)}.
+ * @return a status value or null.
+ */
+ protected FileStatus getStatus() {
+ return status;
+ }
}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/OpenFileParameters.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/OpenFileParameters.java
new file mode 100644
index 0000000000000..77b4ff52696a3
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/OpenFileParameters.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.impl;
+
+import java.util.Set;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+
+import static java.util.Objects.requireNonNull;
+
+/**
+ * All the parameters from the openFile builder for the
+ * {@code openFileWithOptions} commands.
+ *
+ * If/when new attributes added to the builder, this class will be extended.
+ */
+public class OpenFileParameters {
+
+ /**
+ * Set of options declared as mandatory.
+ */
+ private Set mandatoryKeys;
+
+ /**
+ * Options set during the build sequence.
+ */
+ private Configuration options;
+
+ /**
+ * Buffer size.
+ */
+ private int bufferSize;
+
+ /**
+ * Optional file status.
+ */
+ private FileStatus status;
+
+ public OpenFileParameters() {
+ }
+
+ public OpenFileParameters withMandatoryKeys(final Set keys) {
+ this.mandatoryKeys = requireNonNull(keys);
+ return this;
+ }
+
+ public OpenFileParameters withOptions(final Configuration opts) {
+ this.options = requireNonNull(opts);
+ return this;
+ }
+
+ public OpenFileParameters withBufferSize(final int size) {
+ this.bufferSize = size;
+ return this;
+ }
+
+ public OpenFileParameters withStatus(final FileStatus st) {
+ this.status = st;
+ return this;
+ }
+
+ public Set getMandatoryKeys() {
+ return mandatoryKeys;
+ }
+
+ public Configuration getOptions() {
+ return options;
+ }
+
+ public int getBufferSize() {
+ return bufferSize;
+ }
+
+ public FileStatus getStatus() {
+ return status;
+ }
+}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/sftp/SFTPFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/sftp/SFTPFileSystem.java
index ed33357b51d2b..a91b50f2e9fa7 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/sftp/SFTPFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/sftp/SFTPFileSystem.java
@@ -19,7 +19,6 @@
import java.io.FileNotFoundException;
import java.io.IOException;
-import java.io.InputStream;
import java.io.OutputStream;
import java.net.URI;
import java.net.URLDecoder;
@@ -516,20 +515,21 @@ public FSDataInputStream open(Path f, int bufferSize) throws IOException {
disconnect(channel);
throw new IOException(String.format(E_PATH_DIR, f));
}
- InputStream is;
try {
// the path could be a symbolic link, so get the real path
absolute = new Path("/", channel.realpath(absolute.toUri().getPath()));
-
- is = channel.get(absolute.toUri().getPath());
} catch (SftpException e) {
throw new IOException(e);
}
- return new FSDataInputStream(new SFTPInputStream(is, statistics)){
+ return new FSDataInputStream(
+ new SFTPInputStream(channel, absolute, statistics)){
@Override
public void close() throws IOException {
- super.close();
- disconnect(channel);
+ try {
+ super.close();
+ } finally {
+ disconnect(channel);
+ }
}
};
}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/sftp/SFTPInputStream.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/sftp/SFTPInputStream.java
index 7af299bd113e1..d0f9a8d0887ca 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/sftp/SFTPInputStream.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/sftp/SFTPInputStream.java
@@ -15,62 +15,107 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.hadoop.fs.sftp;
+import java.io.EOFException;
import java.io.IOException;
import java.io.InputStream;
+import com.jcraft.jsch.ChannelSftp;
+import com.jcraft.jsch.SftpATTRS;
+import com.jcraft.jsch.SftpException;
+
+import org.apache.hadoop.fs.FSExceptionMessages;
import org.apache.hadoop.fs.FSInputStream;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
/** SFTP FileSystem input stream. */
class SFTPInputStream extends FSInputStream {
- public static final String E_SEEK_NOTSUPPORTED = "Seek not supported";
- public static final String E_NULL_INPUTSTREAM = "Null InputStream";
- public static final String E_STREAM_CLOSED = "Stream closed";
-
+ private final ChannelSftp channel;
+ private final Path path;
private InputStream wrappedStream;
private FileSystem.Statistics stats;
private boolean closed;
private long pos;
+ private long nextPos;
+ private long contentLength;
- SFTPInputStream(InputStream stream, FileSystem.Statistics stats) {
-
- if (stream == null) {
- throw new IllegalArgumentException(E_NULL_INPUTSTREAM);
+ SFTPInputStream(ChannelSftp channel, Path path, FileSystem.Statistics stats)
+ throws IOException {
+ try {
+ this.channel = channel;
+ this.path = path;
+ this.stats = stats;
+ this.wrappedStream = channel.get(path.toUri().getPath());
+ SftpATTRS stat = channel.lstat(path.toString());
+ this.contentLength = stat.getSize();
+ } catch (SftpException e) {
+ throw new IOException(e);
}
- this.wrappedStream = stream;
- this.stats = stats;
+ }
- this.pos = 0;
- this.closed = false;
+ @Override
+ public synchronized void seek(long position) throws IOException {
+ checkNotClosed();
+ if (position < 0) {
+ throw new EOFException(FSExceptionMessages.NEGATIVE_SEEK);
+ }
+ nextPos = position;
}
@Override
- public void seek(long position) throws IOException {
- throw new IOException(E_SEEK_NOTSUPPORTED);
+ public synchronized int available() throws IOException {
+ checkNotClosed();
+ long remaining = contentLength - nextPos;
+ if (remaining > Integer.MAX_VALUE) {
+ return Integer.MAX_VALUE;
+ }
+ return (int) remaining;
+ }
+
+ private void seekInternal() throws IOException {
+ if (pos == nextPos) {
+ return;
+ }
+ if (nextPos > pos) {
+ long skipped = wrappedStream.skip(nextPos - pos);
+ pos = pos + skipped;
+ }
+ if (nextPos < pos) {
+ wrappedStream.close();
+ try {
+ wrappedStream = channel.get(path.toUri().getPath());
+ pos = wrappedStream.skip(nextPos);
+ } catch (SftpException e) {
+ throw new IOException(e);
+ }
+ }
}
@Override
public boolean seekToNewSource(long targetPos) throws IOException {
- throw new IOException(E_SEEK_NOTSUPPORTED);
+ return false;
}
@Override
- public long getPos() throws IOException {
- return pos;
+ public synchronized long getPos() throws IOException {
+ return nextPos;
}
@Override
public synchronized int read() throws IOException {
- if (closed) {
- throw new IOException(E_STREAM_CLOSED);
+ checkNotClosed();
+ if (this.contentLength == 0 || (nextPos >= contentLength)) {
+ return -1;
}
-
+ seekInternal();
int byteRead = wrappedStream.read();
if (byteRead >= 0) {
pos++;
+ nextPos++;
}
if (stats != null & byteRead >= 0) {
stats.incrementBytesRead(1);
@@ -78,23 +123,6 @@ public synchronized int read() throws IOException {
return byteRead;
}
- public synchronized int read(byte[] buf, int off, int len)
- throws IOException {
- if (closed) {
- throw new IOException(E_STREAM_CLOSED);
- }
-
- int result = wrappedStream.read(buf, off, len);
- if (result > 0) {
- pos += result;
- }
- if (stats != null & result > 0) {
- stats.incrementBytesRead(result);
- }
-
- return result;
- }
-
public synchronized void close() throws IOException {
if (closed) {
return;
@@ -103,4 +131,12 @@ public synchronized void close() throws IOException {
wrappedStream.close();
closed = true;
}
+
+ private void checkNotClosed() throws IOException {
+ if (closed) {
+ throw new IOException(
+ path.toUri() + ": " + FSExceptionMessages.STREAM_IS_CLOSED
+ );
+ }
+ }
}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java
index 0802a00b01bc8..ca9961aeb65a0 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java
@@ -415,7 +415,6 @@ protected void copyStreamToTarget(InputStream in, PathData target)
targetFs.setWriteChecksum(writeChecksum);
targetFs.writeStreamToFile(in, tempTarget, lazyPersist, direct);
if (!direct) {
- targetFs.deleteOnExit(tempTarget.path);
targetFs.rename(tempTarget, target);
}
} finally {
@@ -491,25 +490,18 @@ void writeStreamToFile(InputStream in, PathData target,
throws IOException {
FSDataOutputStream out = null;
try {
- out = create(target, lazyPersist, direct);
+ out = create(target, lazyPersist);
IOUtils.copyBytes(in, out, getConf(), true);
- } catch (IOException e) {
- // failure: clean up if we got as far as creating the file
- if (!direct && out != null) {
- try {
- fs.delete(target.path, false);
- } catch (IOException ignored) {
- }
- }
- throw e;
} finally {
+ if (!direct) {
+ deleteOnExit(target.path);
+ }
IOUtils.closeStream(out); // just in case copyBytes didn't
}
}
// tag created files as temp files
- FSDataOutputStream create(PathData item, boolean lazyPersist,
- boolean direct)
+ FSDataOutputStream create(PathData item, boolean lazyPersist)
throws IOException {
if (lazyPersist) {
long defaultBlockSize;
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java
index 4622c75fbd410..39958a9cb1c9d 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CopyCommands.java
@@ -239,26 +239,35 @@ protected void processOptions(LinkedList args)
* Copy local files to a remote filesystem
*/
public static class Put extends CommandWithDestination {
+ private ThreadPoolExecutor executor = null;
+ private int numThreads = 1;
+
+ private static final int MAX_THREADS =
+ Runtime.getRuntime().availableProcessors() * 2;
+
public static final String NAME = "put";
public static final String USAGE =
- "[-f] [-p] [-l] [-d] ... ";
+ "[-f] [-p] [-l] [-d] [-t ] ... ";
public static final String DESCRIPTION =
- "Copy files from the local file system " +
- "into fs. Copying fails if the file already " +
- "exists, unless the -f flag is given.\n" +
- "Flags:\n" +
- " -p : Preserves access and modification times, ownership and the mode.\n" +
- " -f : Overwrites the destination if it already exists.\n" +
- " -l : Allow DataNode to lazily persist the file to disk. Forces\n" +
- " replication factor of 1. This flag will result in reduced\n" +
- " durability. Use with care.\n" +
+ "Copy files from the local file system " +
+ "into fs. Copying fails if the file already " +
+ "exists, unless the -f flag is given.\n" +
+ "Flags:\n" +
+ " -p : Preserves timestamps, ownership and the mode.\n" +
+ " -f : Overwrites the destination if it already exists.\n" +
+ " -t : Number of threads to be used, default is 1.\n" +
+ " -l : Allow DataNode to lazily persist the file to disk. Forces" +
+ " replication factor of 1. This flag will result in reduced" +
+ " durability. Use with care.\n" +
" -d : Skip creation of temporary file(._COPYING_).\n";
@Override
protected void processOptions(LinkedList args) throws IOException {
CommandFormat cf =
new CommandFormat(1, Integer.MAX_VALUE, "f", "p", "l", "d");
+ cf.addOptionWithValue("t");
cf.parse(args);
+ setNumberThreads(cf.getOptValue("t"));
setOverwrite(cf.getOpt("f"));
setPreserve(cf.getOpt("p"));
setLazyPersist(cf.getOpt("l"));
@@ -288,32 +297,22 @@ protected void processArguments(LinkedList args)
copyStreamToTarget(System.in, getTargetPath(args.get(0)));
return;
}
- super.processArguments(args);
- }
- }
- public static class CopyFromLocal extends Put {
- private ThreadPoolExecutor executor = null;
- private int numThreads = 1;
+ executor = new ThreadPoolExecutor(numThreads, numThreads, 1,
+ TimeUnit.SECONDS, new ArrayBlockingQueue<>(1024),
+ new ThreadPoolExecutor.CallerRunsPolicy());
+ super.processArguments(args);
- private static final int MAX_THREADS =
- Runtime.getRuntime().availableProcessors() * 2;
- public static final String NAME = "copyFromLocal";
- public static final String USAGE =
- "[-f] [-p] [-l] [-d] [-t ] ... ";
- public static final String DESCRIPTION =
- "Copy files from the local file system " +
- "into fs. Copying fails if the file already " +
- "exists, unless the -f flag is given.\n" +
- "Flags:\n" +
- " -p : Preserves access and modification times, ownership and the" +
- " mode.\n" +
- " -f : Overwrites the destination if it already exists.\n" +
- " -t : Number of threads to be used, default is 1.\n" +
- " -l : Allow DataNode to lazily persist the file to disk. Forces" +
- " replication factor of 1. This flag will result in reduced" +
- " durability. Use with care.\n" +
- " -d : Skip creation of temporary file(._COPYING_).\n";
+ // issue the command and then wait for it to finish
+ executor.shutdown();
+ try {
+ executor.awaitTermination(Long.MAX_VALUE, TimeUnit.MINUTES);
+ } catch (InterruptedException e) {
+ executor.shutdownNow();
+ displayError(e);
+ Thread.currentThread().interrupt();
+ }
+ }
private void setNumberThreads(String numberThreadsString) {
if (numberThreadsString == null) {
@@ -330,22 +329,6 @@ private void setNumberThreads(String numberThreadsString) {
}
}
- @Override
- protected void processOptions(LinkedList args) throws IOException {
- CommandFormat cf =
- new CommandFormat(1, Integer.MAX_VALUE, "f", "p", "l", "d");
- cf.addOptionWithValue("t");
- cf.parse(args);
- setNumberThreads(cf.getOptValue("t"));
- setOverwrite(cf.getOpt("f"));
- setPreserve(cf.getOpt("p"));
- setLazyPersist(cf.getOpt("l"));
- setDirectWrite(cf.getOpt("d"));
- getRemoteDestination(args);
- // should have a -r option
- setRecursive(true);
- }
-
private void copyFile(PathData src, PathData target) throws IOException {
if (isPathRecursable(src)) {
throw new PathIsDirectoryException(src.toString());
@@ -372,25 +355,6 @@ protected void copyFileToTarget(PathData src, PathData target)
executor.submit(task);
}
- @Override
- protected void processArguments(LinkedList args)
- throws IOException {
- executor = new ThreadPoolExecutor(numThreads, numThreads, 1,
- TimeUnit.SECONDS, new ArrayBlockingQueue<>(1024),
- new ThreadPoolExecutor.CallerRunsPolicy());
- super.processArguments(args);
-
- // issue the command and then wait for it to finish
- executor.shutdown();
- try {
- executor.awaitTermination(Long.MAX_VALUE, TimeUnit.MINUTES);
- } catch (InterruptedException e) {
- executor.shutdownNow();
- displayError(e);
- Thread.currentThread().interrupt();
- }
- }
-
@VisibleForTesting
public int getNumThreads() {
return numThreads;
@@ -401,6 +365,12 @@ public ThreadPoolExecutor getExecutor() {
return executor;
}
}
+
+ public static class CopyFromLocal extends Put {
+ public static final String NAME = "copyFromLocal";
+ public static final String USAGE = Put.USAGE;
+ public static final String DESCRIPTION = "Identical to the -put command.";
+ }
public static class CopyToLocal extends Get {
public static final String NAME = "copyToLocal";
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java
index 22d8be53e97a6..ab7e1951bcd3f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java
@@ -56,13 +56,14 @@ public static void registerCommands(CommandFactory factory) {
//return the quota, namespace count and disk space usage.
private static final String OPTION_QUOTA_AND_USAGE = "u";
private static final String OPTION_ECPOLICY = "e";
+ private static final String OPTION_SNAPSHOT_COUNT = "s";
public static final String NAME = "count";
public static final String USAGE =
"[-" + OPTION_QUOTA + "] [-" + OPTION_HUMAN + "] [-" + OPTION_HEADER
+ "] [-" + OPTION_TYPE + " []] [-" +
OPTION_QUOTA_AND_USAGE + "] [-" + OPTION_EXCLUDE_SNAPSHOT
- + "] [-" + OPTION_ECPOLICY
+ + "] [-" + OPTION_ECPOLICY + "] [-" + OPTION_SNAPSHOT_COUNT
+ "] ...";
public static final String DESCRIPTION =
"Count the number of directories, files and bytes under the paths\n" +
@@ -93,7 +94,8 @@ public static void registerCommands(CommandFactory factory) {
"the storage types.\n" +
"The -" + OPTION_QUOTA_AND_USAGE + " option shows the quota and \n" +
"the usage against the quota without the detailed content summary."+
- "The -"+ OPTION_ECPOLICY +" option shows the erasure coding policy.";
+ "The -" + OPTION_ECPOLICY + " option shows the erasure coding policy."
+ + "The -" + OPTION_SNAPSHOT_COUNT + " option shows snapshot counts.";
private boolean showQuotas;
private boolean humanReadable;
@@ -102,6 +104,7 @@ public static void registerCommands(CommandFactory factory) {
private boolean showQuotasAndUsageOnly;
private boolean excludeSnapshots;
private boolean displayECPolicy;
+ private boolean showSnapshot;
/** Constructor */
public Count() {}
@@ -123,7 +126,7 @@ protected void processOptions(LinkedList args) {
CommandFormat cf = new CommandFormat(1, Integer.MAX_VALUE,
OPTION_QUOTA, OPTION_HUMAN, OPTION_HEADER, OPTION_QUOTA_AND_USAGE,
OPTION_EXCLUDE_SNAPSHOT,
- OPTION_ECPOLICY);
+ OPTION_ECPOLICY, OPTION_SNAPSHOT_COUNT);
cf.addOptionWithValue(OPTION_TYPE);
cf.parse(args);
if (args.isEmpty()) { // default path is the current working directory
@@ -134,6 +137,7 @@ protected void processOptions(LinkedList args) {
showQuotasAndUsageOnly = cf.getOpt(OPTION_QUOTA_AND_USAGE);
excludeSnapshots = cf.getOpt(OPTION_EXCLUDE_SNAPSHOT);
displayECPolicy = cf.getOpt(OPTION_ECPOLICY);
+ showSnapshot = cf.getOpt(OPTION_SNAPSHOT_COUNT);
if (showQuotas || showQuotasAndUsageOnly) {
String types = cf.getOptValue(OPTION_TYPE);
@@ -165,6 +169,9 @@ protected void processOptions(LinkedList args) {
if(displayECPolicy){
headString.append("ERASURECODING_POLICY ");
}
+ if (showSnapshot) {
+ headString.append(ContentSummary.getSnapshotHeader());
+ }
headString.append("PATHNAME");
out.println(headString.toString());
}
@@ -205,6 +212,10 @@ protected void processPath(PathData src) throws IOException {
outputString.append(summary.getErasureCodingPolicy())
.append(" ");
}
+ if (showSnapshot) {
+ ContentSummary summary = src.fs.getContentSummary(src.path);
+ outputString.append(summary.toSnapshot(isHumanReadable()));
+ }
outputString.append(src);
out.println(outputString.toString());
}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsCommand.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsCommand.java
index 784bbf33f7826..ea8378dc4551b 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsCommand.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/FsCommand.java
@@ -108,7 +108,7 @@ protected void processRawArguments(LinkedList args)
HADOOP_SHELL_MISSING_DEFAULT_FS_WARNING_KEY,
HADOOP_SHELL_MISSING_DEFAULT_FS_WARNING_DEFAULT);
if (displayWarnings) {
- final String defaultFs = getConf().get(FS_DEFAULT_NAME_KEY);
+ final String defaultFs = getConf().getTrimmed(FS_DEFAULT_NAME_KEY);
final boolean missingDefaultFs =
defaultFs == null || defaultFs.equals(FS_DEFAULT_NAME_DEFAULT);
if (missingDefaultFs) {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/MoveCommands.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/MoveCommands.java
index 5ef42775ea58b..c20293e1a5adb 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/MoveCommands.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/MoveCommands.java
@@ -25,7 +25,7 @@
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.PathIOException;
import org.apache.hadoop.fs.PathExistsException;
-import org.apache.hadoop.fs.shell.CopyCommands.Put;
+import org.apache.hadoop.fs.shell.CopyCommands.CopyFromLocal;
/** Various commands for moving files */
@InterfaceAudience.Private
@@ -41,12 +41,22 @@ public static void registerCommands(CommandFactory factory) {
/**
* Move local files to a remote filesystem
*/
- public static class MoveFromLocal extends Put {
+ public static class MoveFromLocal extends CopyFromLocal {
public static final String NAME = "moveFromLocal";
- public static final String USAGE = " ... ";
+ public static final String USAGE =
+ "[-f] [-p] [-l] [-d] ... ";
public static final String DESCRIPTION =
- "Same as -put, except that the source is " +
- "deleted after it's copied.";
+ "Same as -put, except that the source is " +
+ "deleted after it's copied\n" +
+ "and -t option has not yet implemented.";
+
+ @Override
+ protected void processOptions(LinkedList args) throws IOException {
+ if(args.contains("-t")) {
+ throw new CommandFormat.UnknownOptionException("-t");
+ }
+ super.processOptions(args);
+ }
@Override
protected void processPath(PathData src, PathData target) throws IOException {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ConfigUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ConfigUtil.java
index 4c3dae9a9f99b..6dd1f6589478e 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ConfigUtil.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ConfigUtil.java
@@ -135,6 +135,17 @@ public static void addLinkMerge(Configuration conf, final URI[] targets) {
addLinkMerge(conf, Constants.CONFIG_VIEWFS_DEFAULT_MOUNT_TABLE, targets);
}
+ /**
+ * Add nfly link to configuration for the given mount table.
+ */
+ public static void addLinkNfly(Configuration conf, String mountTableName,
+ String src, String settings, final String targets) {
+ conf.set(
+ getConfigViewFsPrefix(mountTableName) + "."
+ + Constants.CONFIG_VIEWFS_LINK_NFLY + "." + settings + "." + src,
+ targets);
+ }
+
/**
*
* @param conf
@@ -149,9 +160,7 @@ public static void addLinkNfly(Configuration conf, String mountTableName,
settings = settings == null
? "minReplication=2,repairOnRead=true"
: settings;
-
- conf.set(getConfigViewFsPrefix(mountTableName) + "." +
- Constants.CONFIG_VIEWFS_LINK_NFLY + "." + settings + "." + src,
+ addLinkNfly(conf, mountTableName, src, settings,
StringUtils.uriToString(targets));
}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/Constants.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/Constants.java
index 37f1a16800e7d..0a5d4b46ce2d8 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/Constants.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/Constants.java
@@ -30,6 +30,11 @@ public interface Constants {
* Prefix for the config variable prefix for the ViewFs mount-table
*/
public static final String CONFIG_VIEWFS_PREFIX = "fs.viewfs.mounttable";
+
+ /**
+ * Prefix for the config variable for the ViewFs mount-table path.
+ */
+ String CONFIG_VIEWFS_MOUNTTABLE_PATH = CONFIG_VIEWFS_PREFIX + ".path";
/**
* Prefix for the home dir for the mount table - if not specified
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/FsGetter.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/FsGetter.java
new file mode 100644
index 0000000000000..071af11e63bf2
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/FsGetter.java
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.viewfs;
+
+import java.io.IOException;
+import java.net.URI;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+
+/**
+ * File system instance getter.
+ */
+@Private
+class FsGetter {
+
+ /**
+ * Gets new file system instance of given uri.
+ */
+ public FileSystem getNewInstance(URI uri, Configuration conf)
+ throws IOException {
+ return FileSystem.newInstance(uri, conf);
+ }
+
+ /**
+ * Gets file system instance of given uri.
+ */
+ public FileSystem get(URI uri, Configuration conf) throws IOException {
+ return FileSystem.get(uri, conf);
+ }
+}
\ No newline at end of file
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/HCFSMountTableConfigLoader.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/HCFSMountTableConfigLoader.java
new file mode 100644
index 0000000000000..3968e3650cf39
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/HCFSMountTableConfigLoader.java
@@ -0,0 +1,121 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.viewfs;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocatedFileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RemoteIterator;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * An implementation for Apache Hadoop compatible file system based mount-table
+ * file loading.
+ */
+public class HCFSMountTableConfigLoader implements MountTableConfigLoader {
+ private static final String REGEX_DOT = "[.]";
+ private static final Logger LOGGER =
+ LoggerFactory.getLogger(HCFSMountTableConfigLoader.class);
+ private Path mountTable = null;
+
+ /**
+ * Loads the mount-table configuration from hadoop compatible file system and
+ * add the configuration items to given configuration. Mount-table
+ * configuration format should be suffixed with version number.
+ * Format: mount-table..xml
+ * Example: mount-table.1.xml
+ * When user wants to update mount-table, the expectation is to upload new
+ * mount-table configuration file with monotonically increasing integer as
+ * version number. This API loads the highest version number file. We can
+ * also configure single file path directly.
+ *
+ * @param mountTableConfigPath : A directory path where mount-table files
+ * stored or a mount-table file path. We recommend to configure
+ * directory with the mount-table version files.
+ * @param conf : to add the mount table as resource.
+ */
+ @Override
+ public void load(String mountTableConfigPath, Configuration conf)
+ throws IOException {
+ this.mountTable = new Path(mountTableConfigPath);
+ String scheme = mountTable.toUri().getScheme();
+ FsGetter fsGetter = new ViewFileSystemOverloadScheme.ChildFsGetter(scheme);
+ try (FileSystem fs = fsGetter.getNewInstance(mountTable.toUri(), conf)) {
+ RemoteIterator listFiles =
+ fs.listFiles(mountTable, false);
+ LocatedFileStatus lfs = null;
+ int higherVersion = -1;
+ while (listFiles.hasNext()) {
+ LocatedFileStatus curLfs = listFiles.next();
+ String cur = curLfs.getPath().getName();
+ String[] nameParts = cur.split(REGEX_DOT);
+ if (nameParts.length < 2) {
+ logInvalidFileNameFormat(cur);
+ continue; // invalid file name
+ }
+ int curVersion = higherVersion;
+ try {
+ curVersion = Integer.parseInt(nameParts[nameParts.length - 2]);
+ } catch (NumberFormatException nfe) {
+ logInvalidFileNameFormat(cur);
+ continue;
+ }
+
+ if (curVersion > higherVersion) {
+ higherVersion = curVersion;
+ lfs = curLfs;
+ }
+ }
+
+ if (lfs == null) {
+ // No valid mount table file found.
+ // TODO: Should we fail? Currently viewfs init will fail if no mount
+ // links anyway.
+ LOGGER.warn("No valid mount-table file exist at: {}. At least one "
+ + "mount-table file should present with the name format: "
+ + "mount-table..xml", mountTableConfigPath);
+ return;
+ }
+ // Latest version file.
+ Path latestVersionMountTable = lfs.getPath();
+ if (LOGGER.isDebugEnabled()) {
+ LOGGER.debug("Loading the mount-table {} into configuration.",
+ latestVersionMountTable);
+ }
+ try (FSDataInputStream open = fs.open(latestVersionMountTable)) {
+ Configuration newConf = new Configuration(false);
+ newConf.addResource(open);
+ // This will add configuration props as resource, instead of stream
+ // itself. So, that stream can be closed now.
+ conf.addResource(newConf);
+ }
+ }
+ }
+
+ private void logInvalidFileNameFormat(String cur) {
+ LOGGER.warn("Invalid file name format for mount-table version file: {}. "
+ + "The valid file name format is mount-table-name..xml",
+ cur);
+ }
+
+}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/InodeTree.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/InodeTree.java
index 69923438ecc20..50c839b52b654 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/InodeTree.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/InodeTree.java
@@ -123,6 +123,7 @@ static class INodeDir extends INode {
private final Map> children = new HashMap<>();
private T internalDirFs = null; //filesystem of this internal directory
private boolean isRoot = false;
+ private INodeLink fallbackLink = null;
INodeDir(final String pathToNode, final UserGroupInformation aUgi) {
super(pathToNode, aUgi);
@@ -149,6 +150,17 @@ boolean isRoot() {
return isRoot;
}
+ INodeLink getFallbackLink() {
+ return fallbackLink;
+ }
+
+ void addFallbackLink(INodeLink link) throws IOException {
+ if (!isRoot) {
+ throw new IOException("Fallback link can only be added for root");
+ }
+ this.fallbackLink = link;
+ }
+
Map> getChildren() {
return Collections.unmodifiableMap(children);
}
@@ -580,6 +592,7 @@ protected InodeTree(final Configuration config, final String viewName)
}
}
rootFallbackLink = fallbackLink;
+ getRootDir().addFallbackLink(rootFallbackLink);
}
if (!gotMountTableEntry) {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/MountTableConfigLoader.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/MountTableConfigLoader.java
new file mode 100644
index 0000000000000..bc2c3ea93c58c
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/MountTableConfigLoader.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.viewfs;
+
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ * An interface for loading mount-table configuration. This class can have more
+ * APIs like refreshing mount tables automatically etc.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public interface MountTableConfigLoader {
+
+ /**
+ * Loads the mount-table configuration into given configuration.
+ *
+ * @param mountTableConfigPath - Path of the mount table. It can be a file or
+ * a directory in the case of multiple versions of mount-table
+ * files(Recommended option).
+ * @param conf - Configuration object to add mount table.
+ */
+ void load(String mountTableConfigPath, Configuration conf)
+ throws IOException;
+}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/NflyFSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/NflyFSystem.java
index 53966b8afbfcc..85af68af31434 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/NflyFSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/NflyFSystem.java
@@ -18,8 +18,8 @@
package org.apache.hadoop.fs.viewfs;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
@@ -59,7 +59,7 @@
*/
@Private
final class NflyFSystem extends FileSystem {
- private static final Log LOG = LogFactory.getLog(NflyFSystem.class);
+ private static final Logger LOG = LoggerFactory.getLogger(NflyFSystem.class);
private static final String NFLY_TMP_PREFIX = "_nfly_tmp_";
enum NflyKey {
@@ -212,6 +212,21 @@ private static String getRack(String rackString) {
*/
private NflyFSystem(URI[] uris, Configuration conf, int minReplication,
EnumSet nflyFlags) throws IOException {
+ this(uris, conf, minReplication, nflyFlags, null);
+ }
+
+ /**
+ * Creates a new Nfly instance.
+ *
+ * @param uris the list of uris in the mount point
+ * @param conf configuration object
+ * @param minReplication minimum copies to commit a write op
+ * @param nflyFlags modes such readMostRecent
+ * @param fsGetter to get the file system instance with the given uri
+ * @throws IOException
+ */
+ private NflyFSystem(URI[] uris, Configuration conf, int minReplication,
+ EnumSet nflyFlags, FsGetter fsGetter) throws IOException {
if (uris.length < minReplication) {
throw new IOException(minReplication + " < " + uris.length
+ ": Minimum replication < #destinations");
@@ -238,8 +253,14 @@ private NflyFSystem(URI[] uris, Configuration conf, int minReplication,
nodes = new NflyNode[uris.length];
final Iterator rackIter = rackStrings.iterator();
for (int i = 0; i < nodes.length; i++) {
- nodes[i] = new NflyNode(hostStrings.get(i), rackIter.next(), uris[i],
- conf);
+ if (fsGetter != null) {
+ nodes[i] = new NflyNode(hostStrings.get(i), rackIter.next(),
+ new ChRootedFileSystem(fsGetter.getNewInstance(uris[i], conf),
+ uris[i]));
+ } else {
+ nodes[i] =
+ new NflyNode(hostStrings.get(i), rackIter.next(), uris[i], conf);
+ }
}
// sort all the uri's by distance from myNode, the local file system will
// automatically be the the first one.
@@ -921,7 +942,7 @@ private static void processThrowable(NflyNode nflyNode, String op,
* @throws IOException
*/
static FileSystem createFileSystem(URI[] uris, Configuration conf,
- String settings) throws IOException {
+ String settings, FsGetter fsGetter) throws IOException {
// assert settings != null
int minRepl = DEFAULT_MIN_REPLICATION;
EnumSet nflyFlags = EnumSet.noneOf(NflyKey.class);
@@ -946,6 +967,6 @@ static FileSystem createFileSystem(URI[] uris, Configuration conf,
throw new IllegalArgumentException(nflyKey + ": Infeasible");
}
}
- return new NflyFSystem(uris, conf, minRepl, nflyFlags);
+ return new NflyFSystem(uris, conf, minRepl, nflyFlags, fsGetter);
}
}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
index faa374a39789b..4f02feeebec8b 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
@@ -18,9 +18,9 @@
package org.apache.hadoop.fs.viewfs;
import static org.apache.hadoop.fs.impl.PathCapabilitiesSupport.validatePathCapabilityArgs;
-import static org.apache.hadoop.fs.viewfs.Constants.PERMISSION_555;
import static org.apache.hadoop.fs.viewfs.Constants.CONFIG_VIEWFS_ENABLE_INNER_CACHE;
import static org.apache.hadoop.fs.viewfs.Constants.CONFIG_VIEWFS_ENABLE_INNER_CACHE_DEFAULT;
+import static org.apache.hadoop.fs.viewfs.Constants.PERMISSION_555;
import java.io.FileNotFoundException;
import java.io.IOException;
@@ -35,9 +35,9 @@
import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
import java.util.Objects;
import java.util.Set;
-import java.util.Map.Entry;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
@@ -96,16 +96,28 @@ static AccessControlException readOnlyMountTable(final String operation,
return readOnlyMountTable(operation, p.toString());
}
+ /**
+ * Gets file system creator instance.
+ */
+ protected FsGetter fsGetter() {
+ return new FsGetter();
+ }
+
/**
* Caching children filesystems. HADOOP-15565.
*/
static class InnerCache {
private Map map = new HashMap<>();
+ private FsGetter fsCreator;
+
+ InnerCache(FsGetter fsCreator) {
+ this.fsCreator = fsCreator;
+ }
FileSystem get(URI uri, Configuration config) throws IOException {
Key key = new Key(uri);
if (map.get(key) == null) {
- FileSystem fs = FileSystem.newInstance(uri, config);
+ FileSystem fs = fsCreator.getNewInstance(uri, config);
map.put(key, fs);
return fs;
} else {
@@ -193,7 +205,7 @@ public URI[] getTargetFileSystemURIs() {
final long creationTime; // of the the mount table
final UserGroupInformation ugi; // the user/group of user who created mtable
- URI myUri;
+ private URI myUri;
private Path workingDir;
Configuration config;
InodeTree fsState; // the fs state; ie the mount table
@@ -255,13 +267,13 @@ public void initialize(final URI theUri, final Configuration conf)
config = conf;
enableInnerCache = config.getBoolean(CONFIG_VIEWFS_ENABLE_INNER_CACHE,
CONFIG_VIEWFS_ENABLE_INNER_CACHE_DEFAULT);
- final InnerCache innerCache = new InnerCache();
+ FsGetter fsGetter = fsGetter();
+ final InnerCache innerCache = new InnerCache(fsGetter);
// Now build client side view (i.e. client side mount table) from config.
final String authority = theUri.getAuthority();
try {
- myUri = new URI(FsConstants.VIEWFS_SCHEME, authority, "/", null, null);
+ myUri = new URI(getScheme(), authority, "/", null, null);
fsState = new InodeTree(conf, authority) {
-
@Override
protected FileSystem getTargetFileSystem(final URI uri)
throws URISyntaxException, IOException {
@@ -269,7 +281,7 @@ protected FileSystem getTargetFileSystem(final URI uri)
if (enableInnerCache) {
fs = innerCache.get(uri, config);
} else {
- fs = FileSystem.get(uri, config);
+ fs = fsGetter.get(uri, config);
}
return new ChRootedFileSystem(fs, uri);
}
@@ -283,7 +295,8 @@ protected FileSystem getTargetFileSystem(final INodeDir dir)
@Override
protected FileSystem getTargetFileSystem(final String settings,
final URI[] uris) throws URISyntaxException, IOException {
- return NflyFSystem.createFileSystem(uris, config, settings);
+ return NflyFSystem.createFileSystem(uris, config, settings,
+ fsGetter);
}
};
workingDir = this.getHomeDirectory();
@@ -1167,10 +1180,19 @@ public FileStatus getFileStatus(Path f) throws IOException {
}
+ /**
+ * {@inheritDoc}
+ *
+ * Note: listStatus on root("/") considers listing from fallbackLink if
+ * available. If the same directory name is present in configured mount
+ * path as well as in fallback link, then only the configured mount path
+ * will be listed in the returned result.
+ */
@Override
public FileStatus[] listStatus(Path f) throws AccessControlException,
FileNotFoundException, IOException {
checkPathIsSlash(f);
+ FileStatus[] fallbackStatuses = listStatusForFallbackLink();
FileStatus[] result = new FileStatus[theInternalDir.getChildren().size()];
int i = 0;
for (Entry> iEntry :
@@ -1193,7 +1215,45 @@ public FileStatus[] listStatus(Path f) throws AccessControlException,
myUri, null));
}
}
- return result;
+ if (fallbackStatuses.length > 0) {
+ return consolidateFileStatuses(fallbackStatuses, result);
+ } else {
+ return result;
+ }
+ }
+
+ private FileStatus[] consolidateFileStatuses(FileStatus[] fallbackStatuses,
+ FileStatus[] mountPointStatuses) {
+ ArrayList result = new ArrayList<>();
+ Set pathSet = new HashSet<>();
+ for (FileStatus status : mountPointStatuses) {
+ result.add(status);
+ pathSet.add(status.getPath().getName());
+ }
+ for (FileStatus status : fallbackStatuses) {
+ if (!pathSet.contains(status.getPath().getName())) {
+ result.add(status);
+ }
+ }
+ return result.toArray(new FileStatus[0]);
+ }
+
+ private FileStatus[] listStatusForFallbackLink() throws IOException {
+ if (theInternalDir.isRoot() &&
+ theInternalDir.getFallbackLink() != null) {
+ FileSystem linkedFs =
+ theInternalDir.getFallbackLink().getTargetFileSystem();
+ // Fallback link is only applicable for root
+ FileStatus[] statuses = linkedFs.listStatus(new Path("/"));
+ for (FileStatus status : statuses) {
+ // Fix the path back to viewfs scheme
+ status.setPath(
+ new Path(myUri.toString(), status.getPath().getName()));
+ }
+ return statuses;
+ } else {
+ return new FileStatus[0];
+ }
}
@Override
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystemOverloadScheme.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystemOverloadScheme.java
new file mode 100644
index 0000000000000..36f9cd104cb6b
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystemOverloadScheme.java
@@ -0,0 +1,259 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.viewfs;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
+import java.net.URI;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FsConstants;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.UnsupportedFileSystemException;
+
+/******************************************************************************
+ * This class is extended from the ViewFileSystem for the overloaded scheme
+ * file system. Mount link configurations and in-memory mount table
+ * building behaviors are inherited from ViewFileSystem. Unlike ViewFileSystem
+ * scheme (viewfs://), the users would be able to use any scheme.
+ *
+ * To use this class, the following configurations need to be added in
+ * core-site.xml file.
+ * 1) fs..impl
+ * = org.apache.hadoop.fs.viewfs.ViewFileSystemOverloadScheme
+ * 2) fs.viewfs.overload.scheme.target..impl
+ * = "
+ *
+ * Here can be any scheme, but with that scheme there should be a
+ * hadoop compatible file system available. Second configuration value should
+ * be the respective scheme's file system implementation class.
+ * Example: if scheme is configured with "hdfs", then the 2nd configuration
+ * class name will be org.apache.hadoop.hdfs.DistributedFileSystem.
+ * if scheme is configured with "s3a", then the 2nd configuration class name
+ * will be org.apache.hadoop.fs.s3a.S3AFileSystem.
+ *
+ * Use Case 1:
+ * ===========
+ * If users want some of their existing cluster (hdfs://Cluster)
+ * data to mount with other hdfs and object store clusters(hdfs://NN1,
+ * o3fs://bucket1.volume1/, s3a://bucket1/)
+ *
+ * fs.viewfs.mounttable.Cluster./user = hdfs://NN1/user
+ * fs.viewfs.mounttable.Cluster./data = o3fs://bucket1.volume1/data
+ * fs.viewfs.mounttable.Cluster./backup = s3a://bucket1/backup/
+ *
+ * Op1: Create file hdfs://Cluster/user/fileA will go to hdfs://NN1/user/fileA
+ * Op2: Create file hdfs://Cluster/data/datafile will go to
+ * o3fs://bucket1.volume1/data/datafile
+ * Op3: Create file hdfs://Cluster/backup/data.zip will go to
+ * s3a://bucket1/backup/data.zip
+ *
+ * Use Case 2:
+ * ===========
+ * If users want some of their existing cluster (s3a://bucketA/)
+ * data to mount with other hdfs and object store clusters
+ * (hdfs://NN1, o3fs://bucket1.volume1/)
+ *
+ * fs.viewfs.mounttable.bucketA./user = hdfs://NN1/user
+ * fs.viewfs.mounttable.bucketA./data = o3fs://bucket1.volume1/data
+ * fs.viewfs.mounttable.bucketA./salesDB = s3a://bucketA/salesDB/
+ *
+ * Op1: Create file s3a://bucketA/user/fileA will go to hdfs://NN1/user/fileA
+ * Op2: Create file s3a://bucketA/data/datafile will go to
+ * o3fs://bucket1.volume1/data/datafile
+ * Op3: Create file s3a://bucketA/salesDB/dbfile will go to
+ * s3a://bucketA/salesDB/dbfile
+ *****************************************************************************/
+@InterfaceAudience.LimitedPrivate({ "MapReduce", "HBase", "Hive" })
+@InterfaceStability.Evolving
+public class ViewFileSystemOverloadScheme extends ViewFileSystem {
+ private URI myUri;
+ public ViewFileSystemOverloadScheme() throws IOException {
+ super();
+ }
+
+ @Override
+ public String getScheme() {
+ return myUri.getScheme();
+ }
+
+ @Override
+ public void initialize(URI theUri, Configuration conf) throws IOException {
+ this.myUri = theUri;
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Initializing the ViewFileSystemOverloadScheme with the uri: "
+ + theUri);
+ }
+ String mountTableConfigPath =
+ conf.get(Constants.CONFIG_VIEWFS_MOUNTTABLE_PATH);
+ if (null != mountTableConfigPath) {
+ MountTableConfigLoader loader = new HCFSMountTableConfigLoader();
+ loader.load(mountTableConfigPath, conf);
+ } else {
+ // TODO: Should we fail here.?
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(
+ "Missing configuration for fs.viewfs.mounttable.path. Proceeding"
+ + "with core-site.xml mount-table information if avaialable.");
+ }
+ }
+ super.initialize(theUri, conf);
+ }
+
+ /**
+ * This method is overridden because in ViewFileSystemOverloadScheme if
+ * overloaded scheme matches with mounted target fs scheme, file system
+ * should be created without going into fs..impl based resolution.
+ * Otherwise it will end up in an infinite loop as the target will be
+ * resolved again to ViewFileSystemOverloadScheme as fs..impl points
+ * to ViewFileSystemOverloadScheme. So, below method will initialize the
+ * fs.viewfs.overload.scheme.target..impl. Other schemes can
+ * follow fs.newInstance
+ */
+ @Override
+ protected FsGetter fsGetter() {
+ return new ChildFsGetter(getScheme());
+ }
+
+ /**
+ * This class checks whether the rooScheme is same as URI scheme. If both are
+ * same, then it will initialize file systems by using the configured
+ * fs.viewfs.overload.scheme.target..impl class.
+ */
+ static class ChildFsGetter extends FsGetter {
+
+ private final String rootScheme;
+
+ ChildFsGetter(String rootScheme) {
+ this.rootScheme = rootScheme;
+ }
+
+ @Override
+ public FileSystem getNewInstance(URI uri, Configuration conf)
+ throws IOException {
+ if (uri.getScheme().equals(this.rootScheme)) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(
+ "The file system initialized uri scheme is matching with the "
+ + "given target uri scheme. The target uri is: " + uri);
+ }
+ /*
+ * Avoid looping when target fs scheme is matching to overloaded scheme.
+ */
+ return createFileSystem(uri, conf);
+ } else {
+ return FileSystem.newInstance(uri, conf);
+ }
+ }
+
+ /**
+ * When ViewFileSystemOverloadScheme scheme and target uri scheme are
+ * matching, it will not take advantage of FileSystem cache as it will
+ * create instance directly. For caching needs please set
+ * "fs.viewfs.enable.inner.cache" to true.
+ */
+ @Override
+ public FileSystem get(URI uri, Configuration conf) throws IOException {
+ if (uri.getScheme().equals(this.rootScheme)) {
+ // Avoid looping when target fs scheme is matching to overloaded
+ // scheme.
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(
+ "The file system initialized uri scheme is matching with the "
+ + "given target uri scheme. So, the target file system "
+ + "instances will not be cached. To cache fs instances, "
+ + "please set fs.viewfs.enable.inner.cache to true. "
+ + "The target uri is: " + uri);
+ }
+ return createFileSystem(uri, conf);
+ } else {
+ return FileSystem.get(uri, conf);
+ }
+ }
+
+ private FileSystem createFileSystem(URI uri, Configuration conf)
+ throws IOException {
+ final String fsImplConf = String.format(
+ FsConstants.FS_VIEWFS_OVERLOAD_SCHEME_TARGET_FS_IMPL_PATTERN,
+ uri.getScheme());
+ Class clazz = conf.getClass(fsImplConf, null);
+ if (clazz == null) {
+ throw new UnsupportedFileSystemException(
+ String.format("%s=null: %s: %s", fsImplConf,
+ "No overload scheme fs configured", uri.getScheme()));
+ }
+ FileSystem fs = (FileSystem) newInstance(clazz, uri, conf);
+ fs.initialize(uri, conf);
+ return fs;
+ }
+
+ private T newInstance(Class theClass, URI uri, Configuration conf) {
+ T result;
+ try {
+ Constructor meth = theClass.getConstructor();
+ meth.setAccessible(true);
+ result = meth.newInstance();
+ } catch (InvocationTargetException e) {
+ Throwable cause = e.getCause();
+ if (cause instanceof RuntimeException) {
+ throw (RuntimeException) cause;
+ } else {
+ throw new RuntimeException(cause);
+ }
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ return result;
+ }
+
+ }
+
+ /**
+ * This is an admin only API to give access to its child raw file system, if
+ * the path is link. If the given path is an internal directory(path is from
+ * mount paths tree), it will initialize the file system of given path uri
+ * directly. If path cannot be resolved to any internal directory or link, it
+ * will throw NotInMountpointException. Please note, this API will not return
+ * chrooted file system. Instead, this API will get actual raw file system
+ * instances.
+ *
+ * @param path - fs uri path
+ * @param conf - configuration
+ * @throws IOException
+ */
+ public FileSystem getRawFileSystem(Path path, Configuration conf)
+ throws IOException {
+ InodeTree.ResolveResult res;
+ try {
+ res = fsState.resolve(getUriPath(path), true);
+ return res.isInternalDir() ? fsGetter().get(path.toUri(), conf)
+ : ((ChRootedFileSystem) res.targetFileSystem).getMyFs();
+ } catch (FileNotFoundException e) {
+ // No link configured with passed path.
+ throw new NotInMountpointException(path,
+ "No link found for the given path.");
+ }
+ }
+
+}
\ No newline at end of file
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java
index 2c8c1a538e433..607bdb8d423a0 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java
@@ -25,10 +25,12 @@
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.EnumSet;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
+import java.util.Set;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
@@ -950,10 +952,19 @@ public int getUriDefaultPort() {
return -1;
}
+ /**
+ * {@inheritDoc}
+ *
+ * Note: listStatus on root("/") considers listing from fallbackLink if
+ * available. If the same directory name is present in configured mount
+ * path as well as in fallback link, then only the configured mount path
+ * will be listed in the returned result.
+ */
@Override
public FileStatus[] listStatus(final Path f) throws AccessControlException,
IOException {
checkPathIsSlash(f);
+ FileStatus[] fallbackStatuses = listStatusForFallbackLink();
FileStatus[] result = new FileStatus[theInternalDir.getChildren().size()];
int i = 0;
for (Entry> iEntry :
@@ -979,7 +990,45 @@ public FileStatus[] listStatus(final Path f) throws AccessControlException,
myUri, null));
}
}
- return result;
+ if (fallbackStatuses.length > 0) {
+ return consolidateFileStatuses(fallbackStatuses, result);
+ } else {
+ return result;
+ }
+ }
+
+ private FileStatus[] consolidateFileStatuses(FileStatus[] fallbackStatuses,
+ FileStatus[] mountPointStatuses) {
+ ArrayList result = new ArrayList<>();
+ Set pathSet = new HashSet<>();
+ for (FileStatus status : mountPointStatuses) {
+ result.add(status);
+ pathSet.add(status.getPath().getName());
+ }
+ for (FileStatus status : fallbackStatuses) {
+ if (!pathSet.contains(status.getPath().getName())) {
+ result.add(status);
+ }
+ }
+ return result.toArray(new FileStatus[0]);
+ }
+
+ private FileStatus[] listStatusForFallbackLink() throws IOException {
+ if (theInternalDir.isRoot() &&
+ theInternalDir.getFallbackLink() != null) {
+ AbstractFileSystem linkedFs =
+ theInternalDir.getFallbackLink().getTargetFileSystem();
+ // Fallback link is only applicable for root
+ FileStatus[] statuses = linkedFs.listStatus(new Path("/"));
+ for (FileStatus status : statuses) {
+ // Fix the path back to viewfs scheme
+ status.setPath(
+ new Path(myUri.toString(), status.getPath().getName()));
+ }
+ return statuses;
+ } else {
+ return new FileStatus[0];
+ }
}
@Override
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/package-info.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/package-info.java
new file mode 100644
index 0000000000000..89986d0e5ef69
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/package-info.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * ViewFileSystem and ViewFileSystemOverloadScheme classes.
+ */
+@InterfaceAudience.LimitedPrivate({"MapReduce", "HBase", "Hive" })
+@InterfaceStability.Stable
+package org.apache.hadoop.fs.viewfs;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java
index 12de2ef91c413..828a17bcb972e 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java
@@ -579,6 +579,11 @@ public synchronized void processResult(int rc, String path, Object ctx,
fatalError(errorMessage);
}
+ @VisibleForTesting
+ public boolean getWantToBeInElection() {
+ return wantToBeInElection;
+ }
+
/**
* We failed to become active. Re-join the election, but
* sleep for a few seconds after terminating our existing
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAAdmin.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAAdmin.java
index 0693dce4281ec..0950ea7e01c57 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAAdmin.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAAdmin.java
@@ -39,7 +39,6 @@
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
-import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableMap;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -52,16 +51,15 @@
@InterfaceAudience.Private
public abstract class HAAdmin extends Configured implements Tool {
-
- private static final String FORCEFENCE = "forcefence";
- private static final String FORCEACTIVE = "forceactive";
-
+
+ protected static final String FORCEACTIVE = "forceactive";
+
/**
* Undocumented flag which allows an administrator to use manual failover
* state transitions even when auto-failover is enabled. This is an unsafe
* operation, which is why it is not documented in the usage below.
*/
- private static final String FORCEMANUAL = "forcemanual";
+ protected static final String FORCEMANUAL = "forcemanual";
private static final Logger LOG = LoggerFactory.getLogger(HAAdmin.class);
private int rpcTimeoutForChecks = -1;
@@ -72,15 +70,6 @@ public abstract class HAAdmin extends Configured implements Tool {
new UsageInfo("[--"+FORCEACTIVE+"] ", "Transitions the service into Active state"))
.put("-transitionToStandby",
new UsageInfo("", "Transitions the service into Standby state"))
- .put("-transitionToObserver",
- new UsageInfo("",
- "Transitions the service into Observer state"))
- .put("-failover",
- new UsageInfo("[--"+FORCEFENCE+"] [--"+FORCEACTIVE+"] ",
- "Failover from the first service to the second.\n" +
- "Unconditionally fence services if the --"+FORCEFENCE+" option is used.\n" +
- "Try to failover to the target service even if it is not ready if the " +
- "--" + FORCEACTIVE + " option is used."))
.put("-getServiceState",
new UsageInfo("", "Returns the state of the service"))
.put("-getAllServiceState",
@@ -99,6 +88,14 @@ public abstract class HAAdmin extends Configured implements Tool {
protected PrintStream out = System.out;
private RequestSource requestSource = RequestSource.REQUEST_BY_USER;
+ protected RequestSource getRequestSource() {
+ return requestSource;
+ }
+
+ protected void setRequestSource(RequestSource requestSource) {
+ this.requestSource = requestSource;
+ }
+
protected HAAdmin() {
super();
}
@@ -118,34 +115,44 @@ protected String getUsageString() {
return "Usage: HAAdmin";
}
- protected void printUsage(PrintStream errOut) {
- errOut.println(getUsageString());
- for (Map.Entry e : USAGE.entrySet()) {
+ protected void printUsage(PrintStream pStr,
+ Map helpEntries) {
+ pStr.println(getUsageString());
+ for (Map.Entry e : helpEntries.entrySet()) {
String cmd = e.getKey();
UsageInfo usage = e.getValue();
-
+
if (usage.args == null) {
- errOut.println(" [" + cmd + "]");
+ pStr.println(" [" + cmd + "]");
} else {
- errOut.println(" [" + cmd + " " + usage.args + "]");
+ pStr.println(" [" + cmd + " " + usage.args + "]");
}
}
- errOut.println();
- ToolRunner.printGenericCommandUsage(errOut);
+ pStr.println();
+ ToolRunner.printGenericCommandUsage(pStr);
}
-
- private void printUsage(PrintStream errOut, String cmd) {
- UsageInfo usage = USAGE.get(cmd);
+
+ protected void printUsage(PrintStream pStr) {
+ printUsage(pStr, USAGE);
+ }
+
+ protected void printUsage(PrintStream pStr, String cmd,
+ Map helpEntries) {
+ UsageInfo usage = helpEntries.get(cmd);
if (usage == null) {
throw new RuntimeException("No usage for cmd " + cmd);
}
if (usage.args == null) {
- errOut.println(getUsageString() + " [" + cmd + "]");
+ pStr.println(getUsageString() + " [" + cmd + "]");
} else {
- errOut.println(getUsageString() + " [" + cmd + " " + usage.args + "]");
+ pStr.println(getUsageString() + " [" + cmd + " " + usage.args + "]");
}
}
+ protected void printUsage(PrintStream pStr, String cmd) {
+ printUsage(pStr, cmd, USAGE);
+ }
+
private int transitionToActive(final CommandLine cmd)
throws IOException, ServiceFailedException {
String[] argv = cmd.getArgs();
@@ -225,27 +232,6 @@ private int transitionToStandby(final CommandLine cmd)
return 0;
}
- private int transitionToObserver(final CommandLine cmd)
- throws IOException, ServiceFailedException {
- String[] argv = cmd.getArgs();
- if (argv.length != 1) {
- errOut.println("transitionToObserver: incorrect number of arguments");
- printUsage(errOut, "-transitionToObserver");
- return -1;
- }
-
- HAServiceTarget target = resolveTarget(argv[0]);
- if (!checkSupportObserver(target)) {
- return -1;
- }
- if (!checkManualStateManagementOK(target)) {
- return -1;
- }
- HAServiceProtocol proto = target.getProxy(getConf(), 0);
- HAServiceProtocolHelper.transitionToObserver(proto, createReqInfo());
- return 0;
- }
-
/**
* Ensure that we are allowed to manually manage the HA state of the target
* service. If automatic failover is configured, then the automatic
@@ -255,7 +241,7 @@ private int transitionToObserver(final CommandLine cmd)
* @param target the target to check
* @return true if manual state management is allowed
*/
- private boolean checkManualStateManagementOK(HAServiceTarget target) {
+ protected boolean checkManualStateManagementOK(HAServiceTarget target) {
if (target.isAutoFailoverEnabled()) {
if (requestSource != RequestSource.REQUEST_BY_USER_FORCED) {
errOut.println(
@@ -274,93 +260,19 @@ private boolean checkManualStateManagementOK(HAServiceTarget target) {
return true;
}
- /**
- * Check if the target supports the Observer state.
- * @param target the target to check
- * @return true if the target support Observer state, false otherwise.
- */
- private boolean checkSupportObserver(HAServiceTarget target) {
- if (target.supportObserver()) {
- return true;
- } else {
- errOut.println(
- "The target " + target + " doesn't support Observer state.");
- return false;
- }
- }
-
- private StateChangeRequestInfo createReqInfo() {
+ protected StateChangeRequestInfo createReqInfo() {
return new StateChangeRequestInfo(requestSource);
}
- private int failover(CommandLine cmd)
- throws IOException, ServiceFailedException {
- boolean forceFence = cmd.hasOption(FORCEFENCE);
- boolean forceActive = cmd.hasOption(FORCEACTIVE);
-
- int numOpts = cmd.getOptions() == null ? 0 : cmd.getOptions().length;
- final String[] args = cmd.getArgs();
-
- if (numOpts > 3 || args.length != 2) {
- errOut.println("failover: incorrect arguments");
- printUsage(errOut, "-failover");
- return -1;
- }
-
- HAServiceTarget fromNode = resolveTarget(args[0]);
- HAServiceTarget toNode = resolveTarget(args[1]);
-
- // Check that auto-failover is consistently configured for both nodes.
- Preconditions.checkState(
- fromNode.isAutoFailoverEnabled() ==
- toNode.isAutoFailoverEnabled(),
- "Inconsistent auto-failover configs between %s and %s!",
- fromNode, toNode);
-
- if (fromNode.isAutoFailoverEnabled()) {
- if (forceFence || forceActive) {
- // -forceActive doesn't make sense with auto-HA, since, if the node
- // is not healthy, then its ZKFC will immediately quit the election
- // again the next time a health check runs.
- //
- // -forceFence doesn't seem to have any real use cases with auto-HA
- // so it isn't implemented.
- errOut.println(FORCEFENCE + " and " + FORCEACTIVE + " flags not " +
- "supported with auto-failover enabled.");
- return -1;
- }
- try {
- return gracefulFailoverThroughZKFCs(toNode);
- } catch (UnsupportedOperationException e){
- errOut.println("Failover command is not supported with " +
- "auto-failover enabled: " + e.getLocalizedMessage());
- return -1;
- }
- }
-
- FailoverController fc = new FailoverController(getConf(),
- requestSource);
-
- try {
- fc.failover(fromNode, toNode, forceFence, forceActive);
- out.println("Failover from "+args[0]+" to "+args[1]+" successful");
- } catch (FailoverFailedException ffe) {
- errOut.println("Failover failed: " + ffe.getLocalizedMessage());
- return -1;
- }
- return 0;
- }
-
-
/**
* Initiate a graceful failover by talking to the target node's ZKFC.
* This sends an RPC to the ZKFC, which coordinates the failover.
- *
+ *
* @param toNode the node to fail to
* @return status code (0 for success)
* @throws IOException if failover does not succeed
*/
- private int gracefulFailoverThroughZKFCs(HAServiceTarget toNode)
+ protected int gracefulFailoverThroughZKFCs(HAServiceTarget toNode)
throws IOException {
int timeout = FailoverController.getRpcTimeoutToNewActive(getConf());
@@ -443,45 +355,52 @@ public int run(String[] argv) throws Exception {
return -1;
}
}
-
- protected int runCmd(String[] argv) throws Exception {
+
+ protected boolean checkParameterValidity(String[] argv,
+ Map helpEntries){
+
if (argv.length < 1) {
- printUsage(errOut);
- return -1;
+ printUsage(errOut, helpEntries);
+ return false;
}
String cmd = argv[0];
-
if (!cmd.startsWith("-")) {
- errOut.println("Bad command '" + cmd + "': expected command starting with '-'");
- printUsage(errOut);
- return -1;
+ errOut.println("Bad command '" + cmd +
+ "': expected command starting with '-'");
+ printUsage(errOut, helpEntries);
+ return false;
}
-
- if (!USAGE.containsKey(cmd)) {
+
+ if (!helpEntries.containsKey(cmd)) {
errOut.println(cmd.substring(1) + ": Unknown command");
- printUsage(errOut);
+ printUsage(errOut, helpEntries);
+ return false;
+ }
+ return true;
+ }
+
+ protected boolean checkParameterValidity(String[] argv){
+ return checkParameterValidity(argv, USAGE);
+ }
+
+ protected int runCmd(String[] argv) throws Exception {
+ if (!checkParameterValidity(argv, USAGE)){
return -1;
}
-
- Options opts = new Options();
+ String cmd = argv[0];
+ Options opts = new Options();
// Add command-specific options
- if ("-failover".equals(cmd)) {
- addFailoverCliOpts(opts);
- }
if("-transitionToActive".equals(cmd)) {
addTransitionToActiveCliOpts(opts);
}
// Mutative commands take FORCEMANUAL option
if ("-transitionToActive".equals(cmd) ||
- "-transitionToStandby".equals(cmd) ||
- "-transitionToObserver".equals(cmd) ||
- "-failover".equals(cmd)) {
+ "-transitionToStandby".equals(cmd)) {
opts.addOption(FORCEMANUAL, false,
"force manual control even if auto-failover is enabled");
}
-
CommandLine cmdLine = parseOpts(cmd, opts, argv);
if (cmdLine == null) {
// error already printed
@@ -502,10 +421,6 @@ protected int runCmd(String[] argv) throws Exception {
return transitionToActive(cmdLine);
} else if ("-transitionToStandby".equals(cmd)) {
return transitionToStandby(cmdLine);
- } else if ("-transitionToObserver".equals(cmd)) {
- return transitionToObserver(cmdLine);
- } else if ("-failover".equals(cmd)) {
- return failover(cmdLine);
} else if ("-getServiceState".equals(cmd)) {
return getServiceState(cmdLine);
} else if ("-getAllServiceState".equals(cmd)) {
@@ -544,7 +459,7 @@ protected int getAllServiceState() {
return 0;
}
- private boolean confirmForceManual() throws IOException {
+ protected boolean confirmForceManual() throws IOException {
return ToolRunner.confirmPrompt(
"You have specified the --" + FORCEMANUAL + " flag. This flag is " +
"dangerous, as it can induce a split-brain scenario that WILL " +
@@ -559,16 +474,7 @@ private boolean confirmForceManual() throws IOException {
"Are you sure you want to continue?");
}
- /**
- * Add CLI options which are specific to the failover command and no
- * others.
- */
- private void addFailoverCliOpts(Options failoverOpts) {
- failoverOpts.addOption(FORCEFENCE, false, "force fencing");
- failoverOpts.addOption(FORCEACTIVE, false, "force failover");
- // Don't add FORCEMANUAL, since that's added separately for all commands
- // that change state.
- }
+
/**
* Add CLI options which are specific to the transitionToActive command and
@@ -577,39 +483,47 @@ private void addFailoverCliOpts(Options failoverOpts) {
private void addTransitionToActiveCliOpts(Options transitionToActiveCliOpts) {
transitionToActiveCliOpts.addOption(FORCEACTIVE, false, "force active");
}
-
- private CommandLine parseOpts(String cmdName, Options opts, String[] argv) {
+
+ protected CommandLine parseOpts(String cmdName, Options opts, String[] argv,
+ Map helpEntries) {
try {
// Strip off the first arg, since that's just the command name
- argv = Arrays.copyOfRange(argv, 1, argv.length);
+ argv = Arrays.copyOfRange(argv, 1, argv.length);
return new GnuParser().parse(opts, argv);
} catch (ParseException pe) {
errOut.println(cmdName.substring(1) +
": incorrect arguments");
- printUsage(errOut, cmdName);
+ printUsage(errOut, cmdName, helpEntries);
return null;
}
}
- private int help(String[] argv) {
+ protected CommandLine parseOpts(String cmdName, Options opts, String[] argv) {
+ return parseOpts(cmdName, opts, argv, USAGE);
+ }
+ protected int help(String[] argv) {
+ return help(argv, USAGE);
+ }
+
+ protected int help(String[] argv, Map helpEntries) {
if (argv.length == 1) { // only -help
- printUsage(out);
+ printUsage(out, helpEntries);
return 0;
} else if (argv.length != 2) {
- printUsage(errOut, "-help");
+ printUsage(errOut, "-help", helpEntries);
return -1;
}
String cmd = argv[1];
if (!cmd.startsWith("-")) {
cmd = "-" + cmd;
}
- UsageInfo usageInfo = USAGE.get(cmd);
+ UsageInfo usageInfo = helpEntries.get(cmd);
if (usageInfo == null) {
errOut.println(cmd + ": Unknown command");
- printUsage(errOut);
+ printUsage(errOut, helpEntries);
return -1;
}
-
+
if (usageInfo.args == null) {
out.println(cmd + ": " + usageInfo.help);
} else {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ZKFCRpcServer.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ZKFCRpcServer.java
index 7ea5188ad8338..61ea53c420ab1 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ZKFCRpcServer.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ZKFCRpcServer.java
@@ -20,6 +20,7 @@
import java.io.IOException;
import java.net.InetSocketAddress;
+import org.apache.hadoop.HadoopIllegalArgumentException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
@@ -33,7 +34,7 @@
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.authorize.PolicyProvider;
-import com.google.protobuf.BlockingService;
+import org.apache.hadoop.thirdparty.protobuf.BlockingService;
@InterfaceAudience.LimitedPrivate("HDFS")
@InterfaceStability.Evolving
@@ -63,6 +64,12 @@ public class ZKFCRpcServer implements ZKFCProtocol {
// set service-level authorization security policy
if (conf.getBoolean(
CommonConfigurationKeys.HADOOP_SECURITY_AUTHORIZATION, false)) {
+ if (policy == null) {
+ throw new HadoopIllegalArgumentException(
+ CommonConfigurationKeys.HADOOP_SECURITY_AUTHORIZATION
+ + "is configured to true but service-level"
+ + "authorization security policy is null.");
+ }
server.refreshServiceAcl(conf, policy);
}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ZKFailoverController.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ZKFailoverController.java
index ee4ca1a6084a9..3718b7cdb0cc0 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ZKFailoverController.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ZKFailoverController.java
@@ -157,7 +157,10 @@ public HAServiceTarget getLocalTarget() {
return localTarget;
}
- HAServiceState getServiceState() { return serviceState; }
+ @VisibleForTesting
+ public HAServiceState getServiceState() {
+ return serviceState;
+ }
public int run(final String[] args) throws Exception {
if (!localTarget.isAutoFailoverEnabled()) {
@@ -315,9 +318,10 @@ private void initHM() {
healthMonitor.addServiceStateCallback(new ServiceStateCallBacks());
healthMonitor.start();
}
-
+
protected void initRPC() throws IOException {
InetSocketAddress bindAddr = getRpcAddressToBindTo();
+ LOG.info("ZKFC RpcServer binding to {}", bindAddr);
rpcServer = new ZKFCRpcServer(conf, bindAddr, this, getPolicyProvider());
}
@@ -799,7 +803,9 @@ private void recheckElectability() {
switch (lastHealthState) {
case SERVICE_HEALTHY:
- elector.joinElection(targetToData(localTarget));
+ if(serviceState != HAServiceState.OBSERVER) {
+ elector.joinElection(targetToData(localTarget));
+ }
if (quitElectionOnBadState) {
quitElectionOnBadState = false;
}
@@ -909,7 +915,7 @@ protected synchronized void setLastHealthState(HealthMonitor.State newState) {
}
@VisibleForTesting
- ActiveStandbyElector getElectorForTests() {
+ public ActiveStandbyElector getElectorForTests() {
return elector;
}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/protocolPB/HAServiceProtocolClientSideTranslatorPB.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/protocolPB/HAServiceProtocolClientSideTranslatorPB.java
index fec519f3761d4..e53820cd13107 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/protocolPB/HAServiceProtocolClientSideTranslatorPB.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/protocolPB/HAServiceProtocolClientSideTranslatorPB.java
@@ -43,8 +43,8 @@
import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.security.UserGroupInformation;
-import com.google.protobuf.RpcController;
-import com.google.protobuf.ServiceException;
+import org.apache.hadoop.thirdparty.protobuf.RpcController;
+import org.apache.hadoop.thirdparty.protobuf.ServiceException;
/**
* This class is the client side translator to translate the requests made on
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/protocolPB/HAServiceProtocolServerSideTranslatorPB.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/protocolPB/HAServiceProtocolServerSideTranslatorPB.java
index 72787cfe9937a..8613a469779f3 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/protocolPB/HAServiceProtocolServerSideTranslatorPB.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/protocolPB/HAServiceProtocolServerSideTranslatorPB.java
@@ -40,8 +40,8 @@
import org.apache.hadoop.ipc.ProtocolSignature;
import org.apache.hadoop.ipc.RPC;
-import com.google.protobuf.RpcController;
-import com.google.protobuf.ServiceException;
+import org.apache.hadoop.thirdparty.protobuf.RpcController;
+import org.apache.hadoop.thirdparty.protobuf.ServiceException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/protocolPB/ZKFCProtocolClientSideTranslatorPB.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/protocolPB/ZKFCProtocolClientSideTranslatorPB.java
index 62896fa8e7418..7001d93995f0f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/protocolPB/ZKFCProtocolClientSideTranslatorPB.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/protocolPB/ZKFCProtocolClientSideTranslatorPB.java
@@ -34,8 +34,8 @@
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.UserGroupInformation;
-import com.google.protobuf.RpcController;
-import com.google.protobuf.ServiceException;
+import org.apache.hadoop.thirdparty.protobuf.RpcController;
+import org.apache.hadoop.thirdparty.protobuf.ServiceException;
public class ZKFCProtocolClientSideTranslatorPB implements
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/protocolPB/ZKFCProtocolServerSideTranslatorPB.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/protocolPB/ZKFCProtocolServerSideTranslatorPB.java
index 549499885df41..f822200ab9fa0 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/protocolPB/ZKFCProtocolServerSideTranslatorPB.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/protocolPB/ZKFCProtocolServerSideTranslatorPB.java
@@ -29,8 +29,8 @@
import org.apache.hadoop.ipc.ProtocolSignature;
import org.apache.hadoop.ipc.RPC;
-import com.google.protobuf.RpcController;
-import com.google.protobuf.ServiceException;
+import org.apache.hadoop.thirdparty.protobuf.RpcController;
+import org.apache.hadoop.thirdparty.protobuf.ServiceException;
@InterfaceAudience.Private
@InterfaceStability.Stable
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java
index 7e7d64423f169..3fd74f0e89a27 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java
@@ -154,6 +154,10 @@ public final class HttpServer2 implements FilterContainer {
public static final String FILTER_INITIALIZER_PROPERTY
= "hadoop.http.filter.initializers";
+ public static final String HTTP_SNI_HOST_CHECK_ENABLED_KEY
+ = "hadoop.http.sni.host.check.enabled";
+ public static final boolean HTTP_SNI_HOST_CHECK_ENABLED_DEFAULT = false;
+
// The ServletContext attribute where the daemon Configuration
// gets stored.
public static final String CONF_CONTEXT_ATTRIBUTE = "hadoop.conf";
@@ -233,6 +237,8 @@ public static class Builder {
private boolean xFrameEnabled;
private XFrameOption xFrameOption = XFrameOption.SAMEORIGIN;
+ private boolean sniHostCheckEnabled;
+
public Builder setName(String name){
this.name = name;
return this;
@@ -377,6 +383,17 @@ public Builder setXFrameOption(String option) {
return this;
}
+ /**
+ * Enable or disable sniHostCheck.
+ *
+ * @param sniHostCheckEnabled Enable sniHostCheck if true, else disable it.
+ * @return Builder.
+ */
+ public Builder setSniHostCheckEnabled(boolean sniHostCheckEnabled) {
+ this.sniHostCheckEnabled = sniHostCheckEnabled;
+ return this;
+ }
+
/**
* A wrapper of {@link Configuration#getPassword(String)}. It returns
* String instead of char[].
@@ -471,6 +488,13 @@ public HttpServer2 build() throws IOException {
int backlogSize = conf.getInt(HTTP_SOCKET_BACKLOG_SIZE_KEY,
HTTP_SOCKET_BACKLOG_SIZE_DEFAULT);
+ // If setSniHostCheckEnabled() is used to enable SNI hostname check,
+ // configuration lookup is skipped.
+ if (!sniHostCheckEnabled) {
+ sniHostCheckEnabled = conf.getBoolean(HTTP_SNI_HOST_CHECK_ENABLED_KEY,
+ HTTP_SNI_HOST_CHECK_ENABLED_DEFAULT);
+ }
+
for (URI ep : endpoints) {
final ServerConnector connector;
String scheme = ep.getScheme();
@@ -514,22 +538,29 @@ private ServerConnector createHttpChannelConnector(
private ServerConnector createHttpsChannelConnector(
Server server, HttpConfiguration httpConfig) {
httpConfig.setSecureScheme(HTTPS_SCHEME);
- httpConfig.addCustomizer(new SecureRequestCustomizer());
+ httpConfig.addCustomizer(
+ new SecureRequestCustomizer(sniHostCheckEnabled));
ServerConnector conn = createHttpChannelConnector(server, httpConfig);
SslContextFactory.Server sslContextFactory =
new SslContextFactory.Server();
sslContextFactory.setNeedClientAuth(needsClientAuth);
- sslContextFactory.setKeyManagerPassword(keyPassword);
+ if (keyPassword != null) {
+ sslContextFactory.setKeyManagerPassword(keyPassword);
+ }
if (keyStore != null) {
sslContextFactory.setKeyStorePath(keyStore);
sslContextFactory.setKeyStoreType(keyStoreType);
- sslContextFactory.setKeyStorePassword(keyStorePassword);
+ if (keyStorePassword != null) {
+ sslContextFactory.setKeyStorePassword(keyStorePassword);
+ }
}
if (trustStore != null) {
sslContextFactory.setTrustStorePath(trustStore);
sslContextFactory.setTrustStoreType(trustStoreType);
- sslContextFactory.setTrustStorePassword(trustStorePassword);
+ if (trustStorePassword != null) {
+ sslContextFactory.setTrustStorePassword(trustStorePassword);
+ }
}
if(null != excludeCiphers && !excludeCiphers.isEmpty()) {
sslContextFactory.setExcludeCipherSuites(
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ArrayWritable.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ArrayWritable.java
index 44b6aaa7af310..c6bc0536473e9 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ArrayWritable.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ArrayWritable.java
@@ -18,8 +18,10 @@
package org.apache.hadoop.io;
-import java.io.*;
-import java.lang.reflect.Array;
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.Arrays;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
@@ -42,7 +44,7 @@
@InterfaceAudience.Public
@InterfaceStability.Stable
public class ArrayWritable implements Writable {
- private Class valueClass;
+ private final Class valueClass;
private Writable[] values;
public ArrayWritable(Class valueClass) {
@@ -64,7 +66,7 @@ public ArrayWritable(String[] strings) {
}
}
- public Class getValueClass() {
+ public Class getValueClass() {
return valueClass;
}
@@ -77,16 +79,16 @@ public String[] toStrings() {
}
public Object toArray() {
- Object result = Array.newInstance(valueClass, values.length);
- for (int i = 0; i < values.length; i++) {
- Array.set(result, i, values[i]);
- }
- return result;
+ return Arrays.copyOf(values, values.length);
}
- public void set(Writable[] values) { this.values = values; }
+ public void set(Writable[] values) {
+ this.values = values;
+ }
- public Writable[] get() { return values; }
+ public Writable[] get() {
+ return values;
+ }
@Override
public void readFields(DataInput in) throws IOException {
@@ -106,5 +108,11 @@ public void write(DataOutput out) throws IOException {
}
}
+ @Override
+ public String toString() {
+ return "ArrayWritable [valueClass=" + valueClass + ", values="
+ + Arrays.toString(values) + "]";
+ }
+
}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/BytesWritable.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/BytesWritable.java
index 7d7b75ba05a00..c5538c9e56e85 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/BytesWritable.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/BytesWritable.java
@@ -19,6 +19,9 @@
package org.apache.hadoop.io;
import java.io.IOException;
+import java.util.Arrays;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
import java.io.DataInput;
import java.io.DataOutput;
@@ -35,17 +38,22 @@
@InterfaceStability.Stable
public class BytesWritable extends BinaryComparable
implements WritableComparable {
+ private static final int MAX_ARRAY_SIZE = Integer.MAX_VALUE - 8;
private static final int LENGTH_BYTES = 4;
- private static final byte[] EMPTY_BYTES = {};
+
+ private static final byte[] EMPTY_BYTES = new byte[0];
private int size;
private byte[] bytes;
-
+
/**
* Create a zero-size sequence.
*/
- public BytesWritable() {this(EMPTY_BYTES);}
-
+ public BytesWritable() {
+ this.bytes = EMPTY_BYTES;
+ this.size = 0;
+ }
+
/**
* Create a BytesWritable using the byte array as the initial value.
* @param bytes This array becomes the backing storage for the object.
@@ -65,17 +73,15 @@ public BytesWritable(byte[] bytes, int length) {
this.bytes = bytes;
this.size = length;
}
-
+
/**
* Get a copy of the bytes that is exactly the length of the data.
* See {@link #getBytes()} for faster access to the underlying array.
*/
public byte[] copyBytes() {
- byte[] result = new byte[size];
- System.arraycopy(bytes, 0, result, 0, size);
- return result;
+ return Arrays.copyOf(bytes, size);
}
-
+
/**
* Get the data backing the BytesWritable. Please use {@link #copyBytes()}
* if you need the returned array to be precisely the length of the data.
@@ -111,7 +117,7 @@ public int getLength() {
public int getSize() {
return getLength();
}
-
+
/**
* Change the size of the buffer. The values in the old range are preserved
* and any new values are undefined. The capacity is changed if it is
@@ -121,41 +127,37 @@ public int getSize() {
public void setSize(int size) {
if (size > getCapacity()) {
// Avoid overflowing the int too early by casting to a long.
- long newSize = Math.min(Integer.MAX_VALUE, (3L * size) / 2L);
+ long newSize = Math.min(MAX_ARRAY_SIZE, (3L * size) / 2L);
setCapacity((int) newSize);
}
this.size = size;
}
-
+
/**
* Get the capacity, which is the maximum size that could handled without
* resizing the backing storage.
+ *
* @return The number of bytes
*/
public int getCapacity() {
return bytes.length;
}
-
+
/**
- * Change the capacity of the backing storage.
- * The data is preserved.
- * @param new_cap The new capacity in bytes.
+ * Change the capacity of the backing storage. The data is preserved.
+ *
+ * @param capacity The new capacity in bytes.
*/
- public void setCapacity(int new_cap) {
- if (new_cap != getCapacity()) {
- byte[] new_data = new byte[new_cap];
- if (new_cap < size) {
- size = new_cap;
- }
- if (size != 0) {
- System.arraycopy(bytes, 0, new_data, 0, size);
- }
- bytes = new_data;
+ public void setCapacity(final int capacity) {
+ if (capacity != getCapacity()) {
+ this.size = Math.min(size, capacity);
+ this.bytes = Arrays.copyOf(this.bytes, capacity);
}
}
/**
* Set the BytesWritable to the contents of the given newData.
+ *
* @param newData the value to set this BytesWritable to.
*/
public void set(BytesWritable newData) {
@@ -163,7 +165,8 @@ public void set(BytesWritable newData) {
}
/**
- * Set the value to a copy of the given byte range
+ * Set the value to a copy of the given byte range.
+ *
* @param newData the new values to copy in
* @param offset the offset in newData to start at
* @param length the number of bytes to copy
@@ -174,25 +177,18 @@ public void set(byte[] newData, int offset, int length) {
System.arraycopy(newData, offset, bytes, 0, size);
}
- // inherit javadoc
@Override
public void readFields(DataInput in) throws IOException {
setSize(0); // clear the old data
setSize(in.readInt());
in.readFully(bytes, 0, size);
}
-
- // inherit javadoc
+
@Override
public void write(DataOutput out) throws IOException {
out.writeInt(size);
out.write(bytes, 0, size);
}
-
- @Override
- public int hashCode() {
- return super.hashCode();
- }
/**
* Are the two byte sequences equal?
@@ -204,25 +200,19 @@ public boolean equals(Object right_obj) {
return false;
}
+ @Override
+ public int hashCode() {
+ return super.hashCode();
+ }
+
/**
* Generate the stream of bytes as hex pairs separated by ' '.
*/
@Override
- public String toString() {
- StringBuilder sb = new StringBuilder(3*size);
- for (int idx = 0; idx < size; idx++) {
- // if not the first, put a blank separator in
- if (idx != 0) {
- sb.append(' ');
- }
- String num = Integer.toHexString(0xff & bytes[idx]);
- // if it is only one digit, add a leading 0.
- if (num.length() < 2) {
- sb.append('0');
- }
- sb.append(num);
- }
- return sb.toString();
+ public String toString() {
+ return IntStream.range(0, size)
+ .mapToObj(idx -> String.format("%02x", bytes[idx]))
+ .collect(Collectors.joining(" "));
}
/** A Comparator optimized for BytesWritable. */
@@ -230,20 +220,20 @@ public static class Comparator extends WritableComparator {
public Comparator() {
super(BytesWritable.class);
}
-
+
/**
* Compare the buffers in serialized form.
*/
@Override
public int compare(byte[] b1, int s1, int l1,
byte[] b2, int s2, int l2) {
- return compareBytes(b1, s1+LENGTH_BYTES, l1-LENGTH_BYTES,
- b2, s2+LENGTH_BYTES, l2-LENGTH_BYTES);
+ return compareBytes(b1, s1 + LENGTH_BYTES, l1 - LENGTH_BYTES,
+ b2, s2 + LENGTH_BYTES, l2 - LENGTH_BYTES);
}
}
-
+
static { // register this comparator
WritableComparator.define(BytesWritable.class, new Comparator());
}
-
+
}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ObjectWritable.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ObjectWritable.java
index 0f0f5c7405a6c..b35a32f288b4b 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ObjectWritable.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ObjectWritable.java
@@ -30,7 +30,7 @@
import org.apache.hadoop.conf.*;
import org.apache.hadoop.util.ProtoUtil;
-import com.google.protobuf.Message;
+import org.apache.hadoop.thirdparty.protobuf.Message;
/** A polymorphic Writable that writes an instance with it's class name.
* Handles arrays, strings and primitive types without a Writable wrapper.
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Text.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Text.java
index 3ab327fe76a30..716de3deb4278 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Text.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Text.java
@@ -24,11 +24,11 @@
import java.nio.ByteBuffer;
import java.nio.CharBuffer;
import java.nio.charset.CharacterCodingException;
-import java.nio.charset.Charset;
import java.nio.charset.CharsetDecoder;
import java.nio.charset.CharsetEncoder;
import java.nio.charset.CodingErrorAction;
import java.nio.charset.MalformedInputException;
+import java.nio.charset.StandardCharsets;
import java.text.CharacterIterator;
import java.text.StringCharacterIterator;
import java.util.Arrays;
@@ -52,63 +52,67 @@
@InterfaceStability.Stable
public class Text extends BinaryComparable
implements WritableComparable {
-
+
private static final ThreadLocal ENCODER_FACTORY =
new ThreadLocal() {
@Override
protected CharsetEncoder initialValue() {
- return Charset.forName("UTF-8").newEncoder().
+ return StandardCharsets.UTF_8.newEncoder().
onMalformedInput(CodingErrorAction.REPORT).
onUnmappableCharacter(CodingErrorAction.REPORT);
}
};
-
+
private static final ThreadLocal DECODER_FACTORY =
new ThreadLocal() {
@Override
protected CharsetDecoder initialValue() {
- return Charset.forName("UTF-8").newDecoder().
+ return StandardCharsets.UTF_8.newDecoder().
onMalformedInput(CodingErrorAction.REPORT).
onUnmappableCharacter(CodingErrorAction.REPORT);
}
};
-
- private static final byte [] EMPTY_BYTES = new byte[0];
-
- private byte[] bytes;
- private int length;
+ private static final byte[] EMPTY_BYTES = new byte[0];
+
+ private byte[] bytes = EMPTY_BYTES;
+ private int length = 0;
+
+ /**
+ * Construct an empty text string.
+ */
public Text() {
- bytes = EMPTY_BYTES;
}
- /** Construct from a string.
+ /**
+ * Construct from a string.
*/
public Text(String string) {
set(string);
}
- /** Construct from another text. */
+ /**
+ * Construct from another text.
+ */
public Text(Text utf8) {
set(utf8);
}
- /** Construct from a byte array.
+ /**
+ * Construct from a byte array.
*/
public Text(byte[] utf8) {
set(utf8);
}
-
+
/**
* Get a copy of the bytes that is exactly the length of the data.
* See {@link #getBytes()} for faster access to the underlying array.
*/
public byte[] copyBytes() {
- byte[] result = new byte[length];
- System.arraycopy(bytes, 0, result, 0, length);
- return result;
+ return Arrays.copyOf(bytes, length);
}
-
+
/**
* Returns the raw bytes; however, only data up to {@link #getLength()} is
* valid. Please use {@link #copyBytes()} if you
@@ -119,12 +123,14 @@ public byte[] getBytes() {
return bytes;
}
- /** Returns the number of bytes in the byte array */
+ /**
+ * Returns the number of bytes in the byte array.
+ */
@Override
public int getLength() {
return length;
}
-
+
/**
* Returns the Unicode Scalar Value (32-bit integer value)
* for the character at position. Note that this
@@ -136,15 +142,15 @@ public int getLength() {
public int charAt(int position) {
if (position > this.length) return -1; // too long
if (position < 0) return -1; // duh.
-
+
ByteBuffer bb = (ByteBuffer)ByteBuffer.wrap(bytes).position(position);
return bytesToCodePoint(bb.slice());
}
-
+
public int find(String what) {
return find(what, 0);
}
-
+
/**
* Finds any occurrence of what in the backing
* buffer, starting as position start. The starting
@@ -156,11 +162,11 @@ public int find(String what) {
*/
public int find(String what, int start) {
try {
- ByteBuffer src = ByteBuffer.wrap(this.bytes,0,this.length);
+ ByteBuffer src = ByteBuffer.wrap(this.bytes, 0, this.length);
ByteBuffer tgt = encode(what);
byte b = tgt.get();
src.position(start);
-
+
while (src.hasRemaining()) {
if (b == src.get()) { // matching first byte
src.mark(); // save position in loop
@@ -186,54 +192,63 @@ public int find(String what, int start) {
}
return -1; // not found
} catch (CharacterCodingException e) {
- // can't get here
- e.printStackTrace();
- return -1;
+ throw new RuntimeException("Should not have happened", e);
}
- }
- /** Set to contain the contents of a string.
+ }
+
+ /**
+ * Set to contain the contents of a string.
*/
public void set(String string) {
try {
ByteBuffer bb = encode(string, true);
bytes = bb.array();
length = bb.limit();
- }catch(CharacterCodingException e) {
- throw new RuntimeException("Should not have happened ", e);
+ } catch (CharacterCodingException e) {
+ throw new RuntimeException("Should not have happened", e);
}
}
- /** Set to a utf8 byte array
+ /**
+ * Set to a utf8 byte array.
*/
public void set(byte[] utf8) {
set(utf8, 0, utf8.length);
}
-
- /** copy a text. */
+
+ /**
+ * Copy a text.
+ */
public void set(Text other) {
set(other.getBytes(), 0, other.getLength());
}
/**
- * Set the Text to range of bytes
+ * Set the Text to range of bytes.
+ *
* @param utf8 the data to copy from
* @param start the first position of the new string
* @param len the number of bytes of the new string
*/
public void set(byte[] utf8, int start, int len) {
- setCapacity(len, false);
+ ensureCapacity(len);
System.arraycopy(utf8, start, bytes, 0, len);
this.length = len;
}
/**
- * Append a range of bytes to the end of the given text
+ * Append a range of bytes to the end of the given text.
+ *
* @param utf8 the data to copy from
* @param start the first position to append from utf8
* @param len the number of bytes to append
*/
public void append(byte[] utf8, int start, int len) {
- setCapacity(length + len, true);
+ byte[] original = bytes;
+ int capacity = Math.max(length + len, length + (length >> 1));
+ if (ensureCapacity(capacity)) {
+ System.arraycopy(original, 0, bytes, 0, length);
+ }
System.arraycopy(utf8, start, bytes, length, len);
length += len;
}
@@ -250,47 +265,39 @@ public void clear() {
length = 0;
}
- /*
+ /**
* Sets the capacity of this Text object to at least
- * len bytes. If the current buffer is longer,
- * then the capacity and existing content of the buffer are
- * unchanged. If len is larger
- * than the current capacity, the Text object's capacity is
- * increased to match.
- * @param len the number of bytes we need
- * @param keepData should the old data be kept
- */
- private void setCapacity(int len, boolean keepData) {
- if (bytes == null || bytes.length < len) {
- if (bytes != null && keepData) {
- bytes = Arrays.copyOf(bytes, Math.max(len,length << 1));
- } else {
- bytes = new byte[len];
- }
+ * capacity bytes. If the current buffer is longer, then the
+ * capacity and existing content of the buffer are unchanged. If
+ * capacity is larger than the current capacity, the Text
+ * object's capacity is increased to match and any existing data is lost.
+ *
+ * @param capacity the number of bytes we need
+ * @return true if the internal array was resized or false otherwise
+ */
+ private boolean ensureCapacity(final int capacity) {
+ if (bytes.length < capacity) {
+ bytes = new byte[capacity];
+ return true;
}
+ return false;
}
-
- /**
- * Convert text back to string
- * @see java.lang.Object#toString()
- */
+
@Override
public String toString() {
try {
return decode(bytes, 0, length);
} catch (CharacterCodingException e) {
- throw new RuntimeException("Should not have happened " , e);
+ throw new RuntimeException("Should not have happened", e);
}
}
-
- /** deserialize
- */
+
@Override
public void readFields(DataInput in) throws IOException {
int newLength = WritableUtils.readVInt(in);
readWithKnownLength(in, newLength);
}
-
+
public void readFields(DataInput in, int maxLength) throws IOException {
int newLength = WritableUtils.readVInt(in);
if (newLength < 0) {
@@ -303,7 +310,9 @@ public void readFields(DataInput in, int maxLength) throws IOException {
readWithKnownLength(in, newLength);
}
- /** Skips over one Text in the input. */
+ /**
+ * Skips over one Text in the input.
+ */
public static void skip(DataInput in) throws IOException {
int length = WritableUtils.readVInt(in);
WritableUtils.skipFully(in, length);
@@ -315,14 +324,14 @@ public static void skip(DataInput in) throws IOException {
* format.
*/
public void readWithKnownLength(DataInput in, int len) throws IOException {
- setCapacity(len, false);
+ ensureCapacity(len);
in.readFully(bytes, 0, len);
length = len;
}
- /** serialize
- * write this object to out
- * length uses zero-compressed encoding
+ /**
+ * Serialize. Write this object to out length uses zero-compressed encoding.
+ *
* @see Writable#write(DataOutput)
*/
@Override
@@ -341,7 +350,10 @@ public void write(DataOutput out, int maxLength) throws IOException {
out.write(bytes, 0, length);
}
- /** Returns true iff o is a Text with the same contents. */
+ /**
+ * Returns true iff o is a Text with the same length and same
+ * contents.
+ */
@Override
public boolean equals(Object o) {
if (o instanceof Text)
@@ -365,7 +377,7 @@ public int compare(byte[] b1, int s1, int l1,
byte[] b2, int s2, int l2) {
int n1 = WritableUtils.decodeVIntSize(b1[s1]);
int n2 = WritableUtils.decodeVIntSize(b2[s2]);
- return compareBytes(b1, s1+n1, l1-n1, b2, s2+n2, l2-n2);
+ return compareBytes(b1, s1 + n1, l1 - n1, b2, s2 + n2, l2 - n2);
}
}
@@ -383,12 +395,12 @@ public int compare(byte[] b1, int s1, int l1,
public static String decode(byte[] utf8) throws CharacterCodingException {
return decode(ByteBuffer.wrap(utf8), true);
}
-
+
public static String decode(byte[] utf8, int start, int length)
throws CharacterCodingException {
return decode(ByteBuffer.wrap(utf8, start, length), true);
}
-
+
/**
* Converts the provided byte array to a String using the
* UTF-8 encoding. If replace is true, then
@@ -400,7 +412,7 @@ public static String decode(byte[] utf8, int start, int length, boolean replace)
throws CharacterCodingException {
return decode(ByteBuffer.wrap(utf8, start, length), replace);
}
-
+
private static String decode(ByteBuffer utf8, boolean replace)
throws CharacterCodingException {
CharsetDecoder decoder = DECODER_FACTORY.get();
@@ -463,7 +475,7 @@ public static ByteBuffer encode(String string, boolean replace)
public static String readString(DataInput in) throws IOException {
return readString(in, Integer.MAX_VALUE);
}
-
+
/** Read a UTF8 encoded string with a maximum size
*/
public static String readString(DataInput in, int maxLength)
@@ -473,8 +485,9 @@ public static String readString(DataInput in, int maxLength)
in.readFully(bytes, 0, length);
return decode(bytes);
}
-
- /** Write a UTF8 encoded string to out
+
+ /**
+ * Write a UTF8 encoded string to out.
*/
public static int writeString(DataOutput out, String s) throws IOException {
ByteBuffer bytes = encode(s);
@@ -484,7 +497,8 @@ public static int writeString(DataOutput out, String s) throws IOException {
return length;
}
- /** Write a UTF8 encoded string with a maximum size to out
+ /**
+ * Write a UTF8 encoded string with a maximum size to out.
*/
public static int writeString(DataOutput out, String s, int maxLength)
throws IOException {
@@ -501,24 +515,26 @@ public static int writeString(DataOutput out, String s, int maxLength)
}
////// states for validateUTF8
-
+
private static final int LEAD_BYTE = 0;
private static final int TRAIL_BYTE_1 = 1;
private static final int TRAIL_BYTE = 2;
- /**
- * Check if a byte array contains valid utf-8
+ /**
+ * Check if a byte array contains valid UTF-8.
+ *
* @param utf8 byte array
- * @throws MalformedInputException if the byte array contains invalid utf-8
+ * @throws MalformedInputException if the byte array contains invalid UTF-8
*/
public static void validateUTF8(byte[] utf8) throws MalformedInputException {
- validateUTF8(utf8, 0, utf8.length);
+ validateUTF8(utf8, 0, utf8.length);
}
-
+
/**
- * Check to see if a byte array is valid utf-8
+ * Check to see if a byte array is valid UTF-8.
+ *
* @param utf8 the array of bytes
* @param start the offset of the first byte in the array
* @param len the length of the byte sequence
@@ -641,7 +657,6 @@ public static int bytesToCodePoint(ByteBuffer bytes) {
return ch;
}
-
static final int offsetsFromUTF8[] =
{ 0x00000000, 0x00003080,
0x000E2080, 0x03C82080, 0xFA082080, 0x82082080 };
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/BZip2Codec.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/BZip2Codec.java
index 99590eda679af..7fd5633daa698 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/BZip2Codec.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/BZip2Codec.java
@@ -236,7 +236,7 @@ public Decompressor createDecompressor() {
*/
@Override
public String getDefaultExtension() {
- return ".bz2";
+ return CodecConstants.BZIP2_CODEC_EXTENSION;
}
private static class BZip2CompressionOutputStream extends
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CodecConstants.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CodecConstants.java
new file mode 100644
index 0000000000000..96410a18ebcb5
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/CodecConstants.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.io.compress;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * Codec related constants.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public final class CodecConstants {
+
+ private CodecConstants() {
+ }
+ /**
+ * Default extension for {@link org.apache.hadoop.io.compress.DefaultCodec}.
+ */
+ public static final String DEFAULT_CODEC_EXTENSION = ".deflate";
+
+ /**
+ * Default extension for {@link org.apache.hadoop.io.compress.BZip2Codec}.
+ */
+ public static final String BZIP2_CODEC_EXTENSION = ".bz2";
+
+ /**
+ * Default extension for {@link org.apache.hadoop.io.compress.GzipCodec}.
+ */
+ public static final String GZIP_CODEC_EXTENSION = ".gz";
+
+ /**
+ * Default extension for {@link org.apache.hadoop.io.compress.Lz4Codec}.
+ */
+ public static final String LZ4_CODEC_EXTENSION = ".lz4";
+
+ /**
+ * Default extension for
+ * {@link org.apache.hadoop.io.compress.PassthroughCodec}.
+ */
+ public static final String PASSTHROUGH_CODEC_EXTENSION = ".passthrough";
+
+ /**
+ * Default extension for {@link org.apache.hadoop.io.compress.SnappyCodec}.
+ */
+ public static final String SNAPPY_CODEC_EXTENSION = ".snappy";
+
+ /**
+ * Default extension for {@link org.apache.hadoop.io.compress.ZStandardCodec}.
+ */
+ public static final String ZSTANDARD_CODEC_EXTENSION = ".zst";
+}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/DefaultCodec.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/DefaultCodec.java
index 33f39ef9297fb..d2ffb22eaafb3 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/DefaultCodec.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/DefaultCodec.java
@@ -114,7 +114,7 @@ public DirectDecompressor createDirectDecompressor() {
@Override
public String getDefaultExtension() {
- return ".deflate";
+ return CodecConstants.DEFAULT_CODEC_EXTENSION;
}
}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/GzipCodec.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/GzipCodec.java
index 9bd861da9e890..1535e8c3d386e 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/GzipCodec.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/GzipCodec.java
@@ -206,7 +206,7 @@ public DirectDecompressor createDirectDecompressor() {
@Override
public String getDefaultExtension() {
- return ".gz";
+ return CodecConstants.GZIP_CODEC_EXTENSION;
}
static final class GzipZlibCompressor extends ZlibCompressor {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/Lz4Codec.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/Lz4Codec.java
index 45b5e9cdabd28..ba6b487150501 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/Lz4Codec.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/Lz4Codec.java
@@ -221,6 +221,6 @@ public Decompressor createDecompressor() {
*/
@Override
public String getDefaultExtension() {
- return ".lz4";
+ return CodecConstants.LZ4_CODEC_EXTENSION;
}
}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/PassthroughCodec.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/PassthroughCodec.java
index a3f0bffeebc0f..074762c0e8f7a 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/PassthroughCodec.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/PassthroughCodec.java
@@ -77,7 +77,8 @@ public class PassthroughCodec
* This default extension is here so that if no extension has been defined,
* some value is still returned: {@value}..
*/
- public static final String DEFAULT_EXTENSION = ".passthrough";
+ public static final String DEFAULT_EXTENSION =
+ CodecConstants.PASSTHROUGH_CODEC_EXTENSION;
private Configuration conf;
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/SnappyCodec.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/SnappyCodec.java
index cd0c7880376bf..686f30c9f89a2 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/SnappyCodec.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/SnappyCodec.java
@@ -225,6 +225,6 @@ public DirectDecompressor createDirectDecompressor() {
*/
@Override
public String getDefaultExtension() {
- return ".snappy";
+ return CodecConstants.SNAPPY_CODEC_EXTENSION;
}
}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/ZStandardCodec.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/ZStandardCodec.java
index c56bbba3b5959..a7afebc0c49ae 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/ZStandardCodec.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/ZStandardCodec.java
@@ -230,7 +230,7 @@ public Decompressor createDecompressor() {
*/
@Override
public String getDefaultExtension() {
- return ".zst";
+ return CodecConstants.ZSTANDARD_CODEC_EXTENSION;
}
@Override
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java
index 973afa33e3f35..160b8e029e56b 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java
@@ -224,28 +224,31 @@ public long getLength() {
* JNI wrapper of persist memory operations.
*/
public static class Pmem {
- // check whether the address is a Pmem address or DIMM address
+ // Check whether the address is a Pmem address or DIMM address
public static boolean isPmem(long address, long length) {
return NativeIO.POSIX.isPmemCheck(address, length);
}
- // create a pmem file and memory map it
- public static PmemMappedRegion mapBlock(String path, long length) {
- return NativeIO.POSIX.pmemCreateMapFile(path, length);
+ // Map a file in persistent memory, if the given file exists,
+ // directly map it. If not, create the named file on persistent memory
+ // and then map it.
+ public static PmemMappedRegion mapBlock(
+ String path, long length, boolean isFileExist) {
+ return NativeIO.POSIX.pmemMapFile(path, length, isFileExist);
}
- // unmap a pmem file
+ // Unmap a pmem file
public static boolean unmapBlock(long address, long length) {
return NativeIO.POSIX.pmemUnMap(address, length);
}
- // copy data from disk file(src) to pmem file(dest), without flush
+ // Copy data from disk file(src) to pmem file(dest), without flush
public static void memCopy(byte[] src, long dest, boolean isPmem,
long length) {
NativeIO.POSIX.pmemCopy(src, dest, isPmem, length);
}
- // flush the memory content to persistent storage
+ // Flush the memory content to persistent storage
public static void memSync(PmemMappedRegion region) {
if (region.isPmem()) {
NativeIO.POSIX.pmemDrain();
@@ -261,8 +264,8 @@ public static String getPmdkLibPath() {
private static native String getPmdkLibPath();
private static native boolean isPmemCheck(long address, long length);
- private static native PmemMappedRegion pmemCreateMapFile(String path,
- long length);
+ private static native PmemMappedRegion pmemMapFile(String path,
+ long length, boolean isFileExist);
private static native boolean pmemUnMap(long address, long length);
private static native void pmemCopy(byte[] src, long dest, boolean isPmem,
long length);
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryPolicies.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryPolicies.java
index ba07db4c2ae5e..fcbcc868cf6dd 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryPolicies.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryPolicies.java
@@ -34,6 +34,7 @@
import javax.security.sasl.SaslException;
+import org.apache.hadoop.ipc.ObserverRetryOnActiveException;
import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.ipc.RetriableException;
import org.apache.hadoop.ipc.StandbyException;
@@ -678,7 +679,7 @@ public RetryAction shouldRetry(Exception e, int retries,
e instanceof UnknownHostException ||
e instanceof StandbyException ||
e instanceof ConnectTimeoutException ||
- isWrappedStandbyException(e)) {
+ shouldFailoverOnException(e)) {
return new RetryAction(RetryAction.RetryDecision.FAILOVER_AND_RETRY,
getFailoverOrRetrySleepTime(failovers));
} else if (e instanceof RetriableException
@@ -689,7 +690,8 @@ public RetryAction shouldRetry(Exception e, int retries,
} else if (e instanceof InvalidToken) {
return new RetryAction(RetryAction.RetryDecision.FAIL, 0,
"Invalid or Cancelled Token");
- } else if (e instanceof AccessControlException) {
+ } else if (e instanceof AccessControlException ||
+ hasWrappedAccessControlException(e)) {
return new RetryAction(RetryAction.RetryDecision.FAIL, 0,
"Access denied");
} else if (e instanceof SocketException
@@ -729,12 +731,13 @@ private static long calculateExponentialTime(long time, int retries) {
return calculateExponentialTime(time, retries, Long.MAX_VALUE);
}
- private static boolean isWrappedStandbyException(Exception e) {
+ private static boolean shouldFailoverOnException(Exception e) {
if (!(e instanceof RemoteException)) {
return false;
}
Exception unwrapped = ((RemoteException)e).unwrapRemoteException(
- StandbyException.class);
+ StandbyException.class,
+ ObserverRetryOnActiveException.class);
return unwrapped instanceof StandbyException;
}
@@ -759,4 +762,13 @@ static RetriableException getWrappedRetriableException(Exception e) {
return unwrapped instanceof RetriableException ?
(RetriableException) unwrapped : null;
}
+
+ private static boolean hasWrappedAccessControlException(Exception e) {
+ Throwable throwable = e;
+ while (!(throwable instanceof AccessControlException) &&
+ throwable.getCause() != null) {
+ throwable = throwable.getCause();
+ }
+ return throwable instanceof AccessControlException;
+ }
}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryUtils.java
index 7e43974ba9cb3..c035a42d4a751 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryUtils.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryUtils.java
@@ -23,7 +23,7 @@
import org.apache.hadoop.io.retry.RetryPolicies.MultipleLinearRandomRetry;
import org.apache.hadoop.ipc.RemoteException;
-import com.google.protobuf.ServiceException;
+import org.apache.hadoop.thirdparty.protobuf.ServiceException;
import org.apache.hadoop.ipc.RetriableException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallQueueManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallQueueManager.java
index 81b7d34d0d1e0..53ac34b61272f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallQueueManager.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallQueueManager.java
@@ -22,6 +22,7 @@
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
import java.util.AbstractQueue;
+import java.util.Arrays;
import java.util.Collection;
import java.util.Iterator;
import java.util.concurrent.BlockingQueue;
@@ -77,8 +78,10 @@ public CallQueueManager(Class> backingClass,
int priorityLevels = parseNumLevels(namespace, conf);
this.scheduler = createScheduler(schedulerClass, priorityLevels,
namespace, conf);
+ int[] capacityWeights = parseCapacityWeights(priorityLevels,
+ namespace, conf);
BlockingQueue bq = createCallQueueInstance(backingClass,
- priorityLevels, maxQueueSize, namespace, conf);
+ priorityLevels, maxQueueSize, namespace, capacityWeights, conf);
this.clientBackOffEnabled = clientBackOffEnabled;
this.serverFailOverEnabled = conf.getBoolean(
namespace + "." +
@@ -146,13 +149,14 @@ private static T createScheduler(
private > T createCallQueueInstance(
Class theClass, int priorityLevels, int maxLen, String ns,
- Configuration conf) {
+ int[] capacityWeights, Configuration conf) {
// Used for custom, configurable callqueues
try {
Constructor ctor = theClass.getDeclaredConstructor(int.class,
- int.class, String.class, Configuration.class);
- return ctor.newInstance(priorityLevels, maxLen, ns, conf);
+ int.class, String.class, int[].class, Configuration.class);
+ return ctor.newInstance(priorityLevels, maxLen, ns,
+ capacityWeights, conf);
} catch (RuntimeException e) {
throw e;
} catch (InvocationTargetException e) {
@@ -343,6 +347,47 @@ private static int parseNumLevels(String ns, Configuration conf) {
return retval;
}
+ /**
+ * Read the weights of capacity in callqueue and pass the value to
+ * callqueue constructions.
+ */
+ private static int[] parseCapacityWeights(
+ int priorityLevels, String ns, Configuration conf) {
+ int[] weights = conf.getInts(ns + "." +
+ CommonConfigurationKeys.IPC_CALLQUEUE_CAPACITY_WEIGHTS_KEY);
+ if (weights.length == 0) {
+ weights = getDefaultQueueCapacityWeights(priorityLevels);
+ } else if (weights.length != priorityLevels) {
+ throw new IllegalArgumentException(
+ CommonConfigurationKeys.IPC_CALLQUEUE_CAPACITY_WEIGHTS_KEY + " must "
+ + "specify " + priorityLevels + " capacity weights: one for each "
+ + "priority level");
+ } else {
+ // only allow positive numbers
+ for (int w : weights) {
+ if (w <= 0) {
+ throw new IllegalArgumentException(
+ CommonConfigurationKeys.IPC_CALLQUEUE_CAPACITY_WEIGHTS_KEY +
+ " only takes positive weights. " + w + " capacity weight " +
+ "found");
+ }
+ }
+ }
+ return weights;
+ }
+
+ /**
+ * By default, queue capacity is the same for all priority levels.
+ *
+ * @param priorityLevels number of levels
+ * @return default weights
+ */
+ public static int[] getDefaultQueueCapacityWeights(int priorityLevels) {
+ int[] weights = new int[priorityLevels];
+ Arrays.fill(weights, 1);
+ return weights;
+ }
+
/**
* Replaces active queue with the newly requested one and transfers
* all calls to the newQ before returning.
@@ -355,8 +400,9 @@ public synchronized void swapQueue(
this.scheduler.stop();
RpcScheduler newScheduler = createScheduler(schedulerClass, priorityLevels,
ns, conf);
+ int[] capacityWeights = parseCapacityWeights(priorityLevels, ns, conf);
BlockingQueue newQ = createCallQueueInstance(queueClassToUse,
- priorityLevels, maxSize, ns, conf);
+ priorityLevels, maxSize, ns, capacityWeights, conf);
// Our current queue becomes the old queue
BlockingQueue oldQ = putRef.get();
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
index 358c0d7ac3448..688eed647c209 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
@@ -761,8 +761,17 @@ public Object run() throws IOException, InterruptedException {
throw (IOException) new IOException(msg).initCause(ex);
}
} else {
- LOG.warn("Exception encountered while connecting to "
- + "the server : " + ex);
+ // With RequestHedgingProxyProvider, one rpc call will send multiple
+ // requests to all namenodes. After one request return successfully,
+ // all other requests will be interrupted. It's not a big problem,
+ // and should not print a warning log.
+ if (ex instanceof InterruptedIOException) {
+ LOG.debug("Exception encountered while connecting to the server",
+ ex);
+ } else {
+ LOG.warn("Exception encountered while connecting to the server ",
+ ex);
+ }
}
if (ex instanceof RemoteException)
throw (RemoteException) ex;
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/DecayRpcScheduler.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/DecayRpcScheduler.java
index ffeafb5c0dc70..3e952eb63c3ff 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/DecayRpcScheduler.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/DecayRpcScheduler.java
@@ -42,6 +42,7 @@
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.ipc.metrics.DecayRpcSchedulerDetailedMetrics;
import org.apache.hadoop.metrics2.MetricsCollector;
import org.apache.hadoop.metrics2.MetricsRecordBuilder;
import org.apache.hadoop.metrics2.MetricsSource;
@@ -154,6 +155,10 @@ public class DecayRpcScheduler implements RpcScheduler,
private final AtomicDoubleArray responseTimeAvgInLastWindow;
private final AtomicLongArray responseTimeCountInLastWindow;
+ // RPC queue time rates per queue
+ private final DecayRpcSchedulerDetailedMetrics
+ decayRpcSchedulerDetailedMetrics;
+
// Pre-computed scheduling decisions during the decay sweep are
// atomically swapped in as a read-only map
private final AtomicReference> scheduleCacheRef =
@@ -236,6 +241,10 @@ public DecayRpcScheduler(int numLevels, String ns, Configuration conf) {
Preconditions.checkArgument(topUsersCount > 0,
"the number of top users for scheduler metrics must be at least 1");
+ decayRpcSchedulerDetailedMetrics =
+ DecayRpcSchedulerDetailedMetrics.create(ns);
+ decayRpcSchedulerDetailedMetrics.init(numLevels);
+
// Setup delay timer
Timer timer = new Timer(true);
DecayTask task = new DecayTask(this, timer);
@@ -626,6 +635,11 @@ public void addResponseTime(String callName, Schedulable schedulable,
long queueTime = details.get(Timing.QUEUE, TimeUnit.MILLISECONDS);
long processingTime = details.get(Timing.PROCESSING, TimeUnit.MILLISECONDS);
+ this.decayRpcSchedulerDetailedMetrics.addQueueTime(
+ priorityLevel, queueTime);
+ this.decayRpcSchedulerDetailedMetrics.addProcessingTime(
+ priorityLevel, processingTime);
+
responseTimeCountInCurrWindow.getAndIncrement(priorityLevel);
responseTimeTotalInCurrWindow.getAndAdd(priorityLevel,
queueTime+processingTime);
@@ -987,9 +1001,16 @@ private Map getDecayedCallCosts() {
return decayedCallCosts;
}
+ @VisibleForTesting
+ public DecayRpcSchedulerDetailedMetrics
+ getDecayRpcSchedulerDetailedMetrics() {
+ return decayRpcSchedulerDetailedMetrics;
+ }
+
@Override
public void stop() {
metricsProxy.unregisterSource(namespace);
MetricsProxy.removeInstance(namespace);
+ decayRpcSchedulerDetailedMetrics.shutdown();
}
}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/FairCallQueue.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/FairCallQueue.java
index d15a71000bd54..939149fcc5e57 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/FairCallQueue.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/FairCallQueue.java
@@ -80,17 +80,27 @@ private void signalNotEmpty() {
/* Failover if queue is filled up */
private boolean serverFailOverEnabled;
+
+ @VisibleForTesting
+ public FairCallQueue(int priorityLevels, int capacity, String ns,
+ Configuration conf) {
+ this(priorityLevels, capacity, ns,
+ CallQueueManager.getDefaultQueueCapacityWeights(priorityLevels), conf);
+ }
+
/**
* Create a FairCallQueue.
* @param capacity the total size of all sub-queues
* @param ns the prefix to use for configuration
+ * @param capacityWeights the weights array for capacity allocation
+ * among subqueues
* @param conf the configuration to read from
* Notes: Each sub-queue has a capacity of `capacity / numSubqueues`.
* The first or the highest priority sub-queue has an excess capacity
* of `capacity % numSubqueues`
*/
public FairCallQueue(int priorityLevels, int capacity, String ns,
- Configuration conf) {
+ int[] capacityWeights, Configuration conf) {
if(priorityLevels < 1) {
throw new IllegalArgumentException("Number of Priority Levels must be " +
"at least 1");
@@ -101,11 +111,18 @@ public FairCallQueue(int priorityLevels, int capacity, String ns,
this.queues = new ArrayList>(numQueues);
this.overflowedCalls = new ArrayList(numQueues);
- int queueCapacity = capacity / numQueues;
- int capacityForFirstQueue = queueCapacity + (capacity % numQueues);
+ int totalWeights = 0;
+ for (int i = 0; i < capacityWeights.length; i++) {
+ totalWeights += capacityWeights[i];
+ }
+ int residueCapacity = capacity % totalWeights;
+ int unitCapacity = capacity / totalWeights;
+ int queueCapacity;
for(int i=0; i < numQueues; i++) {
+ queueCapacity = unitCapacity * capacityWeights[i];
if (i == 0) {
- this.queues.add(new LinkedBlockingQueue(capacityForFirstQueue));
+ this.queues.add(new LinkedBlockingQueue(
+ queueCapacity + residueCapacity));
} else {
this.queues.add(new LinkedBlockingQueue(queueCapacity));
}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ObserverRetryOnActiveException.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ObserverRetryOnActiveException.java
index 336b304f2d084..80e2898155a29 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ObserverRetryOnActiveException.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ObserverRetryOnActiveException.java
@@ -20,8 +20,6 @@
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
-import java.io.IOException;
-
/**
* Thrown by a remote ObserverNode indicating the operation has failed and the
* client should retry active namenode directly (instead of retry other
@@ -29,7 +27,7 @@
*/
@InterfaceAudience.Private
@InterfaceStability.Evolving
-public class ObserverRetryOnActiveException extends IOException {
+public class ObserverRetryOnActiveException extends StandbyException {
static final long serialVersionUID = 1L;
public ObserverRetryOnActiveException(String msg) {
super(msg);
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufHelper.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufHelper.java
index e30f28a698a43..bb86cfc35bf4e 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufHelper.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufHelper.java
@@ -18,10 +18,16 @@
package org.apache.hadoop.ipc;
import java.io.IOException;
+import java.util.concurrent.ConcurrentHashMap;
import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.proto.SecurityProtos.TokenProto;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
-import com.google.protobuf.ServiceException;
+import org.apache.hadoop.thirdparty.protobuf.ByteString;
+import org.apache.hadoop.thirdparty.protobuf.ServiceException;
/**
* Helper methods for protobuf related RPC implementation
@@ -46,4 +52,67 @@ public static IOException getRemoteException(ServiceException se) {
}
return e instanceof IOException ? (IOException) e : new IOException(se);
}
+
+
+ /**
+ * Map used to cache fixed strings to ByteStrings. Since there is no
+ * automatic expiration policy, only use this for strings from a fixed, small
+ * set.
+ *
+ * This map should not be accessed directly. Used the getFixedByteString
+ * methods instead.
+ */
+ private final static ConcurrentHashMap
+ FIXED_BYTESTRING_CACHE = new ConcurrentHashMap<>();
+
+ /**
+ * Get the ByteString for frequently used fixed and small set strings.
+ * @param key string
+ * @return
+ */
+ public static ByteString getFixedByteString(Text key) {
+ ByteString value = FIXED_BYTESTRING_CACHE.get(key);
+ if (value == null) {
+ value = ByteString.copyFromUtf8(key.toString());
+ FIXED_BYTESTRING_CACHE.put(new Text(key.copyBytes()), value);
+ }
+ return value;
+ }
+
+ /**
+ * Get the ByteString for frequently used fixed and small set strings.
+ * @param key string
+ * @return
+ */
+ public static ByteString getFixedByteString(String key) {
+ ByteString value = FIXED_BYTESTRING_CACHE.get(key);
+ if (value == null) {
+ value = ByteString.copyFromUtf8(key);
+ FIXED_BYTESTRING_CACHE.put(key, value);
+ }
+ return value;
+ }
+
+ public static ByteString getByteString(byte[] bytes) {
+ // return singleton to reduce object allocation
+ return (bytes.length == 0) ? ByteString.EMPTY : ByteString.copyFrom(bytes);
+ }
+
+ public static Token tokenFromProto(
+ TokenProto tokenProto) {
+ Token token = new Token<>(
+ tokenProto.getIdentifier().toByteArray(),
+ tokenProto.getPassword().toByteArray(), new Text(tokenProto.getKind()),
+ new Text(tokenProto.getService()));
+ return token;
+ }
+
+ public static TokenProto protoFromToken(Token tok) {
+ TokenProto.Builder builder = TokenProto.newBuilder().
+ setIdentifier(getByteString(tok.getIdentifier())).
+ setPassword(getByteString(tok.getPassword())).
+ setKindBytes(getFixedByteString(tok.getKind())).
+ setServiceBytes(getFixedByteString(tok.getService()));
+ return builder.build();
+ }
}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java
index c6b3fded7c731..14b356f847acf 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java
@@ -19,8 +19,8 @@
package org.apache.hadoop.ipc;
import com.google.common.annotations.VisibleForTesting;
-import com.google.protobuf.*;
-import com.google.protobuf.Descriptors.MethodDescriptor;
+import org.apache.hadoop.thirdparty.protobuf.*;
+import org.apache.hadoop.thirdparty.protobuf.Descriptors.MethodDescriptor;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngineCallback.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngineCallback.java
index 463593e2abce3..50b70ca4bec1a 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngineCallback.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngineCallback.java
@@ -18,7 +18,7 @@
package org.apache.hadoop.ipc;
-import com.google.protobuf.Message;
+import org.apache.hadoop.thirdparty.protobuf.Message;
public interface ProtobufRpcEngineCallback {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtocolMetaInfoServerSideTranslatorPB.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtocolMetaInfoServerSideTranslatorPB.java
index d9d80a84d860d..a78892e7efeed 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtocolMetaInfoServerSideTranslatorPB.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtocolMetaInfoServerSideTranslatorPB.java
@@ -25,8 +25,8 @@
import org.apache.hadoop.ipc.protobuf.ProtocolInfoProtos.ProtocolSignatureProto;
import org.apache.hadoop.ipc.protobuf.ProtocolInfoProtos.ProtocolVersionProto;
-import com.google.protobuf.RpcController;
-import com.google.protobuf.ServiceException;
+import org.apache.hadoop.thirdparty.protobuf.RpcController;
+import org.apache.hadoop.thirdparty.protobuf.ServiceException;
/**
* This class serves the requests for protocol versions and signatures by
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProxyCombiner.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProxyCombiner.java
index fbafabcde4a6d..99eb487be495c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProxyCombiner.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProxyCombiner.java
@@ -17,9 +17,11 @@
*/
package org.apache.hadoop.ipc;
+import com.google.common.base.Joiner;
import java.io.Closeable;
import java.io.IOException;
import java.lang.reflect.InvocationHandler;
+import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.lang.reflect.Proxy;
@@ -74,7 +76,8 @@ public static T combine(Class combinedProxyInterface,
+ combinedProxyInterface + " do not cover method " + m);
}
- InvocationHandler handler = new CombinedProxyInvocationHandler(proxies);
+ InvocationHandler handler =
+ new CombinedProxyInvocationHandler(combinedProxyInterface, proxies);
return (T) Proxy.newProxyInstance(combinedProxyInterface.getClassLoader(),
new Class[] {combinedProxyInterface}, handler);
}
@@ -82,9 +85,12 @@ public static T combine(Class combinedProxyInterface,
private static final class CombinedProxyInvocationHandler
implements RpcInvocationHandler {
+ private final Class proxyInterface;
private final Object[] proxies;
- private CombinedProxyInvocationHandler(Object[] proxies) {
+ private CombinedProxyInvocationHandler(Class proxyInterface,
+ Object[] proxies) {
+ this.proxyInterface = proxyInterface;
this.proxies = proxies;
}
@@ -97,6 +103,8 @@ public Object invoke(Object proxy, Method method, Object[] args)
return method.invoke(underlyingProxy, args);
} catch (IllegalAccessException|IllegalArgumentException e) {
lastException = e;
+ } catch (InvocationTargetException ite) {
+ throw ite.getCause();
}
}
// This shouldn't happen since the method coverage was verified in build()
@@ -116,6 +124,12 @@ public ConnectionId getConnectionId() {
return RPC.getConnectionIdForProxy(proxies[0]);
}
+ @Override
+ public String toString() {
+ return "CombinedProxy[" + proxyInterface.getSimpleName() + "]["
+ + Joiner.on(",").join(proxies) + "]";
+ }
+
@Override
public void close() throws IOException {
MultipleIOException.Builder exceptionBuilder =
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RPC.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RPC.java
index 0be5cb5fc0fc4..4f95863b03db6 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RPC.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RPC.java
@@ -59,7 +59,7 @@
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.Time;
-import com.google.protobuf.BlockingService;
+import org.apache.hadoop.thirdparty.protobuf.BlockingService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RetryCache.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RetryCache.java
index 6f6ceb5a6caa4..4bde261eab1b0 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RetryCache.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RetryCache.java
@@ -290,7 +290,7 @@ private CacheEntry waitForCompletion(CacheEntry newEntry) {
Thread.currentThread().interrupt();
}
}
- // Previous request has failed, the expectation is is that it will be
+ // Previous request has failed, the expectation is that it will be
// retried again.
if (mapEntry.state != CacheEntry.SUCCESS) {
mapEntry.state = CacheEntry.INPROGRESS;
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcClientUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcClientUtil.java
index da1e69965a901..84ecba1d34e9c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcClientUtil.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcClientUtil.java
@@ -32,8 +32,8 @@
import org.apache.hadoop.ipc.protobuf.ProtocolInfoProtos.ProtocolSignatureProto;
import org.apache.hadoop.net.NetUtils;
-import com.google.protobuf.RpcController;
-import com.google.protobuf.ServiceException;
+import org.apache.hadoop.thirdparty.protobuf.RpcController;
+import org.apache.hadoop.thirdparty.protobuf.ServiceException;
/**
* This class maintains a cache of protocol versions and corresponding protocol
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcWritable.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcWritable.java
index a97af87bdfb01..6604bd0cc1c68 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcWritable.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcWritable.java
@@ -29,9 +29,9 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.Writable;
-import com.google.protobuf.CodedInputStream;
-import com.google.protobuf.CodedOutputStream;
-import com.google.protobuf.Message;
+import org.apache.hadoop.thirdparty.protobuf.CodedInputStream;
+import org.apache.hadoop.thirdparty.protobuf.CodedOutputStream;
+import org.apache.hadoop.thirdparty.protobuf.Message;
// note anything marked public is solely for access by SaslRpcClient
@InterfaceAudience.Private
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
index 36785e147d757..4448164f4b137 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
@@ -124,9 +124,9 @@
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.annotations.VisibleForTesting;
-import com.google.protobuf.ByteString;
-import com.google.protobuf.CodedOutputStream;
-import com.google.protobuf.Message;
+import org.apache.hadoop.thirdparty.protobuf.ByteString;
+import org.apache.hadoop.thirdparty.protobuf.CodedOutputStream;
+import org.apache.hadoop.thirdparty.protobuf.Message;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -382,20 +382,28 @@ public static InetAddress getRemoteIp() {
/**
* Returns the SASL qop for the current call, if the current call is
- * set, and the SASL negotiation is done. Otherwise return null. Note
- * that CurCall is thread local object. So in fact, different handler
- * threads will process different CurCall object.
+ * set, and the SASL negotiation is done. Otherwise return null
+ * Note this only returns established QOP for auxiliary port, and
+ * returns null for primary (non-auxiliary) port.
+ *
+ * Also note that CurCall is thread local object. So in fact, different
+ * handler threads will process different CurCall object.
*
* Also, only return for RPC calls, not supported for other protocols.
* @return the QOP of the current connection.
*/
- public static String getEstablishedQOP() {
+ public static String getAuxiliaryPortEstablishedQOP() {
Call call = CurCall.get();
- if (call == null || !(call instanceof RpcCall)) {
+ if (!(call instanceof RpcCall)) {
return null;
}
RpcCall rpcCall = (RpcCall)call;
- return rpcCall.connection.getEstablishedQOP();
+ if (rpcCall.connection.isOnAuxiliaryPort()) {
+ return rpcCall.connection.getEstablishedQOP();
+ } else {
+ // Not sending back QOP for primary port
+ return null;
+ }
}
/**
@@ -751,8 +759,8 @@ public static class Call implements Schedulable,
private volatile String detailedMetricsName = "";
final int callId; // the client's call id
final int retryCount; // the retry count of the call
- long timestampNanos; // time received when response is null
- // time served when response is not null
+ long timestampNanos; // time the call was received
+ long responseTimestampNanos; // time the call was served
private AtomicInteger responseWaitCount = new AtomicInteger(1);
final RPC.RpcKind rpcKind;
final byte[] clientId;
@@ -789,6 +797,7 @@ public Call(int id, int retryCount, Void ignore1, Void ignore2,
this.callId = id;
this.retryCount = retryCount;
this.timestampNanos = Time.monotonicNowNanos();
+ this.responseTimestampNanos = timestampNanos;
this.rpcKind = kind;
this.clientId = clientId;
this.traceScope = traceScope;
@@ -1184,7 +1193,8 @@ private class Listener extends Thread {
private boolean reuseAddr = conf.getBoolean(
CommonConfigurationKeysPublic.IPC_SERVER_REUSEADDR_KEY,
CommonConfigurationKeysPublic.IPC_SERVER_REUSEADDR_DEFAULT);
-
+ private boolean isOnAuxiliaryPort;
+
Listener(int port) throws IOException {
address = new InetSocketAddress(bindAddress, port);
// Create a new server socket and set to non blocking mode
@@ -1212,6 +1222,11 @@ private class Listener extends Thread {
acceptChannel.register(selector, SelectionKey.OP_ACCEPT);
this.setName("IPC Server listener on " + port);
this.setDaemon(true);
+ this.isOnAuxiliaryPort = false;
+ }
+
+ void setIsAuxiliary() {
+ this.isOnAuxiliaryPort = true;
}
private class Reader extends Thread {
@@ -1380,11 +1395,12 @@ void doAccept(SelectionKey key) throws InterruptedException, IOException, OutOf
channel.socket().setKeepAlive(true);
Reader reader = getReader();
- Connection c = connectionManager.register(channel, this.listenPort);
+ Connection c = connectionManager.register(channel,
+ this.listenPort, this.isOnAuxiliaryPort);
// If the connectionManager can't take it, close the connection.
if (c == null) {
if (channel.isOpen()) {
- IOUtils.cleanup(null, channel);
+ IOUtils.cleanupWithLogger(LOG, channel);
}
connectionManager.droppedConnections.getAndIncrement();
continue;
@@ -1591,7 +1607,7 @@ private void doPurge(RpcCall call, long now) {
Iterator iter = responseQueue.listIterator(0);
while (iter.hasNext()) {
call = iter.next();
- if (now > call.timestampNanos + PURGE_INTERVAL_NANOS) {
+ if (now > call.responseTimestampNanos + PURGE_INTERVAL_NANOS) {
closeConnection(call.connection);
break;
}
@@ -1655,7 +1671,7 @@ private boolean processResponse(LinkedList responseQueue,
if (inHandler) {
// set the serve time when the response has to be sent later
- call.timestampNanos = Time.monotonicNowNanos();
+ call.responseTimestampNanos = Time.monotonicNowNanos();
incPending();
try {
@@ -1778,7 +1794,7 @@ public class Connection {
private SocketChannel channel;
private ByteBuffer data;
- private ByteBuffer dataLengthBuffer;
+ private final ByteBuffer dataLengthBuffer;
private LinkedList responseQueue;
// number of outstanding rpcs
private AtomicInteger rpcCount = new AtomicInteger();
@@ -1804,6 +1820,7 @@ public class Connection {
private int serviceClass;
private boolean shouldClose = false;
private int ingressPort;
+ private boolean isOnAuxiliaryPort;
UserGroupInformation user = null;
public UserGroupInformation attemptingUser = null; // user name before auth
@@ -1816,7 +1833,7 @@ public class Connection {
private boolean useWrap = false;
public Connection(SocketChannel channel, long lastContact,
- int ingressPort) {
+ int ingressPort, boolean isOnAuxiliaryPort) {
this.channel = channel;
this.lastContact = lastContact;
this.data = null;
@@ -1829,6 +1846,7 @@ public Connection(SocketChannel channel, long lastContact,
this.socket = channel.socket();
this.addr = socket.getInetAddress();
this.ingressPort = ingressPort;
+ this.isOnAuxiliaryPort = isOnAuxiliaryPort;
if (addr == null) {
this.hostAddress = "*Unknown*";
} else {
@@ -1874,7 +1892,11 @@ public InetAddress getHostInetAddress() {
public String getEstablishedQOP() {
return establishedQOP;
}
-
+
+ public boolean isOnAuxiliaryPort() {
+ return isOnAuxiliaryPort;
+ }
+
public void setLastContact(long lastContact) {
this.lastContact = lastContact;
}
@@ -2816,16 +2838,15 @@ public void setServiceClass(int serviceClass) {
private synchronized void close() {
disposeSasl();
data = null;
- dataLengthBuffer = null;
if (!channel.isOpen())
return;
try {socket.shutdownOutput();} catch(Exception e) {
LOG.debug("Ignoring socket shutdown exception", e);
}
if (channel.isOpen()) {
- IOUtils.cleanup(null, channel);
+ IOUtils.cleanupWithLogger(LOG, channel);
}
- IOUtils.cleanup(null, socket);
+ IOUtils.cleanupWithLogger(LOG, socket);
}
}
@@ -3112,6 +3133,8 @@ public synchronized void addAuxiliaryListener(int auxiliaryPort)
"There is already a listener binding to: " + auxiliaryPort);
}
Listener newListener = new Listener(auxiliaryPort);
+ newListener.setIsAuxiliary();
+
// in the case of port = 0, the listener would be on a != 0 port.
LOG.info("Adding a server listener on port " +
newListener.getAddress().getPort());
@@ -3731,11 +3754,13 @@ Connection[] toArray() {
return connections.toArray(new Connection[0]);
}
- Connection register(SocketChannel channel, int ingressPort) {
+ Connection register(SocketChannel channel, int ingressPort,
+ boolean isOnAuxiliaryPort) {
if (isFull()) {
return null;
}
- Connection connection = new Connection(channel, Time.now(), ingressPort);
+ Connection connection = new Connection(channel, Time.now(),
+ ingressPort, isOnAuxiliaryPort);
add(connection);
if (LOG.isDebugEnabled()) {
LOG.debug("Server connection from " + connection +
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/metrics/DecayRpcSchedulerDetailedMetrics.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/metrics/DecayRpcSchedulerDetailedMetrics.java
new file mode 100644
index 0000000000000..04a6c0eab1c42
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/metrics/DecayRpcSchedulerDetailedMetrics.java
@@ -0,0 +1,135 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ipc.metrics;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.metrics2.annotation.Metric;
+import org.apache.hadoop.metrics2.annotation.Metrics;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.hadoop.metrics2.lib.MetricsRegistry;
+import org.apache.hadoop.metrics2.lib.MutableRatesWithAggregation;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.annotations.VisibleForTesting;
+
+/**
+ * This class is for maintaining queue (priority) level related
+ * statistics when FairCallQueue is used and publishing them
+ * through the metrics interface.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+@Metrics(about="Per queue(priority) metrics",
+ context="decayrpcschedulerdetailed")
+public class DecayRpcSchedulerDetailedMetrics {
+
+ @Metric private MutableRatesWithAggregation rpcQueueRates;
+ @Metric private MutableRatesWithAggregation rpcProcessingRates;
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(DecayRpcSchedulerDetailedMetrics.class);
+ private final MetricsRegistry registry;
+ private final String name;
+ private String[] queueNamesForLevels;
+ private String[] processingNamesForLevels;
+
+ DecayRpcSchedulerDetailedMetrics(String ns) {
+ name = "DecayRpcSchedulerDetailedMetrics."+ ns;
+ registry = new MetricsRegistry("decayrpcschedulerdetailed")
+ .tag("port", "RPC port", String.valueOf(ns));
+ LOG.debug(registry.info().toString());
+ }
+
+ public static DecayRpcSchedulerDetailedMetrics create(String ns) {
+ DecayRpcSchedulerDetailedMetrics m =
+ new DecayRpcSchedulerDetailedMetrics(ns);
+ return DefaultMetricsSystem.instance().register(m.name, null, m);
+ }
+
+ /**
+ * Initialize the metrics for JMX with priority levels.
+ */
+ public void init(int numLevels) {
+ LOG.info("Initializing RPC stats for {} priority levels", numLevels);
+ queueNamesForLevels = new String[numLevels];
+ processingNamesForLevels = new String[numLevels];
+ for (int i = 0; i < numLevels; i++) {
+ queueNamesForLevels[i] = getQueueName(i+1);
+ processingNamesForLevels[i] = getProcessingName(i+1);
+ }
+ rpcQueueRates.init(queueNamesForLevels);
+ rpcProcessingRates.init(processingNamesForLevels);
+ }
+
+ /**
+ * Instrument a Call queue time based on its priority.
+ *
+ * @param priority of the RPC call
+ * @param queueTime of the RPC call in the queue of the priority
+ */
+ public void addQueueTime(int priority, long queueTime) {
+ rpcQueueRates.add(queueNamesForLevels[priority], queueTime);
+ }
+
+ /**
+ * Instrument a Call processing time based on its priority.
+ *
+ * @param priority of the RPC call
+ * @param processingTime of the RPC call in the queue of the priority
+ */
+ public void addProcessingTime(int priority, long processingTime) {
+ rpcProcessingRates.add(processingNamesForLevels[priority], processingTime);
+ }
+
+ /**
+ * Shutdown the instrumentation process.
+ */
+ public void shutdown() {
+ DefaultMetricsSystem.instance().unregisterSource(name);
+ }
+
+ /**
+ * Returns the rate name inside the metric.
+ */
+ public String getQueueName(int priority) {
+ return "DecayRPCSchedulerPriority."+priority+".RpcQueueTime";
+ }
+
+ /**
+ * Returns the rate name inside the metric.
+ */
+ public String getProcessingName(int priority) {
+ return "DecayRPCSchedulerPriority."+priority+".RpcProcessingTime";
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ @VisibleForTesting
+ MutableRatesWithAggregation getRpcQueueRates() {
+ return rpcQueueRates;
+ }
+
+ @VisibleForTesting
+ MutableRatesWithAggregation getRpcProcessingRates() {
+ return rpcProcessingRates;
+ }
+}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/protocolPB/GenericRefreshProtocolClientSideTranslatorPB.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/protocolPB/GenericRefreshProtocolClientSideTranslatorPB.java
index 078b2dbd5df7e..f8a3e25867f91 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/protocolPB/GenericRefreshProtocolClientSideTranslatorPB.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/protocolPB/GenericRefreshProtocolClientSideTranslatorPB.java
@@ -35,8 +35,8 @@
import org.apache.hadoop.ipc.proto.GenericRefreshProtocolProtos.GenericRefreshResponseProto;
import org.apache.hadoop.ipc.proto.GenericRefreshProtocolProtos.GenericRefreshResponseCollectionProto;
-import com.google.protobuf.RpcController;
-import com.google.protobuf.ServiceException;
+import org.apache.hadoop.thirdparty.protobuf.RpcController;
+import org.apache.hadoop.thirdparty.protobuf.ServiceException;
public class GenericRefreshProtocolClientSideTranslatorPB implements
ProtocolMetaInterface, GenericRefreshProtocol, Closeable {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/protocolPB/GenericRefreshProtocolServerSideTranslatorPB.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/protocolPB/GenericRefreshProtocolServerSideTranslatorPB.java
index ae57cbd367c38..4af72fe04e1b2 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/protocolPB/GenericRefreshProtocolServerSideTranslatorPB.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/protocolPB/GenericRefreshProtocolServerSideTranslatorPB.java
@@ -28,8 +28,8 @@
import org.apache.hadoop.ipc.proto.GenericRefreshProtocolProtos.GenericRefreshResponseProto;
import org.apache.hadoop.ipc.proto.GenericRefreshProtocolProtos.GenericRefreshResponseCollectionProto;
-import com.google.protobuf.RpcController;
-import com.google.protobuf.ServiceException;
+import org.apache.hadoop.thirdparty.protobuf.RpcController;
+import org.apache.hadoop.thirdparty.protobuf.ServiceException;
public class GenericRefreshProtocolServerSideTranslatorPB implements
GenericRefreshProtocolPB {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/protocolPB/RefreshCallQueueProtocolClientSideTranslatorPB.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/protocolPB/RefreshCallQueueProtocolClientSideTranslatorPB.java
index d18df529b084f..e378a93f87906 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/protocolPB/RefreshCallQueueProtocolClientSideTranslatorPB.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/protocolPB/RefreshCallQueueProtocolClientSideTranslatorPB.java
@@ -29,8 +29,8 @@
import org.apache.hadoop.ipc.proto.RefreshCallQueueProtocolProtos.RefreshCallQueueRequestProto;
import org.apache.hadoop.ipc.protocolPB.RefreshCallQueueProtocolPB;
-import com.google.protobuf.RpcController;
-import com.google.protobuf.ServiceException;
+import org.apache.hadoop.thirdparty.protobuf.RpcController;
+import org.apache.hadoop.thirdparty.protobuf.ServiceException;
public class RefreshCallQueueProtocolClientSideTranslatorPB implements
ProtocolMetaInterface, RefreshCallQueueProtocol, Closeable {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/protocolPB/RefreshCallQueueProtocolServerSideTranslatorPB.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/protocolPB/RefreshCallQueueProtocolServerSideTranslatorPB.java
index eb0301165e104..411df9f5674ce 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/protocolPB/RefreshCallQueueProtocolServerSideTranslatorPB.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/protocolPB/RefreshCallQueueProtocolServerSideTranslatorPB.java
@@ -24,8 +24,8 @@
import org.apache.hadoop.ipc.proto.RefreshCallQueueProtocolProtos.RefreshCallQueueRequestProto;
import org.apache.hadoop.ipc.proto.RefreshCallQueueProtocolProtos.RefreshCallQueueResponseProto;
-import com.google.protobuf.RpcController;
-import com.google.protobuf.ServiceException;
+import org.apache.hadoop.thirdparty.protobuf.RpcController;
+import org.apache.hadoop.thirdparty.protobuf.ServiceException;
public class RefreshCallQueueProtocolServerSideTranslatorPB implements
RefreshCallQueueProtocolPB {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MBeanInfoBuilder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MBeanInfoBuilder.java
index 528211913d606..cdd0ba4275ce2 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MBeanInfoBuilder.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/impl/MBeanInfoBuilder.java
@@ -106,7 +106,7 @@ MBeanInfo get() {
}
++curRecNo;
}
- MetricsSystemImpl.LOG.debug(attrs.toString());
+ MetricsSystemImpl.LOG.debug("{}", attrs);
MBeanAttributeInfo[] attrsArray = new MBeanAttributeInfo[attrs.size()];
return new MBeanInfo(name, description, attrs.toArray(attrsArray),
null, null, null); // no ops/ctors/notifications
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableRatesWithAggregation.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableRatesWithAggregation.java
index aa7b7596173e7..5fe0083aa5dce 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableRatesWithAggregation.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/lib/MutableRatesWithAggregation.java
@@ -79,6 +79,18 @@ public void init(Class protocol) {
}
}
+ /**
+ * Initialize the registry with all rate names passed in.
+ * This is an alternative to the above init function since this metric
+ * can be used more than just for rpc name.
+ * @param names the array of all rate names
+ */
+ public void init(String[] names) {
+ for (String name : names) {
+ addMetricIfNotExists(name);
+ }
+ }
+
/**
* Add a rate sample for a rate metric.
* @param name of the rate metric
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/source/JvmMetrics.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/source/JvmMetrics.java
index 5f9afddc57e0d..f19a2be0b4195 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/source/JvmMetrics.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/source/JvmMetrics.java
@@ -31,6 +31,8 @@
import com.google.common.base.Preconditions;
import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.log.metrics.EventCounter;
import org.apache.hadoop.metrics2.MetricsCollector;
import org.apache.hadoop.metrics2.MetricsInfo;
@@ -84,7 +86,7 @@ public synchronized void registerIfNeeded(){
final MemoryMXBean memoryMXBean = ManagementFactory.getMemoryMXBean();
final List gcBeans =
ManagementFactory.getGarbageCollectorMXBeans();
- final ThreadMXBean threadMXBean = ManagementFactory.getThreadMXBean();
+ private ThreadMXBean threadMXBean;
final String processName, sessionId;
private JvmPauseMonitor pauseMonitor = null;
final ConcurrentHashMap gcInfoCache =
@@ -92,9 +94,12 @@ public synchronized void registerIfNeeded(){
private GcTimeMonitor gcTimeMonitor = null;
@VisibleForTesting
- JvmMetrics(String processName, String sessionId) {
+ JvmMetrics(String processName, String sessionId, boolean useThreadMXBean) {
this.processName = processName;
this.sessionId = sessionId;
+ if (useThreadMXBean) {
+ this.threadMXBean = ManagementFactory.getThreadMXBean();
+ }
}
public void setPauseMonitor(final JvmPauseMonitor pauseMonitor) {
@@ -108,8 +113,15 @@ public void setGcTimeMonitor(GcTimeMonitor gcTimeMonitor) {
public static JvmMetrics create(String processName, String sessionId,
MetricsSystem ms) {
+ // Reloading conf instead of getting from outside since it's redundant in
+ // code level to update all the callers across lots of modules,
+ // this method is called at most once for components (NN/DN/RM/NM/...)
+ // so that the overall cost is not expensive.
+ boolean useThreadMXBean = new Configuration().getBoolean(
+ CommonConfigurationKeys.HADOOP_METRICS_JVM_USE_THREAD_MXBEAN,
+ CommonConfigurationKeys.HADOOP_METRICS_JVM_USE_THREAD_MXBEAN_DEFAULT);
return ms.register(JvmMetrics.name(), JvmMetrics.description(),
- new JvmMetrics(processName, sessionId));
+ new JvmMetrics(processName, sessionId, useThreadMXBean));
}
public static void reattach(MetricsSystem ms, JvmMetrics jvmMetrics) {
@@ -137,7 +149,11 @@ public void getMetrics(MetricsCollector collector, boolean all) {
.tag(SessionId, sessionId);
getMemoryUsage(rb);
getGcUsage(rb);
- getThreadUsage(rb);
+ if (threadMXBean != null) {
+ getThreadUsage(rb);
+ } else {
+ getThreadUsageFromGroup(rb);
+ }
getEventCounters(rb);
}
@@ -235,6 +251,39 @@ private void getThreadUsage(MetricsRecordBuilder rb) {
.addGauge(ThreadsTerminated, threadsTerminated);
}
+ private void getThreadUsageFromGroup(MetricsRecordBuilder rb) {
+ int threadsNew = 0;
+ int threadsRunnable = 0;
+ int threadsBlocked = 0;
+ int threadsWaiting = 0;
+ int threadsTimedWaiting = 0;
+ int threadsTerminated = 0;
+ ThreadGroup threadGroup = Thread.currentThread().getThreadGroup();
+ Thread[] threads = new Thread[threadGroup.activeCount()];
+ threadGroup.enumerate(threads);
+ for (Thread thread : threads) {
+ if (thread == null) {
+ // race protection
+ continue;
+ }
+ switch (thread.getState()) {
+ case NEW: threadsNew++; break;
+ case RUNNABLE: threadsRunnable++; break;
+ case BLOCKED: threadsBlocked++; break;
+ case WAITING: threadsWaiting++; break;
+ case TIMED_WAITING: threadsTimedWaiting++; break;
+ case TERMINATED: threadsTerminated++; break;
+ default:
+ }
+ }
+ rb.addGauge(ThreadsNew, threadsNew)
+ .addGauge(ThreadsRunnable, threadsRunnable)
+ .addGauge(ThreadsBlocked, threadsBlocked)
+ .addGauge(ThreadsWaiting, threadsWaiting)
+ .addGauge(ThreadsTimedWaiting, threadsTimedWaiting)
+ .addGauge(ThreadsTerminated, threadsTerminated);
+ }
+
private void getEventCounters(MetricsRecordBuilder rb) {
rb.addCounter(LogFatal, EventCounter.getFatal())
.addCounter(LogError, EventCounter.getError())
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/DNS.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/DNS.java
index 061971cb6817f..531ad80f41722 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/DNS.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/DNS.java
@@ -19,7 +19,6 @@
package org.apache.hadoop.net;
import com.google.common.net.InetAddresses;
-import com.sun.istack.Nullable;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.slf4j.Logger;
@@ -36,6 +35,7 @@
import java.util.List;
import java.util.Vector;
+import javax.annotation.Nullable;
import javax.naming.NamingException;
import javax.naming.directory.Attributes;
import javax.naming.directory.DirContext;
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java
index d98254cb1ca25..c5a5b111b2b3d 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java
@@ -37,6 +37,7 @@
import java.net.UnknownHostException;
import java.net.ConnectException;
import java.nio.channels.SocketChannel;
+import java.nio.channels.UnresolvedAddressException;
import java.util.Map.Entry;
import java.util.regex.Pattern;
import java.util.*;
@@ -534,6 +535,8 @@ public static void connect(Socket socket,
}
} catch (SocketTimeoutException ste) {
throw new ConnectTimeoutException(ste.getMessage());
+ } catch (UnresolvedAddressException uae) {
+ throw new UnknownHostException(uae.getMessage());
}
// There is a very rare case allowed by the TCP specification, such that
@@ -638,6 +641,22 @@ public static String getHostNameOfIP(String ipPort) {
}
}
+ /**
+ * Attempt to normalize the given string to "host:port"
+ * if it like "ip:port".
+ *
+ * @param ipPort maybe lik ip:port or host:port.
+ * @return host:port
+ */
+ public static String normalizeIP2HostName(String ipPort) {
+ if (null == ipPort || !ipPortPattern.matcher(ipPort).matches()) {
+ return ipPort;
+ }
+
+ InetSocketAddress address = createSocketAddr(ipPort);
+ return getHostPortString(address);
+ }
+
/**
* Return hostname without throwing exception.
* The returned hostname String format is "hostname".
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java
index 724cec37ba503..9f52fed9678b9 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java
@@ -31,6 +31,7 @@
import java.util.*;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
+import java.util.function.Consumer;
/** The class represents a cluster of computer with a tree hierarchical
* network topology.
@@ -486,10 +487,10 @@ public Node chooseRandom(final String scope,
protected Node chooseRandom(final String scope, String excludedScope,
final Collection excludedNodes) {
if (excludedScope != null) {
- if (scope.startsWith(excludedScope)) {
+ if (isChildScope(scope, excludedScope)) {
return null;
}
- if (!excludedScope.startsWith(scope)) {
+ if (!isChildScope(excludedScope, scope)) {
excludedScope = null;
}
}
@@ -667,9 +668,12 @@ public int countNumOfAvailableNodes(String scope,
if (node == null) {
continue;
}
- if ((NodeBase.getPath(node) + NodeBase.PATH_SEPARATOR_STR)
- .startsWith(scope + NodeBase.PATH_SEPARATOR_STR)) {
- excludedCountInScope++;
+ if (isNodeInScope(node, scope)) {
+ if (node instanceof InnerNode) {
+ excludedCountInScope += ((InnerNode) node).getNumOfLeaves();
+ } else {
+ excludedCountInScope++;
+ }
} else {
excludedCountOffScope++;
}
@@ -874,11 +878,33 @@ public void sortByDistance(Node reader, Node[] nodes, int activeLen) {
* This method is called if the reader is a datanode,
* so nonDataNodeReader flag is set to false.
*/
- sortByDistance(reader, nodes, activeLen, false);
+ sortByDistance(reader, nodes, activeLen, list -> Collections.shuffle(list));
}
/**
- * Sort nodes array by network distance to reader.
+ * Sort nodes array by network distance to reader with secondary sort.
+ *
+ * In a three-level topology, a node can be either local, on the same rack,
+ * or on a different rack from the reader. Sorting the nodes based on network
+ * distance from the reader reduces network traffic and improves
+ * performance.
+ *
+ * As an additional twist, we also randomize the nodes at each network
+ * distance. This helps with load balancing when there is data skew.
+ *
+ * @param reader Node where data will be read
+ * @param nodes Available replicas with the requested data
+ * @param activeLen Number of active nodes at the front of the array
+ * @param secondarySort a secondary sorting strategy which can inject into
+ * that point from outside to help sort the same distance.
+ */
+ public void sortByDistance(Node reader, T[] nodes,
+ int activeLen, Consumer> secondarySort){
+ sortByDistance(reader, nodes, activeLen, secondarySort, false);
+ }
+
+ /**
+ * Sort nodes array by network distance to reader with secondary sort.
* using network location. This is used when the reader
* is not a datanode. Sorting the nodes based on network distance
* from the reader reduces network traffic and improves
@@ -895,7 +921,27 @@ public void sortByDistanceUsingNetworkLocation(Node reader, Node[] nodes,
* This method is called if the reader is not a datanode,
* so nonDataNodeReader flag is set to true.
*/
- sortByDistance(reader, nodes, activeLen, true);
+ sortByDistanceUsingNetworkLocation(reader, nodes, activeLen,
+ list -> Collections.shuffle(list));
+ }
+
+ /**
+ * Sort nodes array by network distance to reader.
+ *
using network location. This is used when the reader
+ * is not a datanode. Sorting the nodes based on network distance
+ * from the reader reduces network traffic and improves
+ * performance.
+ *
+ *
+ * @param reader Node where data will be read
+ * @param nodes Available replicas with the requested data
+ * @param activeLen Number of active nodes at the front of the array
+ * @param secondarySort a secondary sorting strategy which can inject into
+ * that point from outside to help sort the same distance.
+ */
+ public void sortByDistanceUsingNetworkLocation(Node reader,
+ T[] nodes, int activeLen, Consumer> secondarySort) {
+ sortByDistance(reader, nodes, activeLen, secondarySort, true);
}
/**
@@ -903,13 +949,15 @@ public void sortByDistanceUsingNetworkLocation(Node reader, Node[] nodes,
*
* As an additional twist, we also randomize the nodes at each network
* distance. This helps with load balancing when there is data skew.
+ * And it helps choose node with more fast storage type.
*
* @param reader Node where data will be read
* @param nodes Available replicas with the requested data
* @param activeLen Number of active nodes at the front of the array
* @param nonDataNodeReader True if the reader is not a datanode
*/
- private void sortByDistance(Node reader, Node[] nodes, int activeLen,
+ private void sortByDistance(Node reader, T[] nodes,
+ int activeLen, Consumer> secondarySort,
boolean nonDataNodeReader) {
/** Sort weights for the nodes array */
int[] weights = new int[activeLen];
@@ -921,23 +969,26 @@ private void sortByDistance(Node reader, Node[] nodes, int activeLen,
}
}
// Add weight/node pairs to a TreeMap to sort
- TreeMap> tree = new TreeMap>();
+ TreeMap> tree = new TreeMap<>();
for (int i=0; i list = tree.get(weight);
+ T node = nodes[i];
+ List list = tree.get(weight);
if (list == null) {
list = Lists.newArrayListWithExpectedSize(1);
tree.put(weight, list);
}
list.add(node);
}
-
+ // Sort nodes which have the same weight using secondarySort.
int idx = 0;
- for (List list: tree.values()) {
+ for (List list: tree.values()) {
if (list != null) {
Collections.shuffle(list, r);
- for (Node n: list) {
+ if (secondarySort != null) {
+ secondarySort.accept(list);
+ }
+ for (T n: list) {
nodes[idx] = n;
idx++;
}
@@ -946,4 +997,33 @@ private void sortByDistance(Node reader, Node[] nodes, int activeLen,
Preconditions.checkState(idx == activeLen,
"Sorted the wrong number of nodes!");
}
-}
+
+ /**
+ * Checks whether one scope is contained in the other scope.
+ * @param parentScope the parent scope to check
+ * @param childScope the child scope which needs to be checked.
+ * @return true if childScope is contained within the parentScope
+ */
+ protected static boolean isChildScope(final String parentScope,
+ final String childScope) {
+ String pScope = parentScope.endsWith(NodeBase.PATH_SEPARATOR_STR) ?
+ parentScope : parentScope + NodeBase.PATH_SEPARATOR_STR;
+ String cScope = childScope.endsWith(NodeBase.PATH_SEPARATOR_STR) ?
+ childScope : childScope + NodeBase.PATH_SEPARATOR_STR;
+ return pScope.startsWith(cScope);
+ }
+
+ /**
+ * Checks whether a node belongs to the scope.
+ * @param node the node to check.
+ * @param scope scope to check.
+ * @return true if node lies within the scope
+ */
+ protected static boolean isNodeInScope(Node node, String scope) {
+ if (!scope.endsWith(NodeBase.PATH_SEPARATOR_STR)) {
+ scope += NodeBase.PATH_SEPARATOR_STR;
+ }
+ String nodeLocation = NodeBase.getPath(node) + NodeBase.PATH_SEPARATOR_STR;
+ return nodeLocation.startsWith(scope);
+ }
+}
\ No newline at end of file
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NodeBase.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NodeBase.java
index 9da9ca2948f9e..cc14df8519c96 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NodeBase.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NodeBase.java
@@ -20,6 +20,8 @@
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import java.util.regex.Pattern;
+
/** A base class that implements interface Node
*
*/
@@ -38,6 +40,7 @@ public class NodeBase implements Node {
protected String location; //string representation of this node's location
protected int level; //which level of the tree the node resides
protected Node parent; //its parent
+ private static final Pattern SLASHES = Pattern.compile("/+");
/** Default constructor */
public NodeBase() {
@@ -160,12 +163,15 @@ public static String normalize(String path) {
if (path.length() == 0) {
return ROOT;
}
-
+
if (path.charAt(0) != PATH_SEPARATOR) {
throw new IllegalArgumentException(
"Network Location path does not start with "
+PATH_SEPARATOR_STR+ ": "+path);
}
+
+ // Remove duplicated slashes.
+ path = SLASHES.matcher(path).replaceAll("/");
int len = path.length();
if (path.charAt(len-1) == PATH_SEPARATOR) {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/SocketIOWithTimeout.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/SocketIOWithTimeout.java
index f489581843f6c..312a481f25a86 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/SocketIOWithTimeout.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/SocketIOWithTimeout.java
@@ -326,34 +326,36 @@ int select(SelectableChannel channel, int ops, long timeout)
SelectionKey key = null;
int ret = 0;
+ long timeoutLeft = timeout;
try {
while (true) {
long start = (timeout == 0) ? 0 : Time.now();
key = channel.register(info.selector, ops);
- ret = info.selector.select(timeout);
+ ret = info.selector.select(timeoutLeft);
if (ret != 0) {
return ret;
}
- if (Thread.currentThread().isInterrupted()) {
- throw new InterruptedIOException("Interrupted while waiting for "
- + "IO on channel " + channel + ". " + timeout
- + " millis timeout left.");
- }
-
/* Sometimes select() returns 0 much before timeout for
* unknown reasons. So select again if required.
*/
if (timeout > 0) {
- timeout -= Time.now() - start;
- if (timeout <= 0) {
- return 0;
- }
+ timeoutLeft -= Time.now() - start;
+ timeoutLeft = Math.max(0, timeoutLeft);
}
+ if (Thread.currentThread().isInterrupted()) {
+ throw new InterruptedIOException("Interrupted while waiting for "
+ + "IO on channel " + channel + ". Total timeout mills is "
+ + timeout + ", " + timeoutLeft + " millis timeout left.");
+ }
+
+ if (timeoutLeft == 0) {
+ return 0;
+ }
}
} finally {
if (key != null) {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Credentials.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Credentials.java
index 37cf021d41c51..e71bf6d40dd21 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Credentials.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Credentials.java
@@ -18,7 +18,7 @@
package org.apache.hadoop.security;
-import com.google.protobuf.ByteString;
+import org.apache.hadoop.thirdparty.protobuf.ByteString;
import java.io.BufferedInputStream;
import java.io.DataInput;
@@ -46,6 +46,7 @@
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableUtils;
+import org.apache.hadoop.ipc.ProtobufHelper;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hadoop.security.proto.SecurityProtos.CredentialsKVProto;
@@ -368,7 +369,7 @@ void writeProto(DataOutput out) throws IOException {
CredentialsKVProto.Builder kv = CredentialsKVProto.newBuilder().
setAliasBytes(ByteString.copyFrom(
e.getKey().getBytes(), 0, e.getKey().getLength())).
- setToken(e.getValue().toTokenProto());
+ setToken(ProtobufHelper.protoFromToken(e.getValue()));
storage.addTokens(kv.build());
}
@@ -390,7 +391,7 @@ void readProto(DataInput in) throws IOException {
CredentialsProto storage = CredentialsProto.parseDelimitedFrom((DataInputStream)in);
for (CredentialsKVProto kv : storage.getTokensList()) {
addToken(new Text(kv.getAliasBytes().toByteArray()),
- (Token) new Token(kv.getToken()));
+ ProtobufHelper.tokenFromProto(kv.getToken()));
}
for (CredentialsKVProto kv : storage.getSecretsList()) {
addSecretKey(new Text(kv.getAliasBytes().toByteArray()),
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcClient.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcClient.java
index a63ad4fdbbb21..215f473b9fcc9 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcClient.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcClient.java
@@ -72,7 +72,7 @@
import org.apache.hadoop.util.ProtoUtil;
import com.google.common.annotations.VisibleForTesting;
-import com.google.protobuf.ByteString;
+import org.apache.hadoop.thirdparty.protobuf.ByteString;
import com.google.re2j.Pattern;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
index 91c51f2a32f39..8c84a8d31a063 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
@@ -189,21 +189,15 @@ private T getCanonicalUser(Class cls) {
@Override
public boolean commit() throws LoginException {
- if (LOG.isDebugEnabled()) {
- LOG.debug("hadoop login commit");
- }
+ LOG.debug("hadoop login commit");
// if we already have a user, we are done.
if (!subject.getPrincipals(User.class).isEmpty()) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("using existing subject:"+subject.getPrincipals());
- }
+ LOG.debug("Using existing subject: {}", subject.getPrincipals());
return true;
}
Principal user = getCanonicalUser(KerberosPrincipal.class);
if (user != null) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("using kerberos user:"+user);
- }
+ LOG.debug("Using kerberos user: {}", user);
}
//If we don't have a kerberos user and security is disabled, check
//if user is specified in the environment or properties
@@ -217,15 +211,11 @@ public boolean commit() throws LoginException {
// use the OS user
if (user == null) {
user = getCanonicalUser(OS_PRINCIPAL_CLASS);
- if (LOG.isDebugEnabled()) {
- LOG.debug("using local user:"+user);
- }
+ LOG.debug("Using local user: {}", user);
}
// if we found the user, add our principal
if (user != null) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Using user: \"" + user + "\" with name " + user.getName());
- }
+ LOG.debug("Using user: \"{}\" with name: {}", user, user.getName());
User userEntry = null;
try {
@@ -237,15 +227,12 @@ public boolean commit() throws LoginException {
} catch (Exception e) {
throw (LoginException)(new LoginException(e.toString()).initCause(e));
}
- if (LOG.isDebugEnabled()) {
- LOG.debug("User entry: \"" + userEntry.toString() + "\"" );
- }
+ LOG.debug("User entry: \"{}\"", userEntry);
subject.getPrincipals().add(userEntry);
return true;
}
- LOG.error("Can't find user in " + subject);
- throw new LoginException("Can't find user name");
+ throw new LoginException("Failed to find user in name " + subject);
}
@Override
@@ -256,17 +243,13 @@ public void initialize(Subject subject, CallbackHandler callbackHandler,
@Override
public boolean login() throws LoginException {
- if (LOG.isDebugEnabled()) {
- LOG.debug("hadoop login");
- }
+ LOG.debug("Hadoop login");
return true;
}
@Override
public boolean logout() throws LoginException {
- if (LOG.isDebugEnabled()) {
- LOG.debug("hadoop logout");
- }
+ LOG.debug("Hadoop logout");
return true;
}
}
@@ -444,23 +427,12 @@ static Optional getKerberosLoginRenewalExecutor() {
private static final boolean windows =
System.getProperty("os.name").startsWith("Windows");
- private static final boolean is64Bit =
- System.getProperty("os.arch").contains("64") ||
- System.getProperty("os.arch").contains("s390x");
- private static final boolean aix = System.getProperty("os.name").equals("AIX");
/* Return the OS login module class name */
+ /* For IBM JDK, use the common OS login module class name for all platforms */
private static String getOSLoginModuleName() {
if (IBM_JAVA) {
- if (windows) {
- return is64Bit ? "com.ibm.security.auth.module.Win64LoginModule"
- : "com.ibm.security.auth.module.NTLoginModule";
- } else if (aix) {
- return is64Bit ? "com.ibm.security.auth.module.AIX64LoginModule"
- : "com.ibm.security.auth.module.AIXLoginModule";
- } else {
- return "com.ibm.security.auth.module.LinuxLoginModule";
- }
+ return "com.ibm.security.auth.module.JAASLoginModule";
} else {
return windows ? "com.sun.security.auth.module.NTLoginModule"
: "com.sun.security.auth.module.UnixLoginModule";
@@ -468,23 +440,14 @@ private static String getOSLoginModuleName() {
}
/* Return the OS principal class */
+ /* For IBM JDK, use the common OS principal class for all platforms */
@SuppressWarnings("unchecked")
private static Class getOsPrincipalClass() {
ClassLoader cl = ClassLoader.getSystemClassLoader();
try {
String principalClass = null;
if (IBM_JAVA) {
- if (is64Bit) {
- principalClass = "com.ibm.security.auth.UsernamePrincipal";
- } else {
- if (windows) {
- principalClass = "com.ibm.security.auth.NTUserPrincipal";
- } else if (aix) {
- principalClass = "com.ibm.security.auth.AIXPrincipal";
- } else {
- principalClass = "com.ibm.security.auth.LinuxPrincipal";
- }
- }
+ principalClass = "com.ibm.security.auth.UsernamePrincipal";
} else {
principalClass = windows ? "com.sun.security.auth.NTUserPrincipal"
: "com.sun.security.auth.UnixPrincipal";
@@ -601,6 +564,7 @@ public boolean hasKerberosCredentials() {
@InterfaceAudience.Public
@InterfaceStability.Evolving
public static UserGroupInformation getCurrentUser() throws IOException {
+ ensureInitialized();
AccessControlContext context = AccessController.getContext();
Subject subject = Subject.getSubject(context);
if (subject == null || subject.getPrincipals(User.class).isEmpty()) {
@@ -687,6 +651,7 @@ public static UserGroupInformation getUGIFromSubject(Subject subject)
@InterfaceAudience.Public
@InterfaceStability.Evolving
public static UserGroupInformation getLoginUser() throws IOException {
+ ensureInitialized();
UserGroupInformation loginUser = loginUserRef.get();
// a potential race condition exists only for the initial creation of
// the login user. there's no need to penalize all subsequent calls
@@ -808,12 +773,10 @@ UserGroupInformation createLoginUser(Subject subject) throws IOException {
LOG.debug("Loaded {} base64 tokens", numTokenBase64);
}
} catch (IOException ioe) {
- LOG.debug("failure to load login credentials", ioe);
+ LOG.debug("Failure to load login credentials", ioe);
throw ioe;
}
- if (LOG.isDebugEnabled()) {
- LOG.debug("UGI loginUser:"+loginUser);
- }
+ LOG.debug("UGI loginUser: {}", loginUser);
return loginUser;
}
@@ -994,10 +957,7 @@ public void run() {
do {
try {
long now = Time.now();
- if (LOG.isDebugEnabled()) {
- LOG.debug("Current time is " + now);
- LOG.debug("Next refresh is " + nextRefresh);
- }
+ LOG.debug("Current time is {}, next refresh is {}", now, nextRefresh);
if (now < nextRefresh) {
Thread.sleep(nextRefresh - now);
}
@@ -1033,18 +993,18 @@ public void run() {
try {
tgtEndTime = tgt.getEndTime().getTime();
} catch (NullPointerException npe) {
- LOG.error(String.format("NPE thrown while getting " +
- "KerberosTicket endTime. Aborting renew thread for %s.",
- getUserName()), ie);
+ LOG.error("NPE thrown while getting KerberosTicket endTime. "
+ + "Aborting renew thread for {}.", getUserName(), ie);
return;
}
- LOG.warn(String.format("Exception encountered while running the " +
- "renewal command for %s. " +
- "(TGT end time:%d, renewalFailures: %d, " +
- "renewalFailuresTotal: %d)", getUserName(), tgtEndTime,
- metrics.renewalFailures.value(),
- metrics.renewalFailuresTotal.value()), ie);
+ LOG.warn(
+ "Exception encountered while running the "
+ + "renewal command for {}. "
+ + "(TGT end time:{}, renewalFailures: {}, "
+ + "renewalFailuresTotal: {})",
+ getUserName(), tgtEndTime, metrics.renewalFailures.value(),
+ metrics.renewalFailuresTotal.value(), ie);
if (rp == null) {
// Use a dummy maxRetries to create the policy. The policy will
// only be used to get next retry time with exponential back-off.
@@ -1091,9 +1051,7 @@ final class TicketCacheRenewalRunnable
@Override
public void relogin() throws IOException {
String output = Shell.execCommand(kinitCmd, "-R");
- if (LOG.isDebugEnabled()) {
- LOG.debug("Renewed ticket. kinit output: {}", output);
- }
+ LOG.debug("Renewed ticket. kinit output: {}", output);
reloginFromTicketCache();
}
}
@@ -1193,9 +1151,7 @@ public void logoutUserFromKeytab() throws IOException {
}
try {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Initiating logout for " + getUserName());
- }
+ LOG.debug("Initiating logout for {}", getUserName());
// hadoop login context internally locks credentials.
login.logout();
} catch (LoginException le) {
@@ -1329,9 +1285,7 @@ private void unprotectedRelogin(HadoopLoginContext login) throws IOException {
// register most recent relogin attempt
user.setLastLogin(now);
try {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Initiating logout for " + getUserName());
- }
+ LOG.debug("Initiating logout for {}", getUserName());
//clear up the kerberos state. But the tokens are not cleared! As per
//the Java kerberos login module code, only the kerberos credentials
//are cleared
@@ -1340,9 +1294,7 @@ private void unprotectedRelogin(HadoopLoginContext login) throws IOException {
//have the new credentials (pass it to the LoginContext constructor)
login = newLoginContext(
login.getAppName(), login.getSubject(), login.getConfiguration());
- if (LOG.isDebugEnabled()) {
- LOG.debug("Initiating re-login for " + getUserName());
- }
+ LOG.debug("Initiating re-login for {}", getUserName());
login.login();
// this should be unnecessary. originally added due to improper locking
// of the subject during relogin.
@@ -1748,11 +1700,7 @@ public List getGroups() {
try {
return groups.getGroups(getShortUserName());
} catch (IOException ie) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Failed to get groups for user " + getShortUserName()
- + " by " + ie);
- LOG.trace("TRACE", ie);
- }
+ LOG.debug("Failed to get groups for user {}", getShortUserName(), ie);
return Collections.emptyList();
}
}
@@ -1867,7 +1815,10 @@ protected Subject getSubject() {
@InterfaceAudience.Public
@InterfaceStability.Evolving
public T doAs(PrivilegedAction action) {
- logPrivilegedAction(subject, action);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("PrivilegedAction [as: {}][action: {}]", this, action,
+ new Exception());
+ }
return Subject.doAs(subject, action);
}
@@ -1887,13 +1838,14 @@ public T doAs(PrivilegedAction action) {
public T doAs(PrivilegedExceptionAction action
) throws IOException, InterruptedException {
try {
- logPrivilegedAction(subject, action);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("PrivilegedAction [as: {}][action: {}]", this, action,
+ new Exception());
+ }
return Subject.doAs(subject, action);
} catch (PrivilegedActionException pae) {
Throwable cause = pae.getCause();
- if (LOG.isDebugEnabled()) {
- LOG.debug("PrivilegedActionException as:" + this + " cause:" + cause);
- }
+ LOG.debug("PrivilegedActionException as: {}", this, cause);
if (cause == null) {
throw new RuntimeException("PrivilegedActionException with no " +
"underlying cause. UGI [" + this + "]" +": " + pae, pae);
@@ -1911,14 +1863,6 @@ public T doAs(PrivilegedExceptionAction action
}
}
- private void logPrivilegedAction(Subject subject, Object action) {
- if (LOG.isDebugEnabled()) {
- // would be nice if action included a descriptive toString()
- String where = new Throwable().getStackTrace()[2].toString();
- LOG.debug("PrivilegedAction as:"+this+" from:"+where);
- }
- }
-
/**
* Log current UGI and token information into specified log.
* @param ugi - UGI
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/protocolPB/RefreshAuthorizationPolicyProtocolClientSideTranslatorPB.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/protocolPB/RefreshAuthorizationPolicyProtocolClientSideTranslatorPB.java
index e69f3ad6f87b3..6e7a856aecc39 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/protocolPB/RefreshAuthorizationPolicyProtocolClientSideTranslatorPB.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/protocolPB/RefreshAuthorizationPolicyProtocolClientSideTranslatorPB.java
@@ -29,8 +29,8 @@
import org.apache.hadoop.security.proto.RefreshAuthorizationPolicyProtocolProtos.RefreshServiceAclRequestProto;
import org.apache.hadoop.security.protocolPB.RefreshAuthorizationPolicyProtocolPB;
-import com.google.protobuf.RpcController;
-import com.google.protobuf.ServiceException;
+import org.apache.hadoop.thirdparty.protobuf.RpcController;
+import org.apache.hadoop.thirdparty.protobuf.ServiceException;
public class RefreshAuthorizationPolicyProtocolClientSideTranslatorPB implements
ProtocolMetaInterface, RefreshAuthorizationPolicyProtocol, Closeable {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/protocolPB/RefreshAuthorizationPolicyProtocolServerSideTranslatorPB.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/protocolPB/RefreshAuthorizationPolicyProtocolServerSideTranslatorPB.java
index 051ffcf103a11..cd5a51885a480 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/protocolPB/RefreshAuthorizationPolicyProtocolServerSideTranslatorPB.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/protocolPB/RefreshAuthorizationPolicyProtocolServerSideTranslatorPB.java
@@ -24,8 +24,8 @@
import org.apache.hadoop.security.proto.RefreshAuthorizationPolicyProtocolProtos.RefreshServiceAclRequestProto;
import org.apache.hadoop.security.proto.RefreshAuthorizationPolicyProtocolProtos.RefreshServiceAclResponseProto;
-import com.google.protobuf.RpcController;
-import com.google.protobuf.ServiceException;
+import org.apache.hadoop.thirdparty.protobuf.RpcController;
+import org.apache.hadoop.thirdparty.protobuf.ServiceException;
public class RefreshAuthorizationPolicyProtocolServerSideTranslatorPB implements
RefreshAuthorizationPolicyProtocolPB {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/protocolPB/RefreshUserMappingsProtocolClientSideTranslatorPB.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/protocolPB/RefreshUserMappingsProtocolClientSideTranslatorPB.java
index dbc333d8b121a..ac4003881ba5c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/protocolPB/RefreshUserMappingsProtocolClientSideTranslatorPB.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/protocolPB/RefreshUserMappingsProtocolClientSideTranslatorPB.java
@@ -29,8 +29,8 @@
import org.apache.hadoop.security.proto.RefreshUserMappingsProtocolProtos.RefreshSuperUserGroupsConfigurationRequestProto;
import org.apache.hadoop.security.proto.RefreshUserMappingsProtocolProtos.RefreshUserToGroupsMappingsRequestProto;
-import com.google.protobuf.RpcController;
-import com.google.protobuf.ServiceException;
+import org.apache.hadoop.thirdparty.protobuf.RpcController;
+import org.apache.hadoop.thirdparty.protobuf.ServiceException;
public class RefreshUserMappingsProtocolClientSideTranslatorPB implements
ProtocolMetaInterface, RefreshUserMappingsProtocol, Closeable {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/protocolPB/RefreshUserMappingsProtocolServerSideTranslatorPB.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/protocolPB/RefreshUserMappingsProtocolServerSideTranslatorPB.java
index f9dd649f9fd32..71881ed4898c0 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/protocolPB/RefreshUserMappingsProtocolServerSideTranslatorPB.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/protocolPB/RefreshUserMappingsProtocolServerSideTranslatorPB.java
@@ -26,8 +26,8 @@
import org.apache.hadoop.security.proto.RefreshUserMappingsProtocolProtos.RefreshUserToGroupsMappingsRequestProto;
import org.apache.hadoop.security.proto.RefreshUserMappingsProtocolProtos.RefreshUserToGroupsMappingsResponseProto;
-import com.google.protobuf.RpcController;
-import com.google.protobuf.ServiceException;
+import org.apache.hadoop.thirdparty.protobuf.RpcController;
+import org.apache.hadoop.thirdparty.protobuf.ServiceException;
public class RefreshUserMappingsProtocolServerSideTranslatorPB implements RefreshUserMappingsProtocolPB {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/DelegatingSSLSocketFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/DelegatingSSLSocketFactory.java
index ad97a99c6ddc2..ff650d6c392db 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/DelegatingSSLSocketFactory.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ssl/DelegatingSSLSocketFactory.java
@@ -21,7 +21,6 @@
import java.io.IOException;
import java.net.InetAddress;
import java.net.Socket;
-import java.net.SocketException;
import java.security.KeyManagementException;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
@@ -31,11 +30,9 @@
import javax.net.ssl.SSLSocket;
import javax.net.ssl.SSLSocketFactory;
+import com.google.common.annotations.VisibleForTesting;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.wildfly.openssl.OpenSSLProvider;
-import org.wildfly.openssl.SSL;
-
/**
* A {@link SSLSocketFactory} that can delegate to various SSL implementations.
@@ -58,6 +55,10 @@
* SSL with no modification to the list of enabled ciphers.
*
*
+ *
+ * In order to load OpenSSL, applications must ensure the wildfly-openssl
+ * artifact is on the classpath. Currently, only ABFS declares
+ * wildfly-openssl as an explicit dependency.
*/
public final class DelegatingSSLSocketFactory extends SSLSocketFactory {
@@ -106,7 +107,16 @@ public static synchronized void initializeDefaultFactory(
}
/**
- * Singletone instance of the SSLSocketFactory.
+ * For testing only: reset the socket factory.
+ */
+ @VisibleForTesting
+ public static synchronized void resetDefaultFactory() {
+ LOG.info("Resetting default SSL Socket Factory");
+ instance = null;
+ }
+
+ /**
+ * Singleton instance of the SSLSocketFactory.
*
* SSLSocketFactory must be initialized with appropriate SSLChannelMode
* using initializeDefaultFactory method.
@@ -122,9 +132,7 @@ private DelegatingSSLSocketFactory(SSLChannelMode preferredChannelMode)
throws IOException {
try {
initializeSSLContext(preferredChannelMode);
- } catch (NoSuchAlgorithmException e) {
- throw new IOException(e);
- } catch (KeyManagementException e) {
+ } catch (NoSuchAlgorithmException | KeyManagementException e) {
throw new IOException(e);
}
@@ -142,36 +150,23 @@ private DelegatingSSLSocketFactory(SSLChannelMode preferredChannelMode)
}
private void initializeSSLContext(SSLChannelMode preferredChannelMode)
- throws NoSuchAlgorithmException, KeyManagementException {
+ throws NoSuchAlgorithmException, KeyManagementException, IOException {
+ LOG.debug("Initializing SSL Context to channel mode {}",
+ preferredChannelMode);
switch (preferredChannelMode) {
case Default:
- if (!openSSLProviderRegistered) {
- OpenSSLProvider.register();
- openSSLProviderRegistered = true;
- }
try {
- java.util.logging.Logger logger = java.util.logging.Logger.getLogger(
- SSL.class.getName());
- logger.setLevel(Level.WARNING);
- ctx = SSLContext.getInstance("openssl.TLS");
- ctx.init(null, null, null);
- // Strong reference needs to be kept to logger until initialization of
- // SSLContext finished (see HADOOP-16174):
- logger.setLevel(Level.INFO);
+ bindToOpenSSLProvider();
channelMode = SSLChannelMode.OpenSSL;
- } catch (NoSuchAlgorithmException e) {
- LOG.debug("Failed to load OpenSSL. Falling back to the JSSE default.");
+ } catch (LinkageError | NoSuchAlgorithmException | RuntimeException e) {
+ LOG.debug("Failed to load OpenSSL. Falling back to the JSSE default.",
+ e);
ctx = SSLContext.getDefault();
channelMode = SSLChannelMode.Default_JSSE;
}
break;
case OpenSSL:
- if (!openSSLProviderRegistered) {
- OpenSSLProvider.register();
- openSSLProviderRegistered = true;
- }
- ctx = SSLContext.getInstance("openssl.TLS");
- ctx.init(null, null, null);
+ bindToOpenSSLProvider();
channelMode = SSLChannelMode.OpenSSL;
break;
case Default_JSSE:
@@ -183,11 +178,38 @@ private void initializeSSLContext(SSLChannelMode preferredChannelMode)
channelMode = SSLChannelMode.Default_JSSE_with_GCM;
break;
default:
- throw new NoSuchAlgorithmException("Unknown channel mode: "
+ throw new IOException("Unknown channel mode: "
+ preferredChannelMode);
}
}
+ /**
+ * Bind to the OpenSSL provider via wildfly.
+ * This MUST be the only place where wildfly classes are referenced,
+ * so ensuring that any linkage problems only surface here where they may
+ * be caught by the initialization code.
+ */
+ private void bindToOpenSSLProvider()
+ throws NoSuchAlgorithmException, KeyManagementException {
+ if (!openSSLProviderRegistered) {
+ LOG.debug("Attempting to register OpenSSL provider");
+ org.wildfly.openssl.OpenSSLProvider.register();
+ openSSLProviderRegistered = true;
+ }
+ // Strong reference needs to be kept to logger until initialization of
+ // SSLContext finished (see HADOOP-16174):
+ java.util.logging.Logger logger = java.util.logging.Logger.getLogger(
+ "org.wildfly.openssl.SSL");
+ Level originalLevel = logger.getLevel();
+ try {
+ logger.setLevel(Level.WARNING);
+ ctx = SSLContext.getInstance("openssl.TLS");
+ ctx.init(null, null, null);
+ } finally {
+ logger.setLevel(originalLevel);
+ }
+ }
+
public String getProviderName() {
return providerName;
}
@@ -202,21 +224,26 @@ public String[] getSupportedCipherSuites() {
return ciphers.clone();
}
+ /**
+ * Get the channel mode of this instance.
+ * @return a channel mode.
+ */
+ public SSLChannelMode getChannelMode() {
+ return channelMode;
+ }
+
public Socket createSocket() throws IOException {
SSLSocketFactory factory = ctx.getSocketFactory();
- SSLSocket ss = (SSLSocket) factory.createSocket();
- configureSocket(ss);
- return ss;
+ return configureSocket(factory.createSocket());
}
@Override
public Socket createSocket(Socket s, String host, int port,
boolean autoClose) throws IOException {
SSLSocketFactory factory = ctx.getSocketFactory();
- SSLSocket ss = (SSLSocket) factory.createSocket(s, host, port, autoClose);
- configureSocket(ss);
- return ss;
+ return configureSocket(
+ factory.createSocket(s, host, port, autoClose));
}
@Override
@@ -224,52 +251,41 @@ public Socket createSocket(InetAddress address, int port,
InetAddress localAddress, int localPort)
throws IOException {
SSLSocketFactory factory = ctx.getSocketFactory();
- SSLSocket ss = (SSLSocket) factory
- .createSocket(address, port, localAddress, localPort);
-
- configureSocket(ss);
- return ss;
+ return configureSocket(factory
+ .createSocket(address, port, localAddress, localPort));
}
@Override
public Socket createSocket(String host, int port, InetAddress localHost,
int localPort) throws IOException {
SSLSocketFactory factory = ctx.getSocketFactory();
- SSLSocket ss = (SSLSocket) factory
- .createSocket(host, port, localHost, localPort);
- configureSocket(ss);
-
- return ss;
+ return configureSocket(factory
+ .createSocket(host, port, localHost, localPort));
}
@Override
public Socket createSocket(InetAddress host, int port) throws IOException {
SSLSocketFactory factory = ctx.getSocketFactory();
- SSLSocket ss = (SSLSocket) factory.createSocket(host, port);
-
- configureSocket(ss);
- return ss;
+ return configureSocket(factory.createSocket(host, port));
}
@Override
public Socket createSocket(String host, int port) throws IOException {
SSLSocketFactory factory = ctx.getSocketFactory();
- SSLSocket ss = (SSLSocket) factory.createSocket(host, port);
-
- configureSocket(ss);
- return ss;
+ return configureSocket(factory.createSocket(host, port));
}
- private void configureSocket(SSLSocket ss) throws SocketException {
- ss.setEnabledCipherSuites(ciphers);
+ private Socket configureSocket(Socket socket) {
+ ((SSLSocket) socket).setEnabledCipherSuites(ciphers);
+ return socket;
}
private String[] alterCipherList(String[] defaultCiphers) {
- ArrayList preferredSuits = new ArrayList<>();
+ ArrayList preferredSuites = new ArrayList<>();
// Remove GCM mode based ciphers from the supported list.
for (int i = 0; i < defaultCiphers.length; i++) {
@@ -277,11 +293,11 @@ private String[] alterCipherList(String[] defaultCiphers) {
LOG.debug("Removed Cipher - {} from list of enabled SSLSocket ciphers",
defaultCiphers[i]);
} else {
- preferredSuits.add(defaultCiphers[i]);
+ preferredSuites.add(defaultCiphers[i]);
}
}
- ciphers = preferredSuits.toArray(new String[0]);
+ ciphers = preferredSuites.toArray(new String[0]);
return ciphers;
}
-}
\ No newline at end of file
+}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/Token.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/Token.java
index 487dd4625202e..4f0f6fc4d444a 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/Token.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/Token.java
@@ -19,7 +19,6 @@
package org.apache.hadoop.security.token;
import com.google.common.collect.Maps;
-import com.google.protobuf.ByteString;
import com.google.common.primitives.Bytes;
import org.apache.commons.codec.binary.Base64;
@@ -28,7 +27,6 @@
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.*;
-import org.apache.hadoop.security.proto.SecurityProtos.TokenProto;
import org.apache.hadoop.util.ReflectionUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -117,32 +115,6 @@ public Token copyToken() {
return new Token(this);
}
- /**
- * Construct a Token from a TokenProto.
- * @param tokenPB the TokenProto object
- */
- public Token(TokenProto tokenPB) {
- this.identifier = tokenPB.getIdentifier().toByteArray();
- this.password = tokenPB.getPassword().toByteArray();
- this.kind = new Text(tokenPB.getKindBytes().toByteArray());
- this.service = new Text(tokenPB.getServiceBytes().toByteArray());
- }
-
- /**
- * Construct a TokenProto from this Token instance.
- * @return a new TokenProto object holding copies of data in this instance
- */
- public TokenProto toTokenProto() {
- return TokenProto.newBuilder().
- setIdentifier(ByteString.copyFrom(this.getIdentifier())).
- setPassword(ByteString.copyFrom(this.getPassword())).
- setKindBytes(ByteString.copyFrom(
- this.getKind().getBytes(), 0, this.getKind().getLength())).
- setServiceBytes(ByteString.copyFrom(
- this.getService().getBytes(), 0, this.getService().getLength())).
- build();
- }
-
/**
* Get the token identifier's byte representation.
* @return the token identifier's byte representation
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/ZKDelegationTokenSecretManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/ZKDelegationTokenSecretManager.java
index f61590c28ebce..cd3b8c0c0f279 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/ZKDelegationTokenSecretManager.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/ZKDelegationTokenSecretManager.java
@@ -98,12 +98,16 @@ public abstract class ZKDelegationTokenSecretManager versionedValue = sharedCount.getVersionedValue();
- if (sharedCount.trySetCount(versionedValue, versionedValue.getValue() + 1)) {
- break;
+ if (sharedCount.trySetCount(
+ versionedValue, versionedValue.getValue() + batchSize)) {
+ return versionedValue.getValue();
}
}
}
@Override
protected int incrementDelegationTokenSeqNum() {
- try {
- incrSharedCount(delTokSeqCounter);
- } catch (InterruptedException e) {
- // The ExpirationThread is just finishing.. so dont do anything..
- LOG.debug("Thread interrupted while performing token counter increment", e);
- Thread.currentThread().interrupt();
- } catch (Exception e) {
- throw new RuntimeException("Could not increment shared counter !!", e);
+ // The secret manager will keep a local range of seq num which won't be
+ // seen by peers, so only when the range is exhausted it will ask zk for
+ // another range again
+ if (currentSeqNum >= currentMaxSeqNum) {
+ try {
+ // after a successful batch request, we can get the range starting point
+ currentSeqNum = incrSharedCount(delTokSeqCounter, seqNumBatchSize);
+ currentMaxSeqNum = currentSeqNum + seqNumBatchSize;
+ LOG.info("Fetched new range of seq num, from {} to {} ",
+ currentSeqNum+1, currentMaxSeqNum);
+ } catch (InterruptedException e) {
+ // The ExpirationThread is just finishing.. so dont do anything..
+ LOG.debug(
+ "Thread interrupted while performing token counter increment", e);
+ Thread.currentThread().interrupt();
+ } catch (Exception e) {
+ throw new RuntimeException("Could not increment shared counter !!", e);
+ }
}
- return delTokSeqCounter.getCount();
+
+ return ++currentSeqNum;
}
@Override
@@ -603,7 +631,7 @@ protected int getCurrentKeyId() {
@Override
protected int incrementCurrentKeyId() {
try {
- incrSharedCount(keyIdSeqCounter);
+ incrSharedCount(keyIdSeqCounter, 1);
} catch (InterruptedException e) {
// The ExpirationThread is just finishing.. so dont do anything..
LOG.debug("Thread interrupted while performing keyId increment", e);
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tools/protocolPB/GetUserMappingsProtocolClientSideTranslatorPB.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tools/protocolPB/GetUserMappingsProtocolClientSideTranslatorPB.java
index ed97f0526527e..b0c0fda542448 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tools/protocolPB/GetUserMappingsProtocolClientSideTranslatorPB.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tools/protocolPB/GetUserMappingsProtocolClientSideTranslatorPB.java
@@ -28,8 +28,8 @@
import org.apache.hadoop.tools.proto.GetUserMappingsProtocolProtos.GetGroupsForUserRequestProto;
import org.apache.hadoop.tools.proto.GetUserMappingsProtocolProtos.GetGroupsForUserResponseProto;
-import com.google.protobuf.RpcController;
-import com.google.protobuf.ServiceException;
+import org.apache.hadoop.thirdparty.protobuf.RpcController;
+import org.apache.hadoop.thirdparty.protobuf.ServiceException;
public class GetUserMappingsProtocolClientSideTranslatorPB implements
ProtocolMetaInterface, GetUserMappingsProtocol, Closeable {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tools/protocolPB/GetUserMappingsProtocolServerSideTranslatorPB.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tools/protocolPB/GetUserMappingsProtocolServerSideTranslatorPB.java
index 4f49418a093ba..a3a7af79b530b 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tools/protocolPB/GetUserMappingsProtocolServerSideTranslatorPB.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tools/protocolPB/GetUserMappingsProtocolServerSideTranslatorPB.java
@@ -24,8 +24,8 @@
import org.apache.hadoop.tools.proto.GetUserMappingsProtocolProtos.GetGroupsForUserRequestProto;
import org.apache.hadoop.tools.proto.GetUserMappingsProtocolProtos.GetGroupsForUserResponseProto;
-import com.google.protobuf.RpcController;
-import com.google.protobuf.ServiceException;
+import org.apache.hadoop.thirdparty.protobuf.RpcController;
+import org.apache.hadoop.thirdparty.protobuf.ServiceException;
public class GetUserMappingsProtocolServerSideTranslatorPB implements
GetUserMappingsProtocolPB {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tracing/TraceAdminProtocolServerSideTranslatorPB.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tracing/TraceAdminProtocolServerSideTranslatorPB.java
index f6a4ea33b9bb8..5b49e2e0d5ef0 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tracing/TraceAdminProtocolServerSideTranslatorPB.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tracing/TraceAdminProtocolServerSideTranslatorPB.java
@@ -32,8 +32,8 @@
import org.apache.hadoop.tracing.TraceAdminPB.ConfigPair;
import org.apache.hadoop.tracing.TraceAdminPB.RemoveSpanReceiverRequestProto;
import org.apache.hadoop.tracing.TraceAdminPB.RemoveSpanReceiverResponseProto;
-import com.google.protobuf.RpcController;
-import com.google.protobuf.ServiceException;
+import org.apache.hadoop.thirdparty.protobuf.RpcController;
+import org.apache.hadoop.thirdparty.protobuf.ServiceException;
@InterfaceAudience.Private
public class TraceAdminProtocolServerSideTranslatorPB
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tracing/TraceAdminProtocolTranslatorPB.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tracing/TraceAdminProtocolTranslatorPB.java
index edee2e2fa1367..a5cba39844c83 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tracing/TraceAdminProtocolTranslatorPB.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/tracing/TraceAdminProtocolTranslatorPB.java
@@ -33,7 +33,7 @@
import org.apache.hadoop.tracing.TraceAdminPB.RemoveSpanReceiverRequestProto;
import org.apache.hadoop.tracing.TraceAdminPB.SpanReceiverListInfo;
import org.apache.hadoop.tracing.SpanReceiverInfo.ConfigurationPair;
-import com.google.protobuf.ServiceException;
+import org.apache.hadoop.thirdparty.protobuf.ServiceException;
@InterfaceAudience.Private
public class TraceAdminProtocolTranslatorPB implements
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskChecker.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskChecker.java
index 595aeed87af17..2ee53dc595f99 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskChecker.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskChecker.java
@@ -296,7 +296,7 @@ private static void diskIoCheckWithoutNativeIo(File file)
}
file = null;
} finally {
- IOUtils.cleanup(null, fos);
+ IOUtils.cleanupWithLogger(LOG, fos);
FileUtils.deleteQuietly(file);
}
}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DurationInfo.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DurationInfo.java
index 605d060270f8e..490562bf54ece 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DurationInfo.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DurationInfo.java
@@ -18,6 +18,8 @@
package org.apache.hadoop.util;
+import java.util.function.Supplier;
+
import org.slf4j.Logger;
import org.apache.hadoop.classification.InterfaceAudience.Public;
@@ -33,7 +35,10 @@
@Unstable
public class DurationInfo extends OperationDuration
implements AutoCloseable {
- private final String text;
+
+ private final Supplier text;
+
+ private String textStr;
private final Logger log;
@@ -65,19 +70,25 @@ public DurationInfo(Logger log,
boolean logAtInfo,
String format,
Object... args) {
- this.text = String.format(format, args);
+ this.text = () -> String.format(format, args);
this.log = log;
this.logAtInfo = logAtInfo;
if (logAtInfo) {
- log.info("Starting: {}", text);
+ log.info("Starting: {}", getFormattedText());
} else {
- log.debug("Starting: {}", text);
+ if (log.isDebugEnabled()) {
+ log.debug("Starting: {}", getFormattedText());
+ }
}
}
+ private String getFormattedText() {
+ return (textStr == null) ? (textStr = text.get()) : textStr;
+ }
+
@Override
public String toString() {
- return text + ": duration " + super.toString();
+ return getFormattedText() + ": duration " + super.toString();
}
@Override
@@ -86,7 +97,9 @@ public void close() {
if (logAtInfo) {
log.info("{}", this);
} else {
- log.debug("{}", this);
+ if (log.isDebugEnabled()) {
+ log.debug("{}", this);
+ }
}
}
}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/GcTimeMonitor.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/GcTimeMonitor.java
index 0640fc01e2ffd..4247eb7050b5a 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/GcTimeMonitor.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/GcTimeMonitor.java
@@ -23,6 +23,7 @@
import java.lang.management.GarbageCollectorMXBean;
import java.lang.management.ManagementFactory;
import java.util.List;
+import java.util.concurrent.TimeUnit;
/**
* This class monitors the percentage of time the JVM is paused in GC within
@@ -46,6 +47,52 @@ public class GcTimeMonitor extends Thread {
private final GcData curData = new GcData();
private volatile boolean shouldRun = true;
+ public static class Builder {
+
+ private long observationWindowMs = TimeUnit.MINUTES.toMillis(1);
+ private long sleepIntervalMs = TimeUnit.SECONDS.toMillis(5);
+ private int maxGcTimePercentage = 100;
+ private GcTimeAlertHandler handler = null;
+
+ /**
+ * Set observation window size in milliseconds.
+ */
+ public Builder observationWindowMs(long value) {
+ this.observationWindowMs = value;
+ return this;
+ }
+
+ /**
+ * Set sleep interval in milliseconds.
+ */
+ public Builder sleepIntervalMs(long value) {
+ this.sleepIntervalMs = value;
+ return this;
+ }
+
+ /**
+ * Set the max GC time percentage that triggers the alert handler.
+ */
+ public Builder maxGcTimePercentage(int value) {
+ this.maxGcTimePercentage = value;
+ return this;
+ }
+
+ /**
+ * Set the GC alert handler.
+ */
+ public Builder gcTimeAlertHandler(GcTimeAlertHandler value) {
+ this.handler = value;
+ return this;
+ }
+
+ public GcTimeMonitor build() {
+ return new GcTimeMonitor(observationWindowMs, sleepIntervalMs,
+ maxGcTimePercentage, handler);
+ }
+ }
+
+
/**
* Create an instance of GCTimeMonitor. Once it's started, it will stay alive
* and monitor GC time percentage until shutdown() is called. If you don't
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/GenericsUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/GenericsUtil.java
index a53096a9577d2..0aba34845a676 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/GenericsUtil.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/GenericsUtil.java
@@ -25,7 +25,6 @@
import org.apache.hadoop.classification.InterfaceStability;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.slf4j.impl.Log4jLoggerAdapter;
/**
* Contains utility methods for dealing with Java Generics.
@@ -85,6 +84,11 @@ public static boolean isLog4jLogger(Class clazz) {
return false;
}
Logger log = LoggerFactory.getLogger(clazz);
- return log instanceof Log4jLoggerAdapter;
+ try {
+ Class log4jClass = Class.forName("org.slf4j.impl.Log4jLoggerAdapter");
+ return log4jClass.isInstance(log);
+ } catch (ClassNotFoundException e) {
+ return false;
+ }
}
}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/HostsFileReader.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/HostsFileReader.java
index 6f41b8c750dc9..5141740a3d23e 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/HostsFileReader.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/HostsFileReader.java
@@ -52,6 +52,8 @@ public class HostsFileReader {
.class);
private final AtomicReference current;
+ private final AtomicReference lazyLoaded =
+ new AtomicReference<>();
public HostsFileReader(String inFile,
String exFile) throws IOException {
@@ -187,7 +189,18 @@ static String readFirstTagValue(Element e, String tag) {
public void refresh(String includesFile, String excludesFile)
throws IOException {
- LOG.info("Refreshing hosts (include/exclude) list");
+ refreshInternal(includesFile, excludesFile, false);
+ }
+
+ public void lazyRefresh(String includesFile, String excludesFile)
+ throws IOException {
+ refreshInternal(includesFile, excludesFile, true);
+ }
+
+ private void refreshInternal(String includesFile, String excludesFile,
+ boolean lazy) throws IOException {
+ LOG.info("Refreshing hosts (include/exclude) list (lazy refresh = {})",
+ lazy);
HostDetails oldDetails = current.get();
Set newIncludes = oldDetails.includes;
Map newExcludes = oldDetails.excludes;
@@ -203,7 +216,21 @@ public void refresh(String includesFile, String excludesFile)
}
HostDetails newDetails = new HostDetails(includesFile, newIncludes,
excludesFile, newExcludes);
- current.set(newDetails);
+
+ if (lazy) {
+ lazyLoaded.set(newDetails);
+ } else {
+ current.set(newDetails);
+ }
+ }
+
+ public void finishRefresh() {
+ if (lazyLoaded.get() == null) {
+ throw new IllegalStateException(
+ "Cannot finish refresh - call lazyRefresh() first");
+ }
+ current.set(lazyLoaded.get());
+ lazyLoaded.set(null);
}
@Private
@@ -279,6 +306,10 @@ public HostDetails getHostDetails() {
return current.get();
}
+ public HostDetails getLazyLoadedHostDetails() {
+ return lazyLoaded.get();
+ }
+
public void setIncludesFile(String includesFile) {
LOG.info("Setting the includes file to " + includesFile);
HostDetails oldDetails = current.get();
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/InstrumentedLock.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/InstrumentedLock.java
index 2c1f591a1f2a9..cc0ebdf8b3e39 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/InstrumentedLock.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/InstrumentedLock.java
@@ -55,8 +55,10 @@ public class InstrumentedLock implements Lock {
// Tracking counters for lock statistics.
private volatile long lockAcquireTimestamp;
- private final AtomicLong lastLogTimestamp;
- private final AtomicLong warningsSuppressed = new AtomicLong(0);
+ private final AtomicLong lastHoldLogTimestamp;
+ private final AtomicLong lastWaitLogTimestamp;
+ private final SuppressedStats holdStats = new SuppressedStats();
+ private final SuppressedStats waitStats = new SuppressedStats();
/**
* Create a instrumented lock instance which logs a warning message
@@ -91,19 +93,24 @@ public InstrumentedLock(String name, Logger logger, Lock lock,
this.logger = logger;
minLoggingGap = minLoggingGapMs;
lockWarningThreshold = lockWarningThresholdMs;
- lastLogTimestamp = new AtomicLong(
+ lastHoldLogTimestamp = new AtomicLong(
clock.monotonicNow() - Math.max(minLoggingGap, lockWarningThreshold));
+ lastWaitLogTimestamp = new AtomicLong(lastHoldLogTimestamp.get());
}
@Override
public void lock() {
+ long waitStart = clock.monotonicNow();
lock.lock();
+ check(waitStart, clock.monotonicNow(), false);
startLockTiming();
}
@Override
public void lockInterruptibly() throws InterruptedException {
+ long waitStart = clock.monotonicNow();
lock.lockInterruptibly();
+ check(waitStart, clock.monotonicNow(), false);
startLockTiming();
}
@@ -118,11 +125,14 @@ public boolean tryLock() {
@Override
public boolean tryLock(long time, TimeUnit unit) throws InterruptedException {
+ long waitStart = clock.monotonicNow();
+ boolean retval = false;
if (lock.tryLock(time, unit)) {
startLockTiming();
- return true;
+ retval = true;
}
- return false;
+ check(waitStart, clock.monotonicNow(), false);
+ return retval;
}
@Override
@@ -130,7 +140,7 @@ public void unlock() {
long localLockReleaseTime = clock.monotonicNow();
long localLockAcquireTime = lockAcquireTimestamp;
lock.unlock();
- check(localLockAcquireTime, localLockReleaseTime);
+ check(localLockAcquireTime, localLockReleaseTime, true);
}
@Override
@@ -139,12 +149,25 @@ public Condition newCondition() {
}
@VisibleForTesting
- void logWarning(long lockHeldTime, long suppressed) {
+ void logWarning(long lockHeldTime, SuppressedSnapshot stats) {
logger.warn(String.format("Lock held time above threshold: " +
"lock identifier: %s " +
"lockHeldTimeMs=%d ms. Suppressed %d lock warnings. " +
+ "Longest suppressed LockHeldTimeMs=%d. " +
"The stack trace is: %s" ,
- name, lockHeldTime, suppressed,
+ name, lockHeldTime, stats.getSuppressedCount(),
+ stats.getMaxSuppressedWait(),
+ StringUtils.getStackTrace(Thread.currentThread())));
+ }
+
+ @VisibleForTesting
+ void logWaitWarning(long lockWaitTime, SuppressedSnapshot stats) {
+ logger.warn(String.format("Waited above threshold to acquire lock: " +
+ "lock identifier: %s " +
+ "waitTimeMs=%d ms. Suppressed %d lock wait warnings. " +
+ "Longest suppressed WaitTimeMs=%d. " +
+ "The stack trace is: %s", name, lockWaitTime,
+ stats.getSuppressedCount(), stats.getMaxSuppressedWait(),
StringUtils.getStackTrace(Thread.currentThread())));
}
@@ -163,27 +186,41 @@ protected void startLockTiming() {
* @param acquireTime - timestamp just after acquiring the lock.
* @param releaseTime - timestamp just before releasing the lock.
*/
- protected void check(long acquireTime, long releaseTime) {
+ protected void check(long acquireTime, long releaseTime,
+ boolean checkLockHeld) {
if (!logger.isWarnEnabled()) {
return;
}
final long lockHeldTime = releaseTime - acquireTime;
if (lockWarningThreshold - lockHeldTime < 0) {
+ AtomicLong lastLogTime;
+ SuppressedStats stats;
+ if (checkLockHeld) {
+ lastLogTime = lastHoldLogTimestamp;
+ stats = holdStats;
+ } else {
+ lastLogTime = lastWaitLogTimestamp;
+ stats = waitStats;
+ }
long now;
long localLastLogTs;
do {
now = clock.monotonicNow();
- localLastLogTs = lastLogTimestamp.get();
+ localLastLogTs = lastLogTime.get();
long deltaSinceLastLog = now - localLastLogTs;
// check should print log or not
if (deltaSinceLastLog - minLoggingGap < 0) {
- warningsSuppressed.incrementAndGet();
+ stats.incrementSuppressed(lockHeldTime);
return;
}
- } while (!lastLogTimestamp.compareAndSet(localLastLogTs, now));
- long suppressed = warningsSuppressed.getAndSet(0);
- logWarning(lockHeldTime, suppressed);
+ } while (!lastLogTime.compareAndSet(localLastLogTs, now));
+ SuppressedSnapshot statsSnapshot = stats.snapshot();
+ if (checkLockHeld) {
+ logWarning(lockHeldTime, statsSnapshot);
+ } else {
+ logWaitWarning(lockHeldTime, statsSnapshot);
+ }
}
}
@@ -194,4 +231,60 @@ protected Lock getLock() {
protected Timer getTimer() {
return clock;
}
+
+ /**
+ * Internal class to track statistics about suppressed log messages in an
+ * atomic way.
+ */
+ private static class SuppressedStats {
+ private long suppressedCount = 0;
+ private long maxSuppressedWait = 0;
+
+ /**
+ * Increments the suppressed counter and increases the max wait time if the
+ * passed wait is greater than the current maxSuppressedWait.
+ * @param wait The wait time for this suppressed message
+ */
+ synchronized public void incrementSuppressed(long wait) {
+ suppressedCount++;
+ if (wait > maxSuppressedWait) {
+ maxSuppressedWait = wait;
+ }
+ }
+
+ /**
+ * Captures the current value of the counts into a SuppressedSnapshot object
+ * and resets the values to zero.
+ *
+ * @return SuppressedSnapshot containing the current value of the counters
+ */
+ synchronized public SuppressedSnapshot snapshot() {
+ SuppressedSnapshot snap =
+ new SuppressedSnapshot(suppressedCount, maxSuppressedWait);
+ suppressedCount = 0;
+ maxSuppressedWait = 0;
+ return snap;
+ }
+ }
+
+ /**
+ * Immutable class to capture a snapshot of suppressed log message stats.
+ */
+ protected static class SuppressedSnapshot {
+ private long suppressedCount = 0;
+ private long maxSuppressedWait = 0;
+
+ public SuppressedSnapshot(long suppressedCount, long maxWait) {
+ this.suppressedCount = suppressedCount;
+ this.maxSuppressedWait = maxWait;
+ }
+
+ public long getMaxSuppressedWait() {
+ return maxSuppressedWait;
+ }
+
+ public long getSuppressedCount() {
+ return suppressedCount;
+ }
+ }
}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/InstrumentedReadLock.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/InstrumentedReadLock.java
index e1157184781d9..8ab392ed041d0 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/InstrumentedReadLock.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/InstrumentedReadLock.java
@@ -75,7 +75,7 @@ public void unlock() {
getLock().unlock();
if (needReport) {
readLockHeldTimeStamp.remove();
- check(localLockAcquireTime, localLockReleaseTime);
+ check(localLockAcquireTime, localLockReleaseTime, true);
}
}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/InstrumentedReadWriteLock.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/InstrumentedReadWriteLock.java
index a41052407697c..758f1ff87cff7 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/InstrumentedReadWriteLock.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/InstrumentedReadWriteLock.java
@@ -37,7 +37,7 @@ public class InstrumentedReadWriteLock implements ReadWriteLock {
private final Lock readLock;
private final Lock writeLock;
- InstrumentedReadWriteLock(boolean fair, String name, Logger logger,
+ public InstrumentedReadWriteLock(boolean fair, String name, Logger logger,
long minLoggingGapMs, long lockWarningThresholdMs) {
ReentrantReadWriteLock readWriteLock = new ReentrantReadWriteLock(fair);
readLock = new InstrumentedReadLock(name, logger, readWriteLock,
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NodeHealthScriptRunner.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NodeHealthScriptRunner.java
deleted file mode 100644
index f2a5b242a8d37..0000000000000
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/NodeHealthScriptRunner.java
+++ /dev/null
@@ -1,339 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.util;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Timer;
-import java.util.TimerTask;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.service.AbstractService;
-import org.apache.hadoop.util.Shell.ExitCodeException;
-import org.apache.hadoop.util.Shell.ShellCommandExecutor;
-import org.apache.hadoop.util.Shell;
-import org.apache.hadoop.util.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- *
- * The class which provides functionality of checking the health of the node
- * using the configured node health script and reporting back to the service
- * for which the health checker has been asked to report.
- */
-public class NodeHealthScriptRunner extends AbstractService {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(NodeHealthScriptRunner.class);
-
- /** Absolute path to the health script. */
- private String nodeHealthScript;
- /** Delay after which node health script to be executed */
- private long intervalTime;
- /** Time after which the script should be timedout */
- private long scriptTimeout;
- /** Timer used to schedule node health monitoring script execution */
- private Timer nodeHealthScriptScheduler;
-
- /** ShellCommandExecutor used to execute monitoring script */
- ShellCommandExecutor shexec = null;
-
- /** Pattern used for searching in the output of the node health script */
- static private final String ERROR_PATTERN = "ERROR";
-
- /** Time out error message */
- public static final String NODE_HEALTH_SCRIPT_TIMED_OUT_MSG = "Node health script timed out";
-
- private boolean isHealthy;
-
- private String healthReport;
-
- private long lastReportedTime;
-
- private TimerTask timer;
-
- private enum HealthCheckerExitStatus {
- SUCCESS,
- TIMED_OUT,
- FAILED_WITH_EXIT_CODE,
- FAILED_WITH_EXCEPTION,
- FAILED
- }
-
-
- /**
- * Class which is used by the {@link Timer} class to periodically execute the
- * node health script.
- *
- */
- private class NodeHealthMonitorExecutor extends TimerTask {
-
- String exceptionStackTrace = "";
-
- public NodeHealthMonitorExecutor(String[] args) {
- ArrayList execScript = new ArrayList();
- execScript.add(nodeHealthScript);
- if (args != null) {
- execScript.addAll(Arrays.asList(args));
- }
- shexec = new ShellCommandExecutor(execScript
- .toArray(new String[execScript.size()]), null, null, scriptTimeout);
- }
-
- @Override
- public void run() {
- HealthCheckerExitStatus status = HealthCheckerExitStatus.SUCCESS;
- try {
- shexec.execute();
- } catch (ExitCodeException e) {
- // ignore the exit code of the script
- status = HealthCheckerExitStatus.FAILED_WITH_EXIT_CODE;
- // On Windows, we will not hit the Stream closed IOException
- // thrown by stdout buffered reader for timeout event.
- if (Shell.WINDOWS && shexec.isTimedOut()) {
- status = HealthCheckerExitStatus.TIMED_OUT;
- }
- } catch (Exception e) {
- LOG.warn("Caught exception : " + e.getMessage());
- if (!shexec.isTimedOut()) {
- status = HealthCheckerExitStatus.FAILED_WITH_EXCEPTION;
- } else {
- status = HealthCheckerExitStatus.TIMED_OUT;
- }
- exceptionStackTrace = StringUtils.stringifyException(e);
- } finally {
- if (status == HealthCheckerExitStatus.SUCCESS) {
- if (hasErrors(shexec.getOutput())) {
- status = HealthCheckerExitStatus.FAILED;
- }
- }
- reportHealthStatus(status);
- }
- }
-
- /**
- * Method which is used to parse output from the node health monitor and
- * send to the report address.
- *
- * The timed out script or script which causes IOException output is
- * ignored.
- *
- * The node is marked unhealthy if
- *
- * - The node health script times out
- * - The node health scripts output has a line which begins with ERROR
- * - An exception is thrown while executing the script
- *
- * If the script throws {@link IOException} or {@link ExitCodeException} the
- * output is ignored and node is left remaining healthy, as script might
- * have syntax error.
- *
- * @param status
- */
- void reportHealthStatus(HealthCheckerExitStatus status) {
- long now = System.currentTimeMillis();
- switch (status) {
- case SUCCESS:
- setHealthStatus(true, "", now);
- break;
- case TIMED_OUT:
- setHealthStatus(false, NODE_HEALTH_SCRIPT_TIMED_OUT_MSG);
- break;
- case FAILED_WITH_EXCEPTION:
- setHealthStatus(false, exceptionStackTrace);
- break;
- case FAILED_WITH_EXIT_CODE:
- // see Javadoc above - we don't report bad health intentionally
- setHealthStatus(true, "", now);
- break;
- case FAILED:
- setHealthStatus(false, shexec.getOutput());
- break;
- }
- }
-
- /**
- * Method to check if the output string has line which begins with ERROR.
- *
- * @param output
- * string
- * @return true if output string has error pattern in it.
- */
- private boolean hasErrors(String output) {
- String[] splits = output.split("\n");
- for (String split : splits) {
- if (split.startsWith(ERROR_PATTERN)) {
- return true;
- }
- }
- return false;
- }
- }
-
- public NodeHealthScriptRunner(String scriptName, long chkInterval, long timeout,
- String[] scriptArgs) {
- super(NodeHealthScriptRunner.class.getName());
- this.lastReportedTime = System.currentTimeMillis();
- this.isHealthy = true;
- this.healthReport = "";
- this.nodeHealthScript = scriptName;
- this.intervalTime = chkInterval;
- this.scriptTimeout = timeout;
- this.timer = new NodeHealthMonitorExecutor(scriptArgs);
- }
-
- /*
- * Method which initializes the values for the script path and interval time.
- */
- @Override
- protected void serviceInit(Configuration conf) throws Exception {
- super.serviceInit(conf);
- }
-
- /**
- * Method used to start the Node health monitoring.
- *
- */
- @Override
- protected void serviceStart() throws Exception {
- nodeHealthScriptScheduler = new Timer("NodeHealthMonitor-Timer", true);
- // Start the timer task immediately and
- // then periodically at interval time.
- nodeHealthScriptScheduler.scheduleAtFixedRate(timer, 0, intervalTime);
- super.serviceStart();
- }
-
- /**
- * Method used to terminate the node health monitoring service.
- *
- */
- @Override
- protected void serviceStop() {
- if (nodeHealthScriptScheduler != null) {
- nodeHealthScriptScheduler.cancel();
- }
- if (shexec != null) {
- Process p = shexec.getProcess();
- if (p != null) {
- p.destroy();
- }
- }
- }
-
- /**
- * Gets the if the node is healthy or not
- *
- * @return true if node is healthy
- */
- public boolean isHealthy() {
- return isHealthy;
- }
-
- /**
- * Sets if the node is healthy or not considering disks' health also.
- *
- * @param isHealthy
- * if or not node is healthy
- */
- private synchronized void setHealthy(boolean isHealthy) {
- this.isHealthy = isHealthy;
- }
-
- /**
- * Returns output from health script. if node is healthy then an empty string
- * is returned.
- *
- * @return output from health script
- */
- public String getHealthReport() {
- return healthReport;
- }
-
- /**
- * Sets the health report from the node health script. Also set the disks'
- * health info obtained from DiskHealthCheckerService.
- *
- * @param healthReport
- */
- private synchronized void setHealthReport(String healthReport) {
- this.healthReport = healthReport;
- }
-
- /**
- * Returns time stamp when node health script was last run.
- *
- * @return timestamp when node health script was last run
- */
- public long getLastReportedTime() {
- return lastReportedTime;
- }
-
- /**
- * Sets the last run time of the node health script.
- *
- * @param lastReportedTime
- */
- private synchronized void setLastReportedTime(long lastReportedTime) {
- this.lastReportedTime = lastReportedTime;
- }
-
- /**
- * Method used to determine if or not node health monitoring service should be
- * started or not. Returns true if following conditions are met:
- *
- *
- * - Path to Node health check script is not empty
- * - Node health check script file exists
- *
- *
- * @return true if node health monitoring service can be started.
- */
- public static boolean shouldRun(String healthScript) {
- if (healthScript == null || healthScript.trim().isEmpty()) {
- return false;
- }
- File f = new File(healthScript);
- return f.exists() && FileUtil.canExecute(f);
- }
-
- private synchronized void setHealthStatus(boolean isHealthy, String output) {
- LOG.info("health status being set as " + output);
- this.setHealthy(isHealthy);
- this.setHealthReport(output);
- }
-
- private synchronized void setHealthStatus(boolean isHealthy, String output,
- long time) {
- LOG.info("health status being set as " + output);
- this.setHealthStatus(isHealthy, output);
- this.setLastReportedTime(time);
- }
-
- /**
- * Used only by tests to access the timer task directly
- * @return the timer task
- */
- public TimerTask getTimerTask() {
- return timer;
- }
-}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ProtoUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ProtoUtil.java
index 9a0b05c369b86..2bb19460b3686 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ProtoUtil.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ProtoUtil.java
@@ -32,7 +32,7 @@
import org.apache.htrace.core.Span;
import org.apache.htrace.core.Tracer;
-import com.google.protobuf.ByteString;
+import org.apache.hadoop.thirdparty.protobuf.ByteString;
public abstract class ProtoUtil {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ReflectionUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ReflectionUtils.java
index ff7357a175b79..1ae71d187d3ba 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ReflectionUtils.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ReflectionUtils.java
@@ -31,6 +31,8 @@
import java.lang.reflect.Method;
import java.nio.charset.Charset;
import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Comparator;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
@@ -350,7 +352,13 @@ public static void cloneWritableInto(Writable dst,
public static List getDeclaredFieldsIncludingInherited(Class clazz) {
List fields = new ArrayList();
while (clazz != null) {
- for (Field field : clazz.getDeclaredFields()) {
+ Field[] sortedFields = clazz.getDeclaredFields();
+ Arrays.sort(sortedFields, new Comparator() {
+ public int compare(Field a, Field b) {
+ return a.getName().compareTo(b.getName());
+ }
+ });
+ for (Field field : sortedFields) {
fields.add(field);
}
clazz = clazz.getSuperclass();
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SequentialNumber.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SequentialNumber.java
index 366e679e64ba5..685e92d628136 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SequentialNumber.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/SequentialNumber.java
@@ -45,6 +45,19 @@ public void setCurrentValue(long value) {
currentValue.set(value);
}
+ public boolean setIfGreater(long value) {
+ while(true) {
+ long local = currentValue.get();
+ if(value <= local) {
+ return false; // swap failed
+ }
+ if(currentValue.compareAndSet(local, value)) {
+ return true; // swap successful
+ }
+ // keep trying
+ }
+ }
+
/** Increment and then return the next value. */
public long nextValue() {
return currentValue.incrementAndGet();
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ThreadUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ThreadUtil.java
index 2cda8a443e4bd..f9ea3fcacd381 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ThreadUtil.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ThreadUtil.java
@@ -49,6 +49,34 @@ public static void sleepAtLeastIgnoreInterrupts(long millis) {
}
}
+ /**
+ * Join a thread as uninterruptible.
+ * The call continues to block until the result is available even when the
+ * caller thread is interrupted.
+ * The method will log any {@link InterruptedException} then will re-interrupt
+ * the thread.
+ *
+ * @param toJoin the thread to Join on.
+ */
+ public static void joinUninterruptibly(Thread toJoin) {
+ boolean interrupted = false;
+ try {
+ while (true) {
+ try {
+ toJoin.join();
+ return;
+ } catch (InterruptedException e) {
+ interrupted = true;
+ LOG.warn("interrupted while sleeping", e);
+ }
+ }
+ } finally {
+ if (interrupted) {
+ Thread.currentThread().interrupt();
+ }
+ }
+ }
+
/**
* Convenience method that returns a resource as inputstream from the
* classpath.
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/concurrent/HadoopExecutors.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/concurrent/HadoopExecutors.java
index 7c09d93d115a5..0bbceb59c31e7 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/concurrent/HadoopExecutors.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/concurrent/HadoopExecutors.java
@@ -115,18 +115,18 @@ public static void shutdown(ExecutorService executorService, Logger logger,
try {
executorService.shutdown();
- logger.info(
+ logger.debug(
"Gracefully shutting down executor service. Waiting max {} {}",
timeout, unit);
if (!executorService.awaitTermination(timeout, unit)) {
- logger.info(
+ logger.debug(
"Executor service has not shutdown yet. Forcing. "
+ "Will wait up to an additional {} {} for shutdown",
timeout, unit);
executorService.shutdownNow();
}
if (executorService.awaitTermination(timeout, unit)) {
- logger.info("Succesfully shutdown executor service");
+ logger.debug("Succesfully shutdown executor service");
} else {
logger.error("Unable to shutdown executor service after timeout {} {}",
(2 * timeout), unit);
diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/OpensslSecureRandom.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/OpensslSecureRandom.c
index 26e1fa623e859..3f141be05b549 100644
--- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/OpensslSecureRandom.c
+++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/random/OpensslSecureRandom.c
@@ -42,16 +42,18 @@
#ifdef UNIX
static void * (*dlsym_CRYPTO_malloc) (int, const char *, int);
static void (*dlsym_CRYPTO_free) (void *);
+#if OPENSSL_VERSION_NUMBER < 0x10100000L
static int (*dlsym_CRYPTO_num_locks) (void);
static void (*dlsym_CRYPTO_set_locking_callback) (void (*)());
static void (*dlsym_CRYPTO_set_id_callback) (unsigned long (*)());
static void (*dlsym_ENGINE_load_rdrand) (void);
+static void (*dlsym_ENGINE_cleanup) (void);
+#endif
static ENGINE * (*dlsym_ENGINE_by_id) (const char *);
static int (*dlsym_ENGINE_init) (ENGINE *);
static int (*dlsym_ENGINE_set_default) (ENGINE *, unsigned int);
static int (*dlsym_ENGINE_finish) (ENGINE *);
static int (*dlsym_ENGINE_free) (ENGINE *);
-static void (*dlsym_ENGINE_cleanup) (void);
static int (*dlsym_RAND_bytes) (unsigned char *, int);
static unsigned long (*dlsym_ERR_get_error) (void);
#endif
@@ -113,6 +115,8 @@ JNIEXPORT void JNICALL Java_org_apache_hadoop_crypto_random_OpensslSecureRandom_
dlerror(); // Clear any existing error
LOAD_DYNAMIC_SYMBOL(dlsym_CRYPTO_malloc, env, openssl, "CRYPTO_malloc");
LOAD_DYNAMIC_SYMBOL(dlsym_CRYPTO_free, env, openssl, "CRYPTO_free");
+#if OPENSSL_VERSION_NUMBER < 0x10100000L
+ // pre-1.1.0
LOAD_DYNAMIC_SYMBOL(dlsym_CRYPTO_num_locks, env, openssl, "CRYPTO_num_locks");
LOAD_DYNAMIC_SYMBOL(dlsym_CRYPTO_set_locking_callback, \
env, openssl, "CRYPTO_set_locking_callback");
@@ -120,13 +124,14 @@ JNIEXPORT void JNICALL Java_org_apache_hadoop_crypto_random_OpensslSecureRandom_
openssl, "CRYPTO_set_id_callback");
LOAD_DYNAMIC_SYMBOL(dlsym_ENGINE_load_rdrand, env, \
openssl, "ENGINE_load_rdrand");
+ LOAD_DYNAMIC_SYMBOL(dlsym_ENGINE_cleanup, env, openssl, "ENGINE_cleanup");
+#endif
LOAD_DYNAMIC_SYMBOL(dlsym_ENGINE_by_id, env, openssl, "ENGINE_by_id");
LOAD_DYNAMIC_SYMBOL(dlsym_ENGINE_init, env, openssl, "ENGINE_init");
LOAD_DYNAMIC_SYMBOL(dlsym_ENGINE_set_default, env, \
openssl, "ENGINE_set_default");
LOAD_DYNAMIC_SYMBOL(dlsym_ENGINE_finish, env, openssl, "ENGINE_finish");
LOAD_DYNAMIC_SYMBOL(dlsym_ENGINE_free, env, openssl, "ENGINE_free");
- LOAD_DYNAMIC_SYMBOL(dlsym_ENGINE_cleanup, env, openssl, "ENGINE_cleanup");
LOAD_DYNAMIC_SYMBOL(dlsym_RAND_bytes, env, openssl, "RAND_bytes");
LOAD_DYNAMIC_SYMBOL(dlsym_ERR_get_error, env, openssl, "ERR_get_error");
#endif
@@ -303,9 +308,11 @@ static unsigned long pthreads_thread_id(void)
*/
static ENGINE * openssl_rand_init(void)
{
+#if OPENSSL_VERSION_NUMBER < 0x10100000L
locks_setup();
dlsym_ENGINE_load_rdrand();
+#endif
ENGINE *eng = dlsym_ENGINE_by_id("rdrand");
int ret = -1;
@@ -340,11 +347,12 @@ static void openssl_rand_clean(ENGINE *eng, int clean_locks)
dlsym_ENGINE_finish(eng);
dlsym_ENGINE_free(eng);
}
-
+#if OPENSSL_VERSION_NUMBER < 0x10100000L
dlsym_ENGINE_cleanup();
if (clean_locks) {
locks_cleanup();
}
+#endif
}
static int openssl_rand_bytes(unsigned char *buf, int num)
diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c
index b0b5151cd1238..1d7c508d85c76 100644
--- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c
+++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c
@@ -1486,11 +1486,11 @@ JNIEnv *env, jclass thisClass, jlong address, jlong length) {
/*
* Class: org_apache_hadoop_io_nativeio_NativeIO_POSIX
- * Method: pmemCreateMapFile
+ * Method: pmemMapFile
* Signature: (Ljava/lang/String;J)Lorg/apache/hadoop/io/nativeio/NativeIO/POSIX/PmemMappedRegion;
*/
-JNIEXPORT jobject JNICALL Java_org_apache_hadoop_io_nativeio_NativeIO_00024POSIX_pmemCreateMapFile(
-JNIEnv *env, jclass thisClass, jstring filePath, jlong fileLength) {
+JNIEXPORT jobject JNICALL Java_org_apache_hadoop_io_nativeio_NativeIO_00024POSIX_pmemMapFile(
+JNIEnv *env, jclass thisClass, jstring filePath, jlong fileLength, jboolean isFileExist) {
#if (defined UNIX) && (defined HADOOP_PMDK_LIBRARY)
/* create a pmem file and memory map it */
const char * path = NULL;
@@ -1505,17 +1505,20 @@ JNIEnv *env, jclass thisClass, jstring filePath, jlong fileLength) {
return NULL;
}
- if (fileLength <= 0) {
- (*env)->ReleaseStringUTFChars(env, filePath, path);
- THROW(env, "java/lang/IllegalArgumentException", "File length should be positive");
- return NULL;
+ if (isFileExist) {
+ pmemaddr = pmdkLoader->pmem_map_file(path, 0, 0, 0666, &mapped_len, &is_pmem);
+ } else {
+ if (fileLength <= 0) {
+ (*env)->ReleaseStringUTFChars(env, filePath, path);
+ THROW(env, "java/lang/IllegalArgumentException", "File length should be positive");
+ return NULL;
+ }
+ pmemaddr = pmdkLoader->pmem_map_file(path, fileLength, PMEM_FILE_CREATE|PMEM_FILE_EXCL,
+ 0666, &mapped_len, &is_pmem);
}
- pmemaddr = pmdkLoader->pmem_map_file(path, fileLength, PMEM_FILE_CREATE|PMEM_FILE_EXCL,
- 0666, &mapped_len, &is_pmem);
-
if (!pmemaddr) {
- snprintf(msg, sizeof(msg), "Failed to create pmem file. file: %s, length: %x, error msg: %s", path, fileLength, pmem_errormsg());
+ snprintf(msg, sizeof(msg), "Failed to map file on persistent memory.file: %s, length: %x, error msg: %s", path, fileLength, pmem_errormsg());
THROW(env, "java/io/IOException", msg);
(*env)->ReleaseStringUTFChars(env, filePath, path);
return NULL;
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/common-version-info.properties b/hadoop-common-project/hadoop-common/src/main/resources/common-version-info.properties
index 9b74960cf72bc..6f8558b8d4fe9 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/common-version-info.properties
+++ b/hadoop-common-project/hadoop-common/src/main/resources/common-version-info.properties
@@ -23,4 +23,4 @@ user=${user.name}
date=${version-info.build.time}
url=${version-info.scm.uri}
srcChecksum=${version-info.source.md5}
-protocVersion=${protobuf.version}
+protocVersion=${hadoop.protobuf.version}
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index f0aa44dc999ca..0d583cca57cd0 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -48,6 +48,14 @@
ordering of the filters.