From b1dfdf1c94ddecb53af8b58db122b9843a238c8d Mon Sep 17 00:00:00 2001 From: Siddharth Wagle Date: Mon, 29 Apr 2019 14:22:14 -0700 Subject: [PATCH 1/4] HDDS-1464. Client should have different retry policies for different exceptions. --- .../hadoop/ozone/client/OzoneClientUtils.java | 43 ++++++++++++++----- .../ozone/client/io/KeyOutputStream.java | 21 +++++---- 2 files changed, 45 insertions(+), 19 deletions(-) diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneClientUtils.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneClientUtils.java index b52f39380b0db..2d5381a872f6b 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneClientUtils.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneClientUtils.java @@ -17,22 +17,29 @@ */ package org.apache.hadoop.ozone.client; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + import org.apache.hadoop.hdds.client.OzoneQuota; import org.apache.hadoop.hdds.scm.client.HddsClientUtils; import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerNotOpenException; import org.apache.hadoop.io.retry.RetryPolicies; import org.apache.hadoop.io.retry.RetryPolicy; import org.apache.hadoop.ozone.OzoneConsts; -import org.apache.hadoop.ozone.client.rest.response.*; +import org.apache.hadoop.ozone.client.rest.response.BucketInfo; +import org.apache.hadoop.ozone.client.rest.response.KeyInfo; +import org.apache.hadoop.ozone.client.rest.response.KeyInfoDetails; +import org.apache.hadoop.ozone.client.rest.response.KeyLocation; +import org.apache.hadoop.ozone.client.rest.response.VolumeInfo; +import org.apache.hadoop.ozone.client.rest.response.VolumeOwner; import org.apache.ratis.protocol.AlreadyClosedException; import org.apache.ratis.protocol.GroupMismatchException; import org.apache.ratis.protocol.RaftRetryFailureException; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; - /** A utility class for OzoneClient. */ public final class OzoneClientUtils { @@ -129,14 +136,28 @@ public static KeyInfoDetails asKeyInfoDetails(OzoneKeyDetails key) { public static RetryPolicy createRetryPolicy(int maxRetryCount, long retryInterval) { - // just retry without sleep - RetryPolicy retryPolicy = RetryPolicies - .retryUpToMaximumCountWithFixedSleep(maxRetryCount, retryInterval, - TimeUnit.MILLISECONDS); - return retryPolicy; + // retry with fixed sleep between retries + return RetryPolicies.retryUpToMaximumCountWithFixedSleep( + maxRetryCount, retryInterval, TimeUnit.MILLISECONDS); } public static List> getExceptionList() { return EXCEPTION_LIST; } + + public static Map, RetryPolicy> + getRetryPolicyByException(int maxRetryCount, long retryInterval) { + Map, RetryPolicy> policyMap = new HashMap<>(); + for (Class ex : EXCEPTION_LIST) { + if (ex == TimeoutException.class || + ex == RaftRetryFailureException.class) { + // retry without sleep + policyMap.put(ex, createRetryPolicy(maxRetryCount, 0)); + } else { + // retry with fixed sleep between retries + policyMap.put(ex, createRetryPolicy(maxRetryCount, retryInterval)); + } + } + return policyMap; + } } diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/KeyOutputStream.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/KeyOutputStream.java index 8ab773817a6ae..620e583ffb7dc 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/KeyOutputStream.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/KeyOutputStream.java @@ -22,8 +22,7 @@ import org.apache.hadoop.fs.FSExceptionMessages; import org.apache.hadoop.fs.FileEncryptionInfo; import org.apache.hadoop.hdds.protocol.DatanodeDetails; -import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos - .ChecksumType; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ChecksumType; import org.apache.hadoop.hdds.scm.container.ContainerID; import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerNotOpenException; import org.apache.hadoop.hdds.scm.storage.BufferPool; @@ -52,7 +51,10 @@ import java.util.List; import java.util.Collection; import java.util.ListIterator; +import java.util.Map; import java.util.concurrent.TimeoutException; +import java.util.function.Function; +import java.util.stream.Collectors; /** * Maintaining a list of BlockInputStream. Write based on offset. @@ -95,7 +97,7 @@ enum StreamAction { private OmMultipartCommitUploadPartInfo commitUploadPartInfo; private FileEncryptionInfo feInfo; private ExcludeList excludeList; - private final RetryPolicy retryPolicy; + private final Map, RetryPolicy> retryPolicyMap; private int retryCount; private long offset; /** @@ -121,7 +123,10 @@ public KeyOutputStream() { OzoneConfigKeys.OZONE_CLIENT_CHECKSUM_TYPE_DEFAULT); this.bytesPerChecksum = OzoneConfigKeys .OZONE_CLIENT_BYTES_PER_CHECKSUM_DEFAULT_BYTES; // Default is 1MB - this.retryPolicy = RetryPolicies.TRY_ONCE_THEN_FAIL; + this.retryPolicyMap = OzoneClientUtils.getExceptionList() + .stream() + .collect(Collectors.toMap(Function.identity(), + e -> RetryPolicies.TRY_ONCE_THEN_FAIL)); retryCount = 0; offset = 0; } @@ -200,8 +205,8 @@ public KeyOutputStream(OpenKeySession handler, this.bufferPool = new BufferPool(chunkSize, (int)streamBufferMaxSize / chunkSize); this.excludeList = new ExcludeList(); - this.retryPolicy = OzoneClientUtils.createRetryPolicy(maxRetryCount, - retryInterval); + this.retryPolicyMap = OzoneClientUtils.getRetryPolicyByException( + maxRetryCount, retryInterval); this.retryCount = 0; } @@ -503,9 +508,9 @@ private void markStreamClosed() { private void handleRetry(IOException exception, long len) throws IOException { RetryPolicy.RetryAction action; + RetryPolicy retryPolicy = retryPolicyMap.get(exception.getClass()); try { - action = retryPolicy - .shouldRetry(exception, retryCount, 0, true); + action = retryPolicy.shouldRetry(exception, retryCount, 0, true); } catch (Exception e) { throw e instanceof IOException ? (IOException) e : new IOException(e); } From 3a4bb97e77736656ed23f80b20e6e10dafe9aa84 Mon Sep 17 00:00:00 2001 From: Siddharth Wagle Date: Mon, 29 Apr 2019 14:59:42 -0700 Subject: [PATCH 2/4] HDDS-1464. Add default RetryPolicy. Fix integration test. --- .../apache/hadoop/ozone/client/OzoneClientUtils.java | 7 +++++-- .../apache/hadoop/ozone/client/io/KeyOutputStream.java | 10 +++++++--- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneClientUtils.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneClientUtils.java index 2d5381a872f6b..ea8eccd17fcdf 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneClientUtils.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneClientUtils.java @@ -145,9 +145,9 @@ public static List> getExceptionList() { return EXCEPTION_LIST; } - public static Map, RetryPolicy> + public static Map, RetryPolicy> getRetryPolicyByException(int maxRetryCount, long retryInterval) { - Map, RetryPolicy> policyMap = new HashMap<>(); + Map, RetryPolicy> policyMap = new HashMap<>(); for (Class ex : EXCEPTION_LIST) { if (ex == TimeoutException.class || ex == RaftRetryFailureException.class) { @@ -158,6 +158,9 @@ public static List> getExceptionList() { policyMap.put(ex, createRetryPolicy(maxRetryCount, retryInterval)); } } + // Default retry policy + policyMap.put(Exception.class, createRetryPolicy( + maxRetryCount, retryInterval)); return policyMap; } } diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/KeyOutputStream.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/KeyOutputStream.java index 620e583ffb7dc..055855276ba92 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/KeyOutputStream.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/KeyOutputStream.java @@ -97,7 +97,7 @@ enum StreamAction { private OmMultipartCommitUploadPartInfo commitUploadPartInfo; private FileEncryptionInfo feInfo; private ExcludeList excludeList; - private final Map, RetryPolicy> retryPolicyMap; + private final Map, RetryPolicy> retryPolicyMap; private int retryCount; private long offset; /** @@ -505,10 +505,14 @@ private void markStreamClosed() { cleanup(); closed = true; } - + private void handleRetry(IOException exception, long len) throws IOException { + RetryPolicy retryPolicy = + retryPolicyMap.get(checkForException(exception).getClass()); + if (retryPolicy == null) { + retryPolicy = retryPolicyMap.get(Exception.class); + } RetryPolicy.RetryAction action; - RetryPolicy retryPolicy = retryPolicyMap.get(exception.getClass()); try { action = retryPolicy.shouldRetry(exception, retryCount, 0, true); } catch (Exception e) { From e245865932b19786aa8f39a80e8ec0bac6dc78c6 Mon Sep 17 00:00:00 2001 From: Siddharth Wagle Date: Mon, 29 Apr 2019 17:29:49 -0700 Subject: [PATCH 3/4] HDDS-1464. Fix for whitespace error. --- .../java/org/apache/hadoop/ozone/client/io/KeyOutputStream.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/KeyOutputStream.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/KeyOutputStream.java index 055855276ba92..c4c2524fea3a3 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/KeyOutputStream.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/KeyOutputStream.java @@ -505,7 +505,7 @@ private void markStreamClosed() { cleanup(); closed = true; } - + private void handleRetry(IOException exception, long len) throws IOException { RetryPolicy retryPolicy = retryPolicyMap.get(checkForException(exception).getClass()); From 21cc83eade32326e2d894c097c4404a83fea3d98 Mon Sep 17 00:00:00 2001 From: Siddharth Wagle Date: Thu, 2 May 2019 09:05:22 -0700 Subject: [PATCH 4/4] HDDS-1464. Fix for checkstyle error. --- .../java/org/apache/hadoop/ozone/client/OzoneClientUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneClientUtils.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneClientUtils.java index ea8eccd17fcdf..83632b5c67d1b 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneClientUtils.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneClientUtils.java @@ -146,7 +146,7 @@ public static List> getExceptionList() { } public static Map, RetryPolicy> - getRetryPolicyByException(int maxRetryCount, long retryInterval) { + getRetryPolicyByException(int maxRetryCount, long retryInterval) { Map, RetryPolicy> policyMap = new HashMap<>(); for (Class ex : EXCEPTION_LIST) { if (ex == TimeoutException.class ||