From 98d2901f1ba79a2483e890676e5eed869514caad Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Thu, 12 May 2022 15:26:11 +0530 Subject: [PATCH 01/33] Metric --- .../hadoop/fs/azurebfs/AbfsCountersImpl.java | 8 +++++++ .../hadoop/fs/azurebfs/AbfsDriverMetrics.java | 23 +++++++++++++++++++ .../fs/azurebfs/AzureBlobFileSystem.java | 2 +- .../fs/azurebfs/services/AbfsClient.java | 5 ++-- .../azurebfs/services/AbfsRestOperation.java | 7 ++++-- 5 files changed, 39 insertions(+), 6 deletions(-) create mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsCountersImpl.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsCountersImpl.java index c478256e706bc..d44e70f16eceb 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsCountersImpl.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsCountersImpl.java @@ -36,6 +36,7 @@ import static org.apache.hadoop.fs.azurebfs.AbfsStatistic.*; import static org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.iostatisticsStore; +import java.util.concurrent.atomic.AtomicReference; /** * Instrumentation of Abfs counters. @@ -63,6 +64,8 @@ public class AbfsCountersImpl implements AbfsCounters { private final IOStatisticsStore ioStatisticsStore; + private AtomicReference abfsDriverMetrics = null; + private static final AbfsStatistic[] STATISTIC_LIST = { CALL_CREATE, CALL_OPEN, @@ -117,6 +120,7 @@ public AbfsCountersImpl(URI uri) { ioStatisticsStoreBuilder.withDurationTracking(durationStats.getStatName()); } ioStatisticsStore = ioStatisticsStoreBuilder.build(); + abfsDriverMetrics = new AtomicReference(new AbfsDriverMetrics()); } /** @@ -184,6 +188,10 @@ private MetricsRegistry getRegistry() { return registry; } + public AbfsDriverMetrics getAbfsDriverMetrics() { + return abfsDriverMetrics.get(); + } + /** * {@inheritDoc} * diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java new file mode 100644 index 0000000000000..01393e92cf3cf --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java @@ -0,0 +1,23 @@ +package org.apache.hadoop.fs.azurebfs; + +import java.util.concurrent.atomic.AtomicLong; + +public class AbfsDriverMetrics { + private AtomicLong numberOfRequestsSucceededInFirstRetry; + private AtomicLong numberOfRequestsSucceededInSecondRetry; + private AtomicLong numberOfRequestsSucceededInThirdRetry; + private AtomicLong numberOfRequestsSucceededInFourthRetry; + private AtomicLong numberOfRequestsSucceededInFiveToFifteenRetry; + private AtomicLong numberOfRequestsSucceededInFifteenToTwentyFiveRetry; + private AtomicLong numberOfRequestsSucceededInTwentyFiveToThirtyRetry; + + public AbfsDriverMetrics() { + this.numberOfRequestsSucceededInFirstRetry = new AtomicLong(); + this.numberOfRequestsSucceededInSecondRetry = new AtomicLong(); + this.numberOfRequestsSucceededInThirdRetry = new AtomicLong(); + this.numberOfRequestsSucceededInFourthRetry = new AtomicLong(); + this.numberOfRequestsSucceededInFiveToFifteenRetry = new AtomicLong(); + this.numberOfRequestsSucceededInFifteenToTwentyFiveRetry = new AtomicLong(); + this.numberOfRequestsSucceededInTwentyFiveToThirtyRetry = new AtomicLong(); + } +} diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java index 46141e7c4a838..1b36c0103703d 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java @@ -49,7 +49,7 @@ import org.apache.hadoop.util.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; - +import java.util.concurrent.atomic.AtomicLong; import org.apache.commons.lang3.ArrayUtils; import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.classification.InterfaceAudience; diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java index b701037d0fc41..fcb4d615476c0 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java @@ -308,7 +308,6 @@ public AbfsRestOperation listPath(final String relativePath, final boolean recur public AbfsRestOperation getFilesystemProperties(TracingContext tracingContext) throws AzureBlobFileSystemException { final List requestHeaders = createDefaultHeaders(); - final AbfsUriQueryBuilder abfsUriQueryBuilder = createDefaultUriQueryBuilder(); abfsUriQueryBuilder.addQuery(QUERY_PARAM_RESOURCE, FILESYSTEM); @@ -754,7 +753,8 @@ public AbfsRestOperation setPathProperties(final String path, final String prope public AbfsRestOperation getPathStatus(final String path, final boolean includeProperties, TracingContext tracingContext) throws AzureBlobFileSystemException { final List requestHeaders = createDefaultHeaders(); - + String headerValue = "Path=%2Fsomepath%2Ffile.txt,Operation=ReadFile,ReadAheadHit=10,ReadAheadMiss=5"; + requestHeaders.add(new AbfsHttpHeader(X_MS_CLIENT_METRICS, headerValue)); final AbfsUriQueryBuilder abfsUriQueryBuilder = createDefaultUriQueryBuilder(); String operation = SASTokenProvider.GET_PROPERTIES_OPERATION; if (!includeProperties) { @@ -984,7 +984,6 @@ public AbfsRestOperation getAclStatus(final String path, TracingContext tracingC public AbfsRestOperation getAclStatus(final String path, final boolean useUPN, TracingContext tracingContext) throws AzureBlobFileSystemException { final List requestHeaders = createDefaultHeaders(); - final AbfsUriQueryBuilder abfsUriQueryBuilder = createDefaultUriQueryBuilder(); abfsUriQueryBuilder.addQuery(HttpQueryParams.QUERY_PARAM_ACTION, AbfsHttpConstants.GET_ACCESS_CONTROL); abfsUriQueryBuilder.addQuery(HttpQueryParams.QUERY_PARAM_UPN, String.valueOf(useUPN)); diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java index 74b267d563eb2..e67b57f05a735 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java @@ -36,7 +36,7 @@ import org.apache.hadoop.fs.azurebfs.constants.HttpHeaderConfigurations; import org.apache.hadoop.fs.azurebfs.utils.TracingContext; import org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding; - +import org.apache.hadoop.fs.azurebfs.AbfsDriverMetrics; /** * The AbfsRestOperation for Rest AbfsClient. */ @@ -71,6 +71,8 @@ public class AbfsRestOperation { private AbfsHttpOperation result; private AbfsCounters abfsCounters; + private AbfsDriverMetrics abfsDriverMetrics; + /** * Checks if there is non-null HTTP response. * @return true if there is a non-null HTTP response from the ABFS call. @@ -145,6 +147,7 @@ String getSasToken() { || AbfsHttpConstants.HTTP_METHOD_PATCH.equals(method)); this.sasToken = sasToken; this.abfsCounters = client.getAbfsCounters(); + this.abfsDriverMetrics = abfsCounters.getAbfsDriverMetrics(); } /** @@ -175,6 +178,7 @@ String getSasToken() { this.bufferOffset = bufferOffset; this.bufferLength = bufferLength; this.abfsCounters = client.getAbfsCounters(); + this.abfsDriverMetrics = abfsCounters.getAbfsDriverMetrics(); } /** @@ -225,7 +229,6 @@ private void completeExecute(TracingContext tracingContext) Thread.currentThread().interrupt(); } } - if (result.getStatusCode() >= HttpURLConnection.HTTP_BAD_REQUEST) { throw new AbfsRestOperationException(result.getStatusCode(), result.getStorageErrorCode(), result.getStorageErrorMessage(), null, result); From 16612d30d19550780fd90cf02abde98293525770 Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Mon, 16 May 2022 12:25:12 +0530 Subject: [PATCH 02/33] Metric changes --- .../hadoop/fs/azurebfs/AbfsDriverMetrics.java | 35 ++++++++++++++ .../fs/azurebfs/AzureBlobFileSystem.java | 18 ++++++++ .../fs/azurebfs/AzureBlobFileSystemStore.java | 4 ++ .../constants/FileSystemConfigurations.java | 2 +- .../fs/azurebfs/services/AbfsClient.java | 24 +++++++++- .../fs/azurebfs/services/AbfsCounters.java | 4 +- .../azurebfs/services/AbfsHttpOperation.java | 2 +- .../azurebfs/services/AbfsRestOperation.java | 46 +++++++++++++++++++ .../fs/azurebfs/utils/TracingContext.java | 2 +- 9 files changed, 131 insertions(+), 6 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java index 01393e92cf3cf..7ba28c6c5df04 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java @@ -20,4 +20,39 @@ public AbfsDriverMetrics() { this.numberOfRequestsSucceededInFifteenToTwentyFiveRetry = new AtomicLong(); this.numberOfRequestsSucceededInTwentyFiveToThirtyRetry = new AtomicLong(); } + + public AtomicLong getNumberOfRequestsSucceededInFirstRetry() { + return numberOfRequestsSucceededInFirstRetry; + } + + public AtomicLong getNumberOfRequestsSucceededInSecondRetry() { + return numberOfRequestsSucceededInSecondRetry; + } + + public AtomicLong getNumberOfRequestsSucceededInThirdRetry() { + return numberOfRequestsSucceededInThirdRetry; + } + + public AtomicLong getNumberOfRequestsSucceededInFourthRetry() { + return numberOfRequestsSucceededInFourthRetry; + } + + public AtomicLong getNumberOfRequestsSucceededInFiveToFifteenRetry() { + return numberOfRequestsSucceededInFiveToFifteenRetry; + } + + public AtomicLong getNumberOfRequestsSucceededInFifteenToTwentyFiveRetry() { + return numberOfRequestsSucceededInFifteenToTwentyFiveRetry; + } + + public AtomicLong getNumberOfRequestsSucceededInTwentyFiveToThirtyRetry() { + return numberOfRequestsSucceededInTwentyFiveToThirtyRetry; + } + + @Override + public String toString() { + return "#RCTSI_1R_ " + numberOfRequestsSucceededInFirstRetry + "#RCTSI_2R_ " + numberOfRequestsSucceededInSecondRetry + "#RCTSI_3R_ " + + numberOfRequestsSucceededInThirdRetry + "#RCTSI_4R_ " + numberOfRequestsSucceededInFourthRetry + "#RCTSI_5-15R_ " + numberOfRequestsSucceededInFiveToFifteenRetry + + "#RCTSI_15-25R_ " + numberOfRequestsSucceededInFifteenToTwentyFiveRetry + "#RCTSI_25-30R_ " + numberOfRequestsSucceededInTwentyFiveToThirtyRetry; + } } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java index 1b36c0103703d..6853c2a45e80a 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java @@ -678,6 +678,17 @@ public synchronized void close() throws IOException { if (isClosed) { return; } + try { + //String metric = abfsCounters.getAbfsDriverMetrics().toString(); + String metric = "abcd"; + URI metricUri = new URI( + "abfs://metric@snvarmacanary.dfs.core.windows.net"); + AzureBlobFileSystem metricfs = (AzureBlobFileSystem) FileSystem.get( + metricUri, getConf()); + metricfs.sentMetric(metric); + } catch (Exception ex) { + // do nothing + } // does all the delete-on-exit calls, and may be slow. super.close(); LOG.debug("AzureBlobFileSystem.close"); @@ -694,6 +705,13 @@ public synchronized void close() throws IOException { } } + public void sentMetric(String metric) throws AzureBlobFileSystemException { + TracingContext tracingContext = new TracingContext(clientCorrelationId, + fileSystemId, FSOperationType.SET_ATTR, true, tracingHeaderFormat, + listener); + abfsStore.sentMetric(metric, tracingContext); + } + @Override public FileStatus getFileStatus(final Path f) throws IOException { TracingContext tracingContext = new TracingContext(clientCorrelationId, diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java index 09b48a855f00b..681f991facbe2 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java @@ -1973,4 +1973,8 @@ public static String extractEtagHeader(AbfsHttpOperation result) { } return etag; } + + public void sentMetric(String metric, TracingContext tracingContext) throws AzureBlobFileSystemException{ + AbfsRestOperation op = client.getPathStatusMetric("/..$$@@", tracingContext, metric); // Will sent a GFS calls that will fail to register in MDM x-ms-client-metric + } } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java index 63d62a33b1819..f510824041bce 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java @@ -41,7 +41,7 @@ public final class FileSystemConfigurations { public static final int DEFAULT_MIN_BACKOFF_INTERVAL = 3 * 1000; // 3s public static final int DEFAULT_MAX_BACKOFF_INTERVAL = 30 * 1000; // 30s public static final int DEFAULT_BACKOFF_INTERVAL = 3 * 1000; // 3s - public static final int DEFAULT_MAX_RETRY_ATTEMPTS = 30; + public static final int DEFAULT_MAX_RETRY_ATTEMPTS = 5; public static final int DEFAULT_CUSTOM_TOKEN_FETCH_RETRY_COUNT = 3; // Retry parameter defaults. diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java index fcb4d615476c0..aaa6b384486af 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java @@ -750,11 +750,31 @@ public AbfsRestOperation setPathProperties(final String path, final String prope return op; } + public AbfsRestOperation getPathStatusMetric(final String path, + TracingContext tracingContext, String metric) throws AzureBlobFileSystemException { + final List requestHeaders = createDefaultHeaders(); + requestHeaders.add(new AbfsHttpHeader(X_MS_CLIENT_REQUEST_ID, metric)); + final AbfsUriQueryBuilder abfsUriQueryBuilder = createDefaultUriQueryBuilder(); + String operation = SASTokenProvider.GET_PROPERTIES_OPERATION; + abfsUriQueryBuilder.addQuery(HttpQueryParams.QUERY_PARAM_ACTION, AbfsHttpConstants.GET_STATUS); + operation = SASTokenProvider.GET_STATUS_OPERATION; + abfsUriQueryBuilder.addQuery(HttpQueryParams.QUERY_PARAM_UPN, String.valueOf(abfsConfiguration.isUpnUsed())); + appendSASTokenToQuery(path, operation, abfsUriQueryBuilder); + + final URL url = createRequestUrl(path, abfsUriQueryBuilder.toString()); + final AbfsRestOperation op = new AbfsRestOperation( + AbfsRestOperationType.GetPathStatus, + this, + HTTP_METHOD_HEAD, + url, + requestHeaders); + op.execute(tracingContext); + return op; + } + public AbfsRestOperation getPathStatus(final String path, final boolean includeProperties, TracingContext tracingContext) throws AzureBlobFileSystemException { final List requestHeaders = createDefaultHeaders(); - String headerValue = "Path=%2Fsomepath%2Ffile.txt,Operation=ReadFile,ReadAheadHit=10,ReadAheadMiss=5"; - requestHeaders.add(new AbfsHttpHeader(X_MS_CLIENT_METRICS, headerValue)); final AbfsUriQueryBuilder abfsUriQueryBuilder = createDefaultUriQueryBuilder(); String operation = SASTokenProvider.GET_PROPERTIES_OPERATION; if (!includeProperties) { diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsCounters.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsCounters.java index d01a3598afcf8..91eb03c821566 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsCounters.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsCounters.java @@ -28,7 +28,7 @@ import org.apache.hadoop.fs.statistics.DurationTracker; import org.apache.hadoop.fs.statistics.DurationTrackerFactory; import org.apache.hadoop.fs.statistics.IOStatisticsSource; - +import org.apache.hadoop.fs.azurebfs.AbfsDriverMetrics; /** * An interface for Abfs counters. */ @@ -74,4 +74,6 @@ String formString(String prefix, String separator, String suffix, */ @Override DurationTracker trackDuration(String key); + + AbfsDriverMetrics getAbfsDriverMetrics(); } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsHttpOperation.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsHttpOperation.java index 413bf3686898b..0cfbf03106227 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsHttpOperation.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsHttpOperation.java @@ -345,7 +345,7 @@ public void processResponse(final byte[] buffer, final int offset, final int len } this.statusCode = this.connection.getResponseCode(); - + //this.statusCode = 503; if (this.isTraceEnabled) { this.recvResponseTimeMs = elapsedTimeMs(startTime); } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java index e67b57f05a735..7b3a8aba132d4 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java @@ -37,6 +37,7 @@ import org.apache.hadoop.fs.azurebfs.utils.TracingContext; import org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding; import org.apache.hadoop.fs.azurebfs.AbfsDriverMetrics; +import java.util.concurrent.atomic.AtomicLong; /** * The AbfsRestOperation for Rest AbfsClient. */ @@ -229,6 +230,51 @@ private void completeExecute(TracingContext tracingContext) Thread.currentThread().interrupt(); } } + if(retryCount > 0) { + if (retryCount == 1) { + long noOfRequests1Retry + = abfsDriverMetrics.getNumberOfRequestsSucceededInFirstRetry() + .incrementAndGet(); + abfsDriverMetrics.getNumberOfRequestsSucceededInFirstRetry() + .set(noOfRequests1Retry); + } else if (retryCount == 2) { + long noOfRequests2Retry + = abfsDriverMetrics.getNumberOfRequestsSucceededInSecondRetry() + .incrementAndGet(); + abfsDriverMetrics.getNumberOfRequestsSucceededInSecondRetry() + .set(noOfRequests2Retry); + } else if (retryCount == 3) { + long noOfRequests3Retry + = abfsDriverMetrics.getNumberOfRequestsSucceededInThirdRetry() + .incrementAndGet(); + abfsDriverMetrics.getNumberOfRequestsSucceededInThirdRetry() + .set(noOfRequests3Retry); + } else if (retryCount == 4) { + long noOfRequests4Retry + = abfsDriverMetrics.getNumberOfRequestsSucceededInFourthRetry() + .incrementAndGet(); + abfsDriverMetrics.getNumberOfRequestsSucceededInFourthRetry() + .set(noOfRequests4Retry); + } else if (retryCount >= 5 && retryCount < 15) { + long noOfRequests5_15Retry + = abfsDriverMetrics.getNumberOfRequestsSucceededInFiveToFifteenRetry() + .incrementAndGet(); + abfsDriverMetrics.getNumberOfRequestsSucceededInFiveToFifteenRetry() + .set(noOfRequests5_15Retry); + } else if (retryCount >= 15 && retryCount < 25) { + long noOfRequests15_25Retry + = abfsDriverMetrics.getNumberOfRequestsSucceededInFifteenToTwentyFiveRetry() + .incrementAndGet(); + abfsDriverMetrics.getNumberOfRequestsSucceededInFifteenToTwentyFiveRetry() + .set(noOfRequests15_25Retry); + } else { + long noOfRequests25_30Retry + = abfsDriverMetrics.getNumberOfRequestsSucceededInTwentyFiveToThirtyRetry() + .incrementAndGet(); + abfsDriverMetrics.getNumberOfRequestsSucceededInTwentyFiveToThirtyRetry() + .set(noOfRequests25_30Retry); + } + } if (result.getStatusCode() >= HttpURLConnection.HTTP_BAD_REQUEST) { throw new AbfsRestOperationException(result.getStatusCode(), result.getStorageErrorCode(), result.getStorageErrorMessage(), null, result); diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/TracingContext.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/TracingContext.java index 5a115451df159..05fe33b7a19da 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/TracingContext.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/TracingContext.java @@ -171,7 +171,7 @@ public void constructHeader(AbfsHttpOperation httpOperation) { if (listener != null) { //for testing listener.callTracingHeaderValidator(header, format); } - httpOperation.setRequestProperty(HttpHeaderConfigurations.X_MS_CLIENT_REQUEST_ID, header); + //httpOperation.setRequestProperty(HttpHeaderConfigurations.X_MS_CLIENT_REQUEST_ID, header); } /** From fb2295ea13f10c2ce911137466a9e9c02e68282e Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Fri, 20 May 2022 15:03:14 +0530 Subject: [PATCH 03/33] Metric changes --- .../hadoop/fs/azurebfs/AbfsDriverMetrics.java | 14 ++++++++--- .../fs/azurebfs/AzureBlobFileSystem.java | 23 ++++++++++++++----- .../azurebfs/constants/ConfigurationKeys.java | 5 ++++ .../azurebfs/services/AbfsRestOperation.java | 6 ++++- 4 files changed, 38 insertions(+), 10 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java index 7ba28c6c5df04..69d57c5ac1eb3 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java @@ -11,6 +11,8 @@ public class AbfsDriverMetrics { private AtomicLong numberOfRequestsSucceededInFifteenToTwentyFiveRetry; private AtomicLong numberOfRequestsSucceededInTwentyFiveToThirtyRetry; + private AtomicLong minBackoffForFirstRetry; + public AbfsDriverMetrics() { this.numberOfRequestsSucceededInFirstRetry = new AtomicLong(); this.numberOfRequestsSucceededInSecondRetry = new AtomicLong(); @@ -19,6 +21,7 @@ public AbfsDriverMetrics() { this.numberOfRequestsSucceededInFiveToFifteenRetry = new AtomicLong(); this.numberOfRequestsSucceededInFifteenToTwentyFiveRetry = new AtomicLong(); this.numberOfRequestsSucceededInTwentyFiveToThirtyRetry = new AtomicLong(); + this.minBackoffForFirstRetry = new AtomicLong(Long.MAX_VALUE); } public AtomicLong getNumberOfRequestsSucceededInFirstRetry() { @@ -49,10 +52,15 @@ public AtomicLong getNumberOfRequestsSucceededInTwentyFiveToThirtyRetry() { return numberOfRequestsSucceededInTwentyFiveToThirtyRetry; } + public AtomicLong getMinBackoffForFirstRetry() { + return minBackoffForFirstRetry; + } + @Override public String toString() { - return "#RCTSI_1R_ " + numberOfRequestsSucceededInFirstRetry + "#RCTSI_2R_ " + numberOfRequestsSucceededInSecondRetry + "#RCTSI_3R_ " + - numberOfRequestsSucceededInThirdRetry + "#RCTSI_4R_ " + numberOfRequestsSucceededInFourthRetry + "#RCTSI_5-15R_ " + numberOfRequestsSucceededInFiveToFifteenRetry + - "#RCTSI_15-25R_ " + numberOfRequestsSucceededInFifteenToTwentyFiveRetry + "#RCTSI_25-30R_ " + numberOfRequestsSucceededInTwentyFiveToThirtyRetry; + return "#RCTSI_1R_" + numberOfRequestsSucceededInFirstRetry + " #RCTSI_2R_" + numberOfRequestsSucceededInSecondRetry + " #RCTSI_3R_" + + numberOfRequestsSucceededInThirdRetry + " #RCTSI_4R_" + numberOfRequestsSucceededInFourthRetry + " #RCTSI_5-15R_" + numberOfRequestsSucceededInFiveToFifteenRetry + + " #RCTSI_15-25R_" + numberOfRequestsSucceededInFifteenToTwentyFiveRetry + " #RCTSI_25-30R_" + numberOfRequestsSucceededInTwentyFiveToThirtyRetry + + " #Min_1R_ " + minBackoffForFirstRetry; } } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java index 6853c2a45e80a..8e67f36b368d0 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java @@ -114,8 +114,11 @@ import static org.apache.hadoop.fs.CommonConfigurationKeys.IOSTATISTICS_LOGGING_LEVEL_DEFAULT; import static org.apache.hadoop.fs.azurebfs.AbfsStatistic.*; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.DATA_BLOCKS_BUFFER; +import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_BLOCK_UPLOAD_ACTIVE_BLOCKS; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_BLOCK_UPLOAD_BUFFER_DIR; +import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_METRIC_ACCOUNT_KEY; +import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_METRIC_ACCOUNT_NAME; import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.BLOCK_UPLOAD_ACTIVE_BLOCKS_DEFAULT; import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DATA_BLOCKS_BUFFER_DEFAULT; import static org.apache.hadoop.fs.impl.PathCapabilitiesSupport.validatePathCapabilityArgs; @@ -679,12 +682,20 @@ public synchronized void close() throws IOException { return; } try { - //String metric = abfsCounters.getAbfsDriverMetrics().toString(); - String metric = "abcd"; - URI metricUri = new URI( - "abfs://metric@snvarmacanary.dfs.core.windows.net"); - AzureBlobFileSystem metricfs = (AzureBlobFileSystem) FileSystem.get( - metricUri, getConf()); + String metric = abfsCounters.getAbfsDriverMetrics().toString(); + Configuration metricConfig = getConf(); + String metricAccountName = getConf().get(FS_AZURE_METRIC_ACCOUNT_NAME); + String metricAccountKey = getConf().get(FS_AZURE_METRIC_ACCOUNT_KEY); + final String abfsMetricUrl = "metrics" + "@" + metricAccountName; + metricConfig.set(FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME, metricAccountKey); + URI metricUri = null; + try { + metricUri = new URI(getScheme(), abfsMetricUrl, null, null, null); + } catch (Exception ex) { + throw new AssertionError(ex); + } + AzureBlobFileSystem metricfs = (AzureBlobFileSystem) FileSystem.newInstance( + metricUri, metricConfig); metricfs.sentMetric(metric); } catch (Exception ex) { // do nothing diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java index 9d3b2d5e82c6e..99963fe0cda89 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java @@ -36,6 +36,11 @@ public final class ConfigurationKeys { */ public static final String FS_AZURE_ACCOUNT_IS_HNS_ENABLED = "fs.azure.account.hns.enabled"; public static final String FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME = "fs.azure.account.key"; + + public static final String FS_AZURE_METRIC_ACCOUNT_NAME = "fs.azure.metric.account.name"; + + public static final String FS_AZURE_METRIC_ACCOUNT_KEY = "fs.azure.metric.account.key"; + public static final String FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME_REGX = "fs\\.azure\\.account\\.key\\.(.*)"; public static final String FS_AZURE_SECURE_MODE = "fs.azure.secure.mode"; diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java index 7b3a8aba132d4..8ff8a281eabdf 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java @@ -219,24 +219,28 @@ private void completeExecute(TracingContext tracingContext) retryCount = 0; LOG.debug("First execution of REST operation - {}", operationType); + long sleepDuration = 0L; while (!executeHttpOperation(retryCount, tracingContext)) { try { ++retryCount; tracingContext.setRetryCount(retryCount); LOG.debug("Retrying REST operation {}. RetryCount = {}", operationType, retryCount); - Thread.sleep(client.getRetryPolicy().getRetryInterval(retryCount)); + sleepDuration = client.getRetryPolicy().getRetryInterval(retryCount); + Thread.sleep(sleepDuration); } catch (InterruptedException ex) { Thread.currentThread().interrupt(); } } if(retryCount > 0) { if (retryCount == 1) { + long minSleepForFirstRetry = Math.min(abfsDriverMetrics.getMinBackoffForFirstRetry().get(), sleepDuration); long noOfRequests1Retry = abfsDriverMetrics.getNumberOfRequestsSucceededInFirstRetry() .incrementAndGet(); abfsDriverMetrics.getNumberOfRequestsSucceededInFirstRetry() .set(noOfRequests1Retry); + abfsDriverMetrics.getMinBackoffForFirstRetry().set(minSleepForFirstRetry); } else if (retryCount == 2) { long noOfRequests2Retry = abfsDriverMetrics.getNumberOfRequestsSucceededInSecondRetry() From 986d2e6ff24078a32a8aedd5fa66482295eb2720 Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Fri, 27 May 2022 11:37:36 +0530 Subject: [PATCH 04/33] Metrics --- .../hadoop/fs/azurebfs/AbfsCountersImpl.java | 2 +- .../hadoop/fs/azurebfs/AbfsDriverMetrics.java | 127 ++++++++++++------ .../fs/azurebfs/AzureBlobFileSystem.java | 5 +- .../constants/FileSystemConfigurations.java | 2 +- .../services/AzureServiceErrorCode.java | 1 + .../azurebfs/services/AbfsRestOperation.java | 98 +++++++------- .../azurebfs/ITestAbfsReadWriteAndSeek.java | 13 +- 7 files changed, 153 insertions(+), 95 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsCountersImpl.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsCountersImpl.java index d44e70f16eceb..1371cee6f3c64 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsCountersImpl.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsCountersImpl.java @@ -120,7 +120,7 @@ public AbfsCountersImpl(URI uri) { ioStatisticsStoreBuilder.withDurationTracking(durationStats.getStatName()); } ioStatisticsStore = ioStatisticsStoreBuilder.build(); - abfsDriverMetrics = new AtomicReference(new AbfsDriverMetrics()); + abfsDriverMetrics = new AtomicReference<>(new AbfsDriverMetrics()); } /** diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java index 69d57c5ac1eb3..ee38e151016b0 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java @@ -1,66 +1,115 @@ package org.apache.hadoop.fs.azurebfs; -import java.util.concurrent.atomic.AtomicLong; - +import java.util.concurrent.atomic.AtomicLong; +import java.util.Map; +import java.util.LinkedHashMap; +import java.util.ArrayList; +import java.util.Arrays; public class AbfsDriverMetrics { - private AtomicLong numberOfRequestsSucceededInFirstRetry; - private AtomicLong numberOfRequestsSucceededInSecondRetry; - private AtomicLong numberOfRequestsSucceededInThirdRetry; - private AtomicLong numberOfRequestsSucceededInFourthRetry; - private AtomicLong numberOfRequestsSucceededInFiveToFifteenRetry; - private AtomicLong numberOfRequestsSucceededInFifteenToTwentyFiveRetry; - private AtomicLong numberOfRequestsSucceededInTwentyFiveToThirtyRetry; + private AtomicLong numberOfRequestsSucceeded; + private AtomicLong minBackoff; + private AtomicLong maxBackoff; + private AtomicLong totalRequests; + private AtomicLong totalBackoff; + private String retryCount; + private AtomicLong numberOfIOPSThrottledRequests; + private AtomicLong numberOfBandwidthThrottledRequests; + + private AtomicLong numberOfOtherThrottledRequests; - private AtomicLong minBackoffForFirstRetry; + private static final Map metricsMap = new LinkedHashMap<>(); public AbfsDriverMetrics() { - this.numberOfRequestsSucceededInFirstRetry = new AtomicLong(); - this.numberOfRequestsSucceededInSecondRetry = new AtomicLong(); - this.numberOfRequestsSucceededInThirdRetry = new AtomicLong(); - this.numberOfRequestsSucceededInFourthRetry = new AtomicLong(); - this.numberOfRequestsSucceededInFiveToFifteenRetry = new AtomicLong(); - this.numberOfRequestsSucceededInFifteenToTwentyFiveRetry = new AtomicLong(); - this.numberOfRequestsSucceededInTwentyFiveToThirtyRetry = new AtomicLong(); - this.minBackoffForFirstRetry = new AtomicLong(Long.MAX_VALUE); + initializeMap(); + this.numberOfIOPSThrottledRequests = new AtomicLong(); + this.numberOfBandwidthThrottledRequests = new AtomicLong(); + this.numberOfOtherThrottledRequests = new AtomicLong(); + } + public AbfsDriverMetrics(String retryCount) { + this.retryCount = retryCount; + this.numberOfRequestsSucceeded = new AtomicLong(); + this.minBackoff = new AtomicLong(Long.MAX_VALUE); + this.maxBackoff = new AtomicLong(); + this.totalRequests = new AtomicLong(); + this.totalBackoff = new AtomicLong(); + } + + private void initializeMap() { + ArrayList retryCountList = new ArrayList(Arrays.asList("1","2","3","4","5_15","15_25","25_30")); + for (String s : retryCountList) { + metricsMap.put(s, new AbfsDriverMetrics(s)); + } + } + + public AtomicLong getNumberOfRequestsSucceeded() { + return numberOfRequestsSucceeded; + } + + public AtomicLong getMinBackoff() { + return minBackoff; } - public AtomicLong getNumberOfRequestsSucceededInFirstRetry() { - return numberOfRequestsSucceededInFirstRetry; + public AtomicLong getMaxBackoff() { + return maxBackoff; } - public AtomicLong getNumberOfRequestsSucceededInSecondRetry() { - return numberOfRequestsSucceededInSecondRetry; + public AtomicLong getTotalRequests() { + return totalRequests; } - public AtomicLong getNumberOfRequestsSucceededInThirdRetry() { - return numberOfRequestsSucceededInThirdRetry; + public AtomicLong getTotalBackoff() { + return totalBackoff; } - public AtomicLong getNumberOfRequestsSucceededInFourthRetry() { - return numberOfRequestsSucceededInFourthRetry; + public String getRetryCount() { + return retryCount; } - public AtomicLong getNumberOfRequestsSucceededInFiveToFifteenRetry() { - return numberOfRequestsSucceededInFiveToFifteenRetry; + public AtomicLong getNumberOfIOPSThrottledRequests() { + return numberOfIOPSThrottledRequests; } - public AtomicLong getNumberOfRequestsSucceededInFifteenToTwentyFiveRetry() { - return numberOfRequestsSucceededInFifteenToTwentyFiveRetry; + public AtomicLong getNumberOfBandwidthThrottledRequests() { + return numberOfBandwidthThrottledRequests; } - public AtomicLong getNumberOfRequestsSucceededInTwentyFiveToThirtyRetry() { - return numberOfRequestsSucceededInTwentyFiveToThirtyRetry; + public AtomicLong getNumberOfOtherThrottledRequests() { + return numberOfOtherThrottledRequests; } - public AtomicLong getMinBackoffForFirstRetry() { - return minBackoffForFirstRetry; + public Map getMetricsMap() { + return metricsMap; } @Override public String toString() { - return "#RCTSI_1R_" + numberOfRequestsSucceededInFirstRetry + " #RCTSI_2R_" + numberOfRequestsSucceededInSecondRetry + " #RCTSI_3R_" + - numberOfRequestsSucceededInThirdRetry + " #RCTSI_4R_" + numberOfRequestsSucceededInFourthRetry + " #RCTSI_5-15R_" + numberOfRequestsSucceededInFiveToFifteenRetry + - " #RCTSI_15-25R_" + numberOfRequestsSucceededInFifteenToTwentyFiveRetry + " #RCTSI_25-30R_" + numberOfRequestsSucceededInTwentyFiveToThirtyRetry + - " #Min_1R_ " + minBackoffForFirstRetry; - } + StringBuilder requestsSucceeded = new StringBuilder(); + StringBuilder minMaxAvg = new StringBuilder(); + StringBuilder throttledRequests = new StringBuilder(); + for (Map.Entry entry : metricsMap.entrySet()) { + requestsSucceeded.append("#RCTSI#_").append(entry.getKey()) + .append("R_").append("=") + .append(entry.getValue().getNumberOfRequestsSucceeded()).append(" "); + long totalRequests = entry.getValue().getTotalRequests().get(); + if(totalRequests > 0) { + minMaxAvg.append("MinMaxAvg#_").append(entry.getKey()) + .append("R_").append("=") + .append(String.format("%.5f", (double) entry.getValue().getMinBackoff().get() / 1000L)) + .append(" seconds ") + .append(String.format("%.5f", (double) entry.getValue().getMaxBackoff().get() / 1000L)) + .append(" seconds ") + .append(String.format("%.5f", (double) ((entry.getValue().getTotalBackoff().get() / totalRequests) / 1000L))) + .append(" seconds "); + }else { + minMaxAvg.append("MinMaxAvg#_").append(entry.getKey()) + .append("R_").append("= 0 seconds "); + } + } + throttledRequests.append("BandwidthThrottled = ").append(numberOfBandwidthThrottledRequests) + .append("IOPSThrottled = ").append(numberOfIOPSThrottledRequests) + .append("OtherThrottled = ").append(numberOfOtherThrottledRequests); + + return requestsSucceeded + " " + minMaxAvg + " " + throttledRequests; + } } + diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java index 8e67f36b368d0..345d3d03d642a 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java @@ -688,14 +688,13 @@ public synchronized void close() throws IOException { String metricAccountKey = getConf().get(FS_AZURE_METRIC_ACCOUNT_KEY); final String abfsMetricUrl = "metrics" + "@" + metricAccountName; metricConfig.set(FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME, metricAccountKey); - URI metricUri = null; + URI metricUri; try { metricUri = new URI(getScheme(), abfsMetricUrl, null, null, null); } catch (Exception ex) { throw new AssertionError(ex); } - AzureBlobFileSystem metricfs = (AzureBlobFileSystem) FileSystem.newInstance( - metricUri, metricConfig); + AzureBlobFileSystem metricfs = (AzureBlobFileSystem) FileSystem.newInstance(metricUri, metricConfig); metricfs.sentMetric(metric); } catch (Exception ex) { // do nothing diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java index f510824041bce..63d62a33b1819 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java @@ -41,7 +41,7 @@ public final class FileSystemConfigurations { public static final int DEFAULT_MIN_BACKOFF_INTERVAL = 3 * 1000; // 3s public static final int DEFAULT_MAX_BACKOFF_INTERVAL = 30 * 1000; // 30s public static final int DEFAULT_BACKOFF_INTERVAL = 3 * 1000; // 3s - public static final int DEFAULT_MAX_RETRY_ATTEMPTS = 5; + public static final int DEFAULT_MAX_RETRY_ATTEMPTS = 30; public static final int DEFAULT_CUSTOM_TOKEN_FETCH_RETRY_COUNT = 3; // Retry parameter defaults. diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/services/AzureServiceErrorCode.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/services/AzureServiceErrorCode.java index 8bc31c4f92b2a..5c4b01b12dfac 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/services/AzureServiceErrorCode.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/services/AzureServiceErrorCode.java @@ -44,6 +44,7 @@ public enum AzureServiceErrorCode { INVALID_RENAME_SOURCE_PATH("InvalidRenameSourcePath", HttpURLConnection.HTTP_CONFLICT, null), INGRESS_OVER_ACCOUNT_LIMIT(null, HttpURLConnection.HTTP_UNAVAILABLE, "Ingress is over the account limit."), EGRESS_OVER_ACCOUNT_LIMIT(null, HttpURLConnection.HTTP_UNAVAILABLE, "Egress is over the account limit."), + REQUEST_OVER_ACCOUNT_LIMIT(null, HttpURLConnection.HTTP_UNAVAILABLE, "Operations per second is over the account limit."), INVALID_QUERY_PARAMETER_VALUE("InvalidQueryParameterValue", HttpURLConnection.HTTP_BAD_REQUEST, null), AUTHORIZATION_PERMISSION_MISS_MATCH("AuthorizationPermissionMismatch", HttpURLConnection.HTTP_FORBIDDEN, null), ACCOUNT_REQUIRES_HTTPS("AccountRequiresHttps", HttpURLConnection.HTTP_BAD_REQUEST, null), diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java index 8ff8a281eabdf..ea8dcdfed2d6a 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java @@ -36,8 +36,10 @@ import org.apache.hadoop.fs.azurebfs.constants.HttpHeaderConfigurations; import org.apache.hadoop.fs.azurebfs.utils.TracingContext; import org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding; +import org.apache.hadoop.fs.azurebfs.contracts.services.AzureServiceErrorCode; +import java.util.Map; import org.apache.hadoop.fs.azurebfs.AbfsDriverMetrics; -import java.util.concurrent.atomic.AtomicLong; + /** * The AbfsRestOperation for Rest AbfsClient. */ @@ -73,7 +75,7 @@ public class AbfsRestOperation { private AbfsCounters abfsCounters; private AbfsDriverMetrics abfsDriverMetrics; - + private Map metricsMap; /** * Checks if there is non-null HTTP response. * @return true if there is a non-null HTTP response from the ABFS call. @@ -149,6 +151,7 @@ String getSasToken() { this.sasToken = sasToken; this.abfsCounters = client.getAbfsCounters(); this.abfsDriverMetrics = abfsCounters.getAbfsDriverMetrics(); + this.metricsMap = abfsDriverMetrics.getMetricsMap(); } /** @@ -180,6 +183,7 @@ String getSasToken() { this.bufferLength = bufferLength; this.abfsCounters = client.getAbfsCounters(); this.abfsDriverMetrics = abfsCounters.getAbfsDriverMetrics(); + this.metricsMap = abfsDriverMetrics.getMetricsMap(); } /** @@ -227,56 +231,25 @@ private void completeExecute(TracingContext tracingContext) LOG.debug("Retrying REST operation {}. RetryCount = {}", operationType, retryCount); sleepDuration = client.getRetryPolicy().getRetryInterval(retryCount); + updateTimeMetrics(retryCount, sleepDuration); Thread.sleep(sleepDuration); } catch (InterruptedException ex) { Thread.currentThread().interrupt(); } } if(retryCount > 0) { - if (retryCount == 1) { - long minSleepForFirstRetry = Math.min(abfsDriverMetrics.getMinBackoffForFirstRetry().get(), sleepDuration); - long noOfRequests1Retry - = abfsDriverMetrics.getNumberOfRequestsSucceededInFirstRetry() - .incrementAndGet(); - abfsDriverMetrics.getNumberOfRequestsSucceededInFirstRetry() - .set(noOfRequests1Retry); - abfsDriverMetrics.getMinBackoffForFirstRetry().set(minSleepForFirstRetry); - } else if (retryCount == 2) { - long noOfRequests2Retry - = abfsDriverMetrics.getNumberOfRequestsSucceededInSecondRetry() - .incrementAndGet(); - abfsDriverMetrics.getNumberOfRequestsSucceededInSecondRetry() - .set(noOfRequests2Retry); - } else if (retryCount == 3) { - long noOfRequests3Retry - = abfsDriverMetrics.getNumberOfRequestsSucceededInThirdRetry() - .incrementAndGet(); - abfsDriverMetrics.getNumberOfRequestsSucceededInThirdRetry() - .set(noOfRequests3Retry); - } else if (retryCount == 4) { - long noOfRequests4Retry - = abfsDriverMetrics.getNumberOfRequestsSucceededInFourthRetry() - .incrementAndGet(); - abfsDriverMetrics.getNumberOfRequestsSucceededInFourthRetry() - .set(noOfRequests4Retry); - } else if (retryCount >= 5 && retryCount < 15) { - long noOfRequests5_15Retry - = abfsDriverMetrics.getNumberOfRequestsSucceededInFiveToFifteenRetry() - .incrementAndGet(); - abfsDriverMetrics.getNumberOfRequestsSucceededInFiveToFifteenRetry() - .set(noOfRequests5_15Retry); - } else if (retryCount >= 15 && retryCount < 25) { - long noOfRequests15_25Retry - = abfsDriverMetrics.getNumberOfRequestsSucceededInFifteenToTwentyFiveRetry() - .incrementAndGet(); - abfsDriverMetrics.getNumberOfRequestsSucceededInFifteenToTwentyFiveRetry() - .set(noOfRequests15_25Retry); - } else { - long noOfRequests25_30Retry - = abfsDriverMetrics.getNumberOfRequestsSucceededInTwentyFiveToThirtyRetry() - .incrementAndGet(); - abfsDriverMetrics.getNumberOfRequestsSucceededInTwentyFiveToThirtyRetry() - .set(noOfRequests25_30Retry); + updateCount(retryCount); + } + if(result.getStatusCode() == HttpURLConnection.HTTP_UNAVAILABLE){ + AzureServiceErrorCode serviceErrorCode = + AzureServiceErrorCode.getAzureServiceCode(result.getStatusCode(), result.getStorageErrorCode(), result.getStorageErrorMessage()); + if(serviceErrorCode.equals(AzureServiceErrorCode.INGRESS_OVER_ACCOUNT_LIMIT) || + serviceErrorCode.equals(AzureServiceErrorCode.EGRESS_OVER_ACCOUNT_LIMIT)){ + abfsDriverMetrics.getNumberOfBandwidthThrottledRequests().getAndIncrement(); + }else if(serviceErrorCode.equals(AzureServiceErrorCode.REQUEST_OVER_ACCOUNT_LIMIT)){ + abfsDriverMetrics.getNumberOfIOPSThrottledRequests().getAndIncrement(); + }else{ + abfsDriverMetrics.getNumberOfOtherThrottledRequests().getAndIncrement(); } } if (result.getStatusCode() >= HttpURLConnection.HTTP_BAD_REQUEST) { @@ -387,7 +360,7 @@ private boolean executeHttpOperation(final int retryCount, /** * Incrementing Abfs counters with a long value. * - * @param statistic the Abfs statistic that needs to be incremented. + * @param statistic the Abfs statistic that needs to be incremented.f * @param value the value to be incremented by. */ private void incrementCounter(AbfsStatistic statistic, long value) { @@ -395,4 +368,35 @@ private void incrementCounter(AbfsStatistic statistic, long value) { abfsCounters.incrementCounter(statistic, value); } } + + private void updateCount(int retryCount){ + String retryCounter = getKey(retryCount); + metricsMap.get(retryCounter).getNumberOfRequestsSucceeded().getAndIncrement(); + } + + private void updateTimeMetrics(int retryCount, long sleepDuration){ + String retryCounter = getKey(retryCount); + long minBackoffTime = Math.min(metricsMap.get(retryCounter).getMinBackoff().get(), sleepDuration); + long maxBackoffForTime = Math.max(metricsMap.get(retryCounter).getMaxBackoff().get(), sleepDuration); + long totalBackoffTime = metricsMap.get(retryCounter).getTotalBackoff().get() + sleepDuration; + long totalRequests = metricsMap.get(retryCounter).getTotalRequests().incrementAndGet(); + metricsMap.get(retryCounter).getMinBackoff().set(minBackoffTime); + metricsMap.get(retryCounter).getMaxBackoff().set(maxBackoffForTime); + metricsMap.get(retryCounter).getTotalBackoff().set(totalBackoffTime); + metricsMap.get(retryCounter).getTotalRequests().set(totalRequests); + } + + private String getKey(int retryCount) { + String retryCounter; + if(retryCount >= 1 && retryCount <= 4){ + retryCounter = Integer.toString(retryCount); + }else if(retryCount >= 5 && retryCount < 15){ + retryCounter = "5_15"; + }else if(retryCount >= 15 && retryCount < 25){ + retryCounter = "15_25"; + }else{ + retryCounter = "25_30"; + } + return retryCounter; + } } diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsReadWriteAndSeek.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsReadWriteAndSeek.java index 5bd6eaff42e84..ff60306dac9b5 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsReadWriteAndSeek.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsReadWriteAndSeek.java @@ -50,12 +50,17 @@ public class ITestAbfsReadWriteAndSeek extends AbstractAbfsScaleTest { private static final String TEST_PATH = "/testfile"; +// @Parameterized.Parameters(name = "Size={0}") +// public static Iterable sizes() { +// return Arrays.asList(new Object[][]{{MIN_BUFFER_SIZE}, +// {DEFAULT_READ_BUFFER_SIZE}, +// {APPENDBLOB_MAX_WRITE_BUFFER_SIZE}, +// {MAX_BUFFER_SIZE}}); +// } + @Parameterized.Parameters(name = "Size={0}") public static Iterable sizes() { - return Arrays.asList(new Object[][]{{MIN_BUFFER_SIZE}, - {DEFAULT_READ_BUFFER_SIZE}, - {APPENDBLOB_MAX_WRITE_BUFFER_SIZE}, - {MAX_BUFFER_SIZE}}); + return Arrays.asList(new Object[][]{{MIN_BUFFER_SIZE}}); } private final int size; From 23d9b472d45a21ebc8ab7060f77353fc969d243f Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Mon, 6 Jun 2022 17:04:51 +0530 Subject: [PATCH 05/33] Added final metrics --- .../hadoop/fs/azurebfs/AbfsDriverMetrics.java | 38 ++++++++++----- .../constants/FileSystemConfigurations.java | 2 +- .../services/AzureServiceErrorCode.java | 33 +++++++------ .../azurebfs/services/AbfsRestOperation.java | 48 +++++++++++-------- .../src/test/resources/log4j.properties | 2 + 5 files changed, 76 insertions(+), 47 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java index ee38e151016b0..0de1895c2b224 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java @@ -14,9 +14,9 @@ public class AbfsDriverMetrics { private String retryCount; private AtomicLong numberOfIOPSThrottledRequests; private AtomicLong numberOfBandwidthThrottledRequests; - private AtomicLong numberOfOtherThrottledRequests; - + private AtomicLong maxRetryCount; + private AtomicLong totalNumberOfRequests; private static final Map metricsMap = new LinkedHashMap<>(); public AbfsDriverMetrics() { @@ -24,6 +24,8 @@ public AbfsDriverMetrics() { this.numberOfIOPSThrottledRequests = new AtomicLong(); this.numberOfBandwidthThrottledRequests = new AtomicLong(); this.numberOfOtherThrottledRequests = new AtomicLong(); + this.totalNumberOfRequests = new AtomicLong(); + this.maxRetryCount = new AtomicLong(); } public AbfsDriverMetrics(String retryCount) { this.retryCount = retryCount; @@ -77,22 +79,30 @@ public AtomicLong getNumberOfOtherThrottledRequests() { return numberOfOtherThrottledRequests; } + public AtomicLong getMaxRetryCount() { + return maxRetryCount; + } + + public AtomicLong getTotalNumberOfRequests() { + return totalNumberOfRequests; + } + public Map getMetricsMap() { return metricsMap; } @Override public String toString() { - StringBuilder requestsSucceeded = new StringBuilder(); - StringBuilder minMaxAvg = new StringBuilder(); - StringBuilder throttledRequests = new StringBuilder(); + StringBuilder metricString = new StringBuilder(); + long totalRequestsThrottled = numberOfBandwidthThrottledRequests.get() + numberOfIOPSThrottledRequests.get() + numberOfOtherThrottledRequests.get(); + double percentageOfRequestsThrottled = ((double)totalRequestsThrottled/totalNumberOfRequests.get())*100; for (Map.Entry entry : metricsMap.entrySet()) { - requestsSucceeded.append("#RCTSI#_").append(entry.getKey()) + metricString.append(" #RCTSI#_").append(entry.getKey()) .append("R_").append("=") .append(entry.getValue().getNumberOfRequestsSucceeded()).append(" "); long totalRequests = entry.getValue().getTotalRequests().get(); if(totalRequests > 0) { - minMaxAvg.append("MinMaxAvg#_").append(entry.getKey()) + metricString.append(" MinMaxAvg#_").append(entry.getKey()) .append("R_").append("=") .append(String.format("%.5f", (double) entry.getValue().getMinBackoff().get() / 1000L)) .append(" seconds ") @@ -101,15 +111,19 @@ public String toString() { .append(String.format("%.5f", (double) ((entry.getValue().getTotalBackoff().get() / totalRequests) / 1000L))) .append(" seconds "); }else { - minMaxAvg.append("MinMaxAvg#_").append(entry.getKey()) + metricString.append("MinMaxAvg#_").append(entry.getKey()) .append("R_").append("= 0 seconds "); } } - throttledRequests.append("BandwidthThrottled = ").append(numberOfBandwidthThrottledRequests) - .append("IOPSThrottled = ").append(numberOfIOPSThrottledRequests) - .append("OtherThrottled = ").append(numberOfOtherThrottledRequests); + metricString.append(" BandwidthThrottled = ").append(numberOfBandwidthThrottledRequests) + .append(" IOPSThrottled = ").append(numberOfIOPSThrottledRequests) + .append(" OtherThrottled = ").append(numberOfOtherThrottledRequests) + .append(" Total number of requests = ").append(totalNumberOfRequests); + + metricString.append(" Max retry count is ").append(maxRetryCount); + metricString.append(" Percentage of throttled requests = ").append(percentageOfRequestsThrottled); - return requestsSucceeded + " " + minMaxAvg + " " + throttledRequests; + return metricString + " "; } } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java index 63d62a33b1819..49d0a1fd18917 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java @@ -38,7 +38,7 @@ public final class FileSystemConfigurations { private static final int SIXTY_SECONDS = 60 * 1000; // Retry parameter defaults. - public static final int DEFAULT_MIN_BACKOFF_INTERVAL = 3 * 1000; // 3s + public static final int DEFAULT_MIN_BACKOFF_INTERVAL = 25 * 10; // 3s public static final int DEFAULT_MAX_BACKOFF_INTERVAL = 30 * 1000; // 30s public static final int DEFAULT_BACKOFF_INTERVAL = 3 * 1000; // 3s public static final int DEFAULT_MAX_RETRY_ATTEMPTS = 30; diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/services/AzureServiceErrorCode.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/services/AzureServiceErrorCode.java index 5c4b01b12dfac..ce53b1fd1bfe1 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/services/AzureServiceErrorCode.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/services/AzureServiceErrorCode.java @@ -21,7 +21,8 @@ import java.net.HttpURLConnection; import java.util.ArrayList; import java.util.List; - +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; @@ -42,9 +43,9 @@ public enum AzureServiceErrorCode { INVALID_SOURCE_OR_DESTINATION_RESOURCE_TYPE("InvalidSourceOrDestinationResourceType", HttpURLConnection.HTTP_CONFLICT, null), RENAME_DESTINATION_PARENT_PATH_NOT_FOUND("RenameDestinationParentPathNotFound", HttpURLConnection.HTTP_NOT_FOUND, null), INVALID_RENAME_SOURCE_PATH("InvalidRenameSourcePath", HttpURLConnection.HTTP_CONFLICT, null), - INGRESS_OVER_ACCOUNT_LIMIT(null, HttpURLConnection.HTTP_UNAVAILABLE, "Ingress is over the account limit."), - EGRESS_OVER_ACCOUNT_LIMIT(null, HttpURLConnection.HTTP_UNAVAILABLE, "Egress is over the account limit."), - REQUEST_OVER_ACCOUNT_LIMIT(null, HttpURLConnection.HTTP_UNAVAILABLE, "Operations per second is over the account limit."), + INGRESS_OVER_ACCOUNT_LIMIT("ServerBusy", HttpURLConnection.HTTP_UNAVAILABLE, "Ingress is over the account limit."), + EGRESS_OVER_ACCOUNT_LIMIT("ServerBusy", HttpURLConnection.HTTP_UNAVAILABLE, "Egress is over the account limit."), + REQUEST_OVER_ACCOUNT_LIMIT("ServerBusy", HttpURLConnection.HTTP_UNAVAILABLE, "Operations per second is over the account limit."), INVALID_QUERY_PARAMETER_VALUE("InvalidQueryParameterValue", HttpURLConnection.HTTP_BAD_REQUEST, null), AUTHORIZATION_PERMISSION_MISS_MATCH("AuthorizationPermissionMismatch", HttpURLConnection.HTTP_FORBIDDEN, null), ACCOUNT_REQUIRES_HTTPS("AccountRequiresHttps", HttpURLConnection.HTTP_BAD_REQUEST, null), @@ -53,6 +54,9 @@ public enum AzureServiceErrorCode { private final String errorCode; private final int httpStatusCode; private final String errorMessage; + + private static final Logger LOG1 = LoggerFactory.getLogger(AzureServiceErrorCode.class); + AzureServiceErrorCode(String errorCode, int httpStatusCodes, String errorMessage) { this.errorCode = errorCode; this.httpStatusCode = httpStatusCodes; @@ -67,6 +71,10 @@ public String getErrorCode() { return this.errorCode; } + public String getErrorMessage() { + return this.errorMessage; + } + public static List getAzureServiceCode(int httpStatusCode) { List errorCodes = new ArrayList<>(); if (httpStatusCode == UNKNOWN.httpStatusCode) { @@ -94,24 +102,21 @@ public static AzureServiceErrorCode getAzureServiceCode(int httpStatusCode, Stri return azureServiceErrorCode; } } - return UNKNOWN; } - public static AzureServiceErrorCode getAzureServiceCode(int httpStatusCode, String errorCode, final String errorMessage) { + public static AzureServiceErrorCode getAzureServiceCode(int httpStatusCode, String errorCode, String errorMessage) { if (errorCode == null || errorCode.isEmpty() || httpStatusCode == UNKNOWN.httpStatusCode || errorMessage == null || errorMessage.isEmpty()) { return UNKNOWN; } - + String[] errorMessages = errorMessage.split(System.lineSeparator(), 2); for (AzureServiceErrorCode azureServiceErrorCode : AzureServiceErrorCode.values()) { - if (azureServiceErrorCode.httpStatusCode == httpStatusCode - && errorCode.equalsIgnoreCase(azureServiceErrorCode.errorCode) - && errorMessage.equalsIgnoreCase(azureServiceErrorCode.errorMessage) - ) { - return azureServiceErrorCode; + if (azureServiceErrorCode.getStatusCode() == httpStatusCode + && azureServiceErrorCode.getErrorCode().equalsIgnoreCase(errorCode) && + azureServiceErrorCode.getErrorMessage().equalsIgnoreCase(errorMessages[0])) { + return azureServiceErrorCode; + } } - } - return UNKNOWN; } } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java index ea8dcdfed2d6a..af89a9595db7f 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java @@ -63,19 +63,19 @@ public class AbfsRestOperation { private final String sasToken; private static final Logger LOG = LoggerFactory.getLogger(AbfsClient.class); - + private static final Logger LOG1 = LoggerFactory.getLogger(AbfsRestOperation.class); // For uploads, this is the request entity body. For downloads, // this will hold the response entity body. private byte[] buffer; private int bufferOffset; private int bufferLength; private int retryCount = 0; - + private boolean isThrottledRequest = false; + private long maxRetryCount = 0L; private AbfsHttpOperation result; private AbfsCounters abfsCounters; - private AbfsDriverMetrics abfsDriverMetrics; - private Map metricsMap; + private static Map metricsMap; /** * Checks if there is non-null HTTP response. * @return true if there is a non-null HTTP response from the ABFS call. @@ -123,6 +123,8 @@ String getSasToken() { final URL url, final List requestHeaders) { this(operationType, client, method, url, requestHeaders, null); + this.abfsDriverMetrics = abfsCounters.getAbfsDriverMetrics(); + metricsMap = abfsDriverMetrics.getMetricsMap(); } /** @@ -151,7 +153,7 @@ String getSasToken() { this.sasToken = sasToken; this.abfsCounters = client.getAbfsCounters(); this.abfsDriverMetrics = abfsCounters.getAbfsDriverMetrics(); - this.metricsMap = abfsDriverMetrics.getMetricsMap(); + metricsMap = abfsDriverMetrics.getMetricsMap(); } /** @@ -183,7 +185,7 @@ String getSasToken() { this.bufferLength = bufferLength; this.abfsCounters = client.getAbfsCounters(); this.abfsDriverMetrics = abfsCounters.getAbfsDriverMetrics(); - this.metricsMap = abfsDriverMetrics.getMetricsMap(); + metricsMap = abfsDriverMetrics.getMetricsMap(); } /** @@ -237,20 +239,11 @@ private void completeExecute(TracingContext tracingContext) Thread.currentThread().interrupt(); } } + abfsDriverMetrics.getTotalNumberOfRequests().getAndIncrement(); if(retryCount > 0) { - updateCount(retryCount); - } - if(result.getStatusCode() == HttpURLConnection.HTTP_UNAVAILABLE){ - AzureServiceErrorCode serviceErrorCode = - AzureServiceErrorCode.getAzureServiceCode(result.getStatusCode(), result.getStorageErrorCode(), result.getStorageErrorMessage()); - if(serviceErrorCode.equals(AzureServiceErrorCode.INGRESS_OVER_ACCOUNT_LIMIT) || - serviceErrorCode.equals(AzureServiceErrorCode.EGRESS_OVER_ACCOUNT_LIMIT)){ - abfsDriverMetrics.getNumberOfBandwidthThrottledRequests().getAndIncrement(); - }else if(serviceErrorCode.equals(AzureServiceErrorCode.REQUEST_OVER_ACCOUNT_LIMIT)){ - abfsDriverMetrics.getNumberOfIOPSThrottledRequests().getAndIncrement(); - }else{ - abfsDriverMetrics.getNumberOfOtherThrottledRequests().getAndIncrement(); - } + maxRetryCount = Math.max(abfsDriverMetrics.getMaxRetryCount().get(), retryCount); + abfsDriverMetrics.getMaxRetryCount().set(maxRetryCount); + updateCount(retryCount); } if (result.getStatusCode() >= HttpURLConnection.HTTP_BAD_REQUEST) { throw new AbfsRestOperationException(result.getStatusCode(), result.getStorageErrorCode(), @@ -314,7 +307,22 @@ private boolean executeHttpOperation(final int retryCount, } httpOperation.processResponse(buffer, bufferOffset, bufferLength); - incrementCounter(AbfsStatistic.GET_RESPONSES, 1); + if(!isThrottledRequest && httpOperation.getStatusCode() == HttpURLConnection.HTTP_UNAVAILABLE){ + isThrottledRequest = true; + AzureServiceErrorCode serviceErrorCode = + AzureServiceErrorCode.getAzureServiceCode(httpOperation.getStatusCode(), httpOperation.getStorageErrorCode(), httpOperation.getStorageErrorMessage()); + LOG1.trace("Service code is " + serviceErrorCode + " status code is " + httpOperation.getStatusCode() + " error code is " + httpOperation.getStorageErrorCode() + + " error message is " + httpOperation.getStorageErrorMessage()); + if(serviceErrorCode.equals(AzureServiceErrorCode.INGRESS_OVER_ACCOUNT_LIMIT) || + serviceErrorCode.equals(AzureServiceErrorCode.EGRESS_OVER_ACCOUNT_LIMIT)){ + abfsDriverMetrics.getNumberOfBandwidthThrottledRequests().getAndIncrement(); + }else if(serviceErrorCode.equals(AzureServiceErrorCode.REQUEST_OVER_ACCOUNT_LIMIT)){ + abfsDriverMetrics.getNumberOfIOPSThrottledRequests().getAndIncrement(); + }else{ + abfsDriverMetrics.getNumberOfOtherThrottledRequests().getAndIncrement(); + } + } + incrementCounter(AbfsStatistic.GET_RESPONSES, 1); //Only increment bytesReceived counter when the status code is 2XX. if (httpOperation.getStatusCode() >= HttpURLConnection.HTTP_OK && httpOperation.getStatusCode() <= HttpURLConnection.HTTP_PARTIAL) { diff --git a/hadoop-tools/hadoop-azure/src/test/resources/log4j.properties b/hadoop-tools/hadoop-azure/src/test/resources/log4j.properties index 9f72d03653306..c279febef8851 100644 --- a/hadoop-tools/hadoop-azure/src/test/resources/log4j.properties +++ b/hadoop-tools/hadoop-azure/src/test/resources/log4j.properties @@ -26,6 +26,8 @@ log4j.logger.org.apache.hadoop.fs.azure.AzureFileSystemThreadPoolExecutor=DEBUG log4j.logger.org.apache.hadoop.fs.azure.BlockBlobAppendStream=DEBUG log4j.logger.org.apache.hadoop.fs.azurebfs.contracts.services.TracingService=TRACE log4j.logger.org.apache.hadoop.fs.azurebfs.services.AbfsClient=DEBUG +log4j.logger.org.apache.hadoop.fs.azurebfs.services.AbfsRestOperation=TRACE +log4j.logger.org.apache.hadoop.fs.azurebfs.contracts.services.AzureServiceErrorCode=TRACE # after here: turn off log messages from other parts of the system # which only clutter test reports. From d57a746e2e14c4121a4841182e4ceee3c200e0d0 Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Tue, 7 Jun 2022 15:38:57 +0530 Subject: [PATCH 06/33] Checkstyle fixes --- .../hadoop/fs/azurebfs/AbfsDriverMetrics.java | 20 ++++++++++++++++++- .../fs/azurebfs/AzureBlobFileSystem.java | 4 ++-- .../fs/azurebfs/AzureBlobFileSystemStore.java | 2 +- .../azurebfs/services/AbfsRestOperation.java | 5 +---- 4 files changed, 23 insertions(+), 8 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java index 0de1895c2b224..f1986ab2a88bb 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java @@ -1,3 +1,21 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.hadoop.fs.azurebfs; import java.util.concurrent.atomic.AtomicLong; @@ -87,7 +105,7 @@ public AtomicLong getTotalNumberOfRequests() { return totalNumberOfRequests; } - public Map getMetricsMap() { + public static Map getMetricsMap() { return metricsMap; } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java index 345d3d03d642a..32612c1a9c624 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java @@ -691,12 +691,12 @@ public synchronized void close() throws IOException { URI metricUri; try { metricUri = new URI(getScheme(), abfsMetricUrl, null, null, null); - } catch (Exception ex) { + } catch (URISyntaxException ex) { throw new AssertionError(ex); } AzureBlobFileSystem metricfs = (AzureBlobFileSystem) FileSystem.newInstance(metricUri, metricConfig); metricfs.sentMetric(metric); - } catch (Exception ex) { + } catch (AzureBlobFileSystemException ex) { // do nothing } // does all the delete-on-exit calls, and may be slow. diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java index 681f991facbe2..17a2ffbabf5e2 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java @@ -1975,6 +1975,6 @@ public static String extractEtagHeader(AbfsHttpOperation result) { } public void sentMetric(String metric, TracingContext tracingContext) throws AzureBlobFileSystemException{ - AbfsRestOperation op = client.getPathStatusMetric("/..$$@@", tracingContext, metric); // Will sent a GFS calls that will fail to register in MDM x-ms-client-metric + client.getPathStatusMetric("/..$$@@", tracingContext, metric); // Will sent a GFS calls that will fail to register in MDM x-ms-client-metric } } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java index af89a9595db7f..0cb11ae922451 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java @@ -75,7 +75,7 @@ public class AbfsRestOperation { private AbfsHttpOperation result; private AbfsCounters abfsCounters; private AbfsDriverMetrics abfsDriverMetrics; - private static Map metricsMap; + private static final Map metricsMap = AbfsDriverMetrics.getMetricsMap(); /** * Checks if there is non-null HTTP response. * @return true if there is a non-null HTTP response from the ABFS call. @@ -124,7 +124,6 @@ String getSasToken() { final List requestHeaders) { this(operationType, client, method, url, requestHeaders, null); this.abfsDriverMetrics = abfsCounters.getAbfsDriverMetrics(); - metricsMap = abfsDriverMetrics.getMetricsMap(); } /** @@ -153,7 +152,6 @@ String getSasToken() { this.sasToken = sasToken; this.abfsCounters = client.getAbfsCounters(); this.abfsDriverMetrics = abfsCounters.getAbfsDriverMetrics(); - metricsMap = abfsDriverMetrics.getMetricsMap(); } /** @@ -185,7 +183,6 @@ String getSasToken() { this.bufferLength = bufferLength; this.abfsCounters = client.getAbfsCounters(); this.abfsDriverMetrics = abfsCounters.getAbfsDriverMetrics(); - metricsMap = abfsDriverMetrics.getMetricsMap(); } /** From 4476d33edf3efa40d5ca88e4557dc3a091bd0c1c Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Tue, 14 Jun 2022 09:04:37 +0530 Subject: [PATCH 07/33] Added new params to metrics --- .../hadoop/fs/azurebfs/AbfsDriverMetrics.java | 29 ++++++++++---- .../fs/azurebfs/AzureBlobFileSystem.java | 40 ++++++++++--------- .../fs/azurebfs/services/AbfsClient.java | 1 - .../azurebfs/services/AbfsRestOperation.java | 11 ++++- .../fs/azurebfs/utils/TracingContext.java | 1 - 5 files changed, 52 insertions(+), 30 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java index f1986ab2a88bb..cfeda84b2355a 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java @@ -20,7 +20,7 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.Map; -import java.util.LinkedHashMap; +import java.util.concurrent.ConcurrentHashMap; import java.util.ArrayList; import java.util.Arrays; public class AbfsDriverMetrics { @@ -35,7 +35,9 @@ public class AbfsDriverMetrics { private AtomicLong numberOfOtherThrottledRequests; private AtomicLong maxRetryCount; private AtomicLong totalNumberOfRequests; - private static final Map metricsMap = new LinkedHashMap<>(); + private AtomicLong numberOfRequestsSucceededWithoutRetrying; + private AtomicLong numberOfRequestsFailed; + private final Map metricsMap = new ConcurrentHashMap<>(); public AbfsDriverMetrics() { initializeMap(); @@ -44,6 +46,8 @@ public AbfsDriverMetrics() { this.numberOfOtherThrottledRequests = new AtomicLong(); this.totalNumberOfRequests = new AtomicLong(); this.maxRetryCount = new AtomicLong(); + this.numberOfRequestsSucceededWithoutRetrying = new AtomicLong(); + this.numberOfRequestsFailed = new AtomicLong(); } public AbfsDriverMetrics(String retryCount) { this.retryCount = retryCount; @@ -105,10 +109,18 @@ public AtomicLong getTotalNumberOfRequests() { return totalNumberOfRequests; } - public static Map getMetricsMap() { + public Map getMetricsMap() { return metricsMap; } + public AtomicLong getNumberOfRequestsSucceededWithoutRetrying() { + return numberOfRequestsSucceededWithoutRetrying; + } + + public AtomicLong getNumberOfRequestsFailed() { + return numberOfRequestsFailed; + } + @Override public String toString() { StringBuilder metricString = new StringBuilder(); @@ -135,11 +147,12 @@ public String toString() { } metricString.append(" BandwidthThrottled = ").append(numberOfBandwidthThrottledRequests) .append(" IOPSThrottled = ").append(numberOfIOPSThrottledRequests) - .append(" OtherThrottled = ").append(numberOfOtherThrottledRequests) - .append(" Total number of requests = ").append(totalNumberOfRequests); - - metricString.append(" Max retry count is ").append(maxRetryCount); - metricString.append(" Percentage of throttled requests = ").append(percentageOfRequestsThrottled); + .append(" OtherThrottled = ").append(numberOfOtherThrottledRequests).append('\n'); + metricString.append(" Percentage of throttled requests = ").append(percentageOfRequestsThrottled).append('\n'); + metricString.append(" Total number of requests which succeeded without retrying = ").append(numberOfRequestsSucceededWithoutRetrying).append('\n'); + metricString.append(" Total number of requests which failed = ").append(numberOfRequestsFailed).append('\n'); + metricString.append(" Total number of requests = ").append(totalNumberOfRequests).append('\n'); + metricString.append(" Max retry count = ").append(maxRetryCount).append('\n'); return metricString + " "; } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java index 32612c1a9c624..645bd11db354c 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java @@ -681,23 +681,27 @@ public synchronized void close() throws IOException { if (isClosed) { return; } - try { - String metric = abfsCounters.getAbfsDriverMetrics().toString(); - Configuration metricConfig = getConf(); - String metricAccountName = getConf().get(FS_AZURE_METRIC_ACCOUNT_NAME); - String metricAccountKey = getConf().get(FS_AZURE_METRIC_ACCOUNT_KEY); - final String abfsMetricUrl = "metrics" + "@" + metricAccountName; - metricConfig.set(FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME, metricAccountKey); - URI metricUri; - try { - metricUri = new URI(getScheme(), abfsMetricUrl, null, null, null); - } catch (URISyntaxException ex) { - throw new AssertionError(ex); - } - AzureBlobFileSystem metricfs = (AzureBlobFileSystem) FileSystem.newInstance(metricUri, metricConfig); - metricfs.sentMetric(metric); - } catch (AzureBlobFileSystemException ex) { - // do nothing + if(abfsCounters.getAbfsDriverMetrics().getTotalNumberOfRequests().get() > 0) { + try { + String metric = abfsCounters.getAbfsDriverMetrics().toString(); + Configuration metricConfig = getConf(); + String metricAccountName = getConf().get(FS_AZURE_METRIC_ACCOUNT_NAME); + String metricAccountKey = getConf().get(FS_AZURE_METRIC_ACCOUNT_KEY); + final String abfsMetricUrl = "metrics" + "@" + metricAccountName; + metricConfig.set(FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME, metricAccountKey); + URI metricUri; + try { + metricUri = new URI(getScheme(), abfsMetricUrl, null, null, null); + } catch (URISyntaxException ex) { + throw new AssertionError(ex); + } + AzureBlobFileSystem metricfs + = (AzureBlobFileSystem) FileSystem.newInstance(metricUri, + metricConfig); + metricfs.sentMetric(metric); + }catch (AzureBlobFileSystemException ex) { + // do nothing + } } // does all the delete-on-exit calls, and may be slow. super.close(); @@ -717,7 +721,7 @@ public synchronized void close() throws IOException { public void sentMetric(String metric) throws AzureBlobFileSystemException { TracingContext tracingContext = new TracingContext(clientCorrelationId, - fileSystemId, FSOperationType.SET_ATTR, true, tracingHeaderFormat, + fileSystemId, FSOperationType.GET_ATTR, true, tracingHeaderFormat, listener); abfsStore.sentMetric(metric, tracingContext); } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java index aaa6b384486af..d6a09c2fdf582 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java @@ -760,7 +760,6 @@ public AbfsRestOperation getPathStatusMetric(final String path, operation = SASTokenProvider.GET_STATUS_OPERATION; abfsUriQueryBuilder.addQuery(HttpQueryParams.QUERY_PARAM_UPN, String.valueOf(abfsConfiguration.isUpnUsed())); appendSASTokenToQuery(path, operation, abfsUriQueryBuilder); - final URL url = createRequestUrl(path, abfsUriQueryBuilder.toString()); final AbfsRestOperation op = new AbfsRestOperation( AbfsRestOperationType.GetPathStatus, diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java index 0cb11ae922451..ad6c9e540942c 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java @@ -75,7 +75,7 @@ public class AbfsRestOperation { private AbfsHttpOperation result; private AbfsCounters abfsCounters; private AbfsDriverMetrics abfsDriverMetrics; - private static final Map metricsMap = AbfsDriverMetrics.getMetricsMap(); + private Map metricsMap; /** * Checks if there is non-null HTTP response. * @return true if there is a non-null HTTP response from the ABFS call. @@ -124,6 +124,7 @@ String getSasToken() { final List requestHeaders) { this(operationType, client, method, url, requestHeaders, null); this.abfsDriverMetrics = abfsCounters.getAbfsDriverMetrics(); + this.metricsMap = abfsDriverMetrics.getMetricsMap(); } /** @@ -152,6 +153,7 @@ String getSasToken() { this.sasToken = sasToken; this.abfsCounters = client.getAbfsCounters(); this.abfsDriverMetrics = abfsCounters.getAbfsDriverMetrics(); + this.metricsMap = abfsDriverMetrics.getMetricsMap(); } /** @@ -183,6 +185,7 @@ String getSasToken() { this.bufferLength = bufferLength; this.abfsCounters = client.getAbfsCounters(); this.abfsDriverMetrics = abfsCounters.getAbfsDriverMetrics(); + this.metricsMap = abfsDriverMetrics.getMetricsMap(); } /** @@ -237,10 +240,14 @@ private void completeExecute(TracingContext tracingContext) } } abfsDriverMetrics.getTotalNumberOfRequests().getAndIncrement(); - if(retryCount > 0) { + if(retryCount > 0 && retryCount <= 30) { maxRetryCount = Math.max(abfsDriverMetrics.getMaxRetryCount().get(), retryCount); abfsDriverMetrics.getMaxRetryCount().set(maxRetryCount); updateCount(retryCount); + } else if(retryCount > 30){ + abfsDriverMetrics.getNumberOfRequestsFailed().getAndIncrement(); + }else{ + abfsDriverMetrics.getNumberOfRequestsSucceededWithoutRetrying().getAndIncrement(); } if (result.getStatusCode() >= HttpURLConnection.HTTP_BAD_REQUEST) { throw new AbfsRestOperationException(result.getStatusCode(), result.getStorageErrorCode(), diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/TracingContext.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/TracingContext.java index 05fe33b7a19da..85580919e910c 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/TracingContext.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/TracingContext.java @@ -113,7 +113,6 @@ public TracingContext(TracingContext originalTracingContext) { this.listener = originalTracingContext.listener.getClone(); } } - public static String validateClientCorrelationID(String clientCorrelationID) { if ((clientCorrelationID.length() > MAX_CLIENT_CORRELATION_ID_LENGTH) || (!clientCorrelationID.matches(CLIENT_CORRELATION_ID_PATTERN))) { From d7b6be8302d39ba161b6a9ed912367a0cc115cb8 Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Tue, 21 Jun 2022 16:32:21 +0530 Subject: [PATCH 08/33] Incorrect total requests fix --- .../hadoop/fs/azurebfs/AbfsDriverMetrics.java | 2 +- .../azurebfs/services/AbfsRestOperation.java | 30 ++++++++++++------- 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java index cfeda84b2355a..c3355e73da80b 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java @@ -18,9 +18,9 @@ package org.apache.hadoop.fs.azurebfs; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicLong; import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; import java.util.ArrayList; import java.util.Arrays; public class AbfsDriverMetrics { diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java index ad6c9e540942c..049034fdb5ab9 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java @@ -226,6 +226,7 @@ private void completeExecute(TracingContext tracingContext) retryCount = 0; LOG.debug("First execution of REST operation - {}", operationType); long sleepDuration = 0L; + abfsDriverMetrics.getTotalNumberOfRequests().getAndIncrement(); while (!executeHttpOperation(retryCount, tracingContext)) { try { ++retryCount; @@ -239,16 +240,7 @@ private void completeExecute(TracingContext tracingContext) Thread.currentThread().interrupt(); } } - abfsDriverMetrics.getTotalNumberOfRequests().getAndIncrement(); - if(retryCount > 0 && retryCount <= 30) { - maxRetryCount = Math.max(abfsDriverMetrics.getMaxRetryCount().get(), retryCount); - abfsDriverMetrics.getMaxRetryCount().set(maxRetryCount); - updateCount(retryCount); - } else if(retryCount > 30){ - abfsDriverMetrics.getNumberOfRequestsFailed().getAndIncrement(); - }else{ - abfsDriverMetrics.getNumberOfRequestsSucceededWithoutRetrying().getAndIncrement(); - } + updateDriverMetrics(retryCount, result.getStatusCode()); if (result.getStatusCode() >= HttpURLConnection.HTTP_BAD_REQUEST) { throw new AbfsRestOperationException(result.getStatusCode(), result.getStorageErrorCode(), result.getStorageErrorMessage(), null, result); @@ -257,6 +249,22 @@ private void completeExecute(TracingContext tracingContext) LOG.trace("{} REST operation complete", operationType); } + private void updateDriverMetrics(int retryCount, int statusCode){ + if (statusCode == HttpURLConnection.HTTP_UNAVAILABLE) { + if (retryCount == 30) { + abfsDriverMetrics.getNumberOfRequestsFailed().getAndIncrement(); + } + } else { + if (retryCount > 0 && retryCount <= 30) { + maxRetryCount = Math.max(abfsDriverMetrics.getMaxRetryCount().get(), retryCount); + abfsDriverMetrics.getMaxRetryCount().set(maxRetryCount); + updateCount(retryCount); + } else { + abfsDriverMetrics.getNumberOfRequestsSucceededWithoutRetrying().getAndIncrement(); + } + } + } + /** * Executes a single HTTP operation to complete the REST operation. If it * fails, there may be a retry. The retryCount is incremented with each @@ -341,6 +349,7 @@ private boolean executeHttpOperation(final int retryCount, LOG.warn("Unknown host name: {}. Retrying to resolve the host name...", hostname); if (!client.getRetryPolicy().shouldRetry(retryCount, -1)) { + updateDriverMetrics(retryCount, httpOperation.getStatusCode()); throw new InvalidAbfsRestOperationException(ex); } return false; @@ -350,6 +359,7 @@ private boolean executeHttpOperation(final int retryCount, } if (!client.getRetryPolicy().shouldRetry(retryCount, -1)) { + updateDriverMetrics(retryCount, httpOperation.getStatusCode()); throw new InvalidAbfsRestOperationException(ex); } From 324d2c047ec9475b3497e8681b89b7c2e2421649 Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Thu, 23 Jun 2022 10:42:32 +0530 Subject: [PATCH 09/33] Adding thread safety for metrics update --- .../hadoop/fs/azurebfs/AbfsDriverMetrics.java | 20 +++++++++---------- .../azurebfs/services/AbfsRestOperation.java | 2 +- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java index c3355e73da80b..91cdbbceab28a 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java @@ -132,7 +132,7 @@ public String toString() { .append(entry.getValue().getNumberOfRequestsSucceeded()).append(" "); long totalRequests = entry.getValue().getTotalRequests().get(); if(totalRequests > 0) { - metricString.append(" MinMaxAvg#_").append(entry.getKey()) + metricString.append(" #MinMaxAvg#_").append(entry.getKey()) .append("R_").append("=") .append(String.format("%.5f", (double) entry.getValue().getMinBackoff().get() / 1000L)) .append(" seconds ") @@ -141,18 +141,18 @@ public String toString() { .append(String.format("%.5f", (double) ((entry.getValue().getTotalBackoff().get() / totalRequests) / 1000L))) .append(" seconds "); }else { - metricString.append("MinMaxAvg#_").append(entry.getKey()) + metricString.append(" #MinMaxAvg#_").append(entry.getKey()) .append("R_").append("= 0 seconds "); } } - metricString.append(" BandwidthThrottled = ").append(numberOfBandwidthThrottledRequests) - .append(" IOPSThrottled = ").append(numberOfIOPSThrottledRequests) - .append(" OtherThrottled = ").append(numberOfOtherThrottledRequests).append('\n'); - metricString.append(" Percentage of throttled requests = ").append(percentageOfRequestsThrottled).append('\n'); - metricString.append(" Total number of requests which succeeded without retrying = ").append(numberOfRequestsSucceededWithoutRetrying).append('\n'); - metricString.append(" Total number of requests which failed = ").append(numberOfRequestsFailed).append('\n'); - metricString.append(" Total number of requests = ").append(totalNumberOfRequests).append('\n'); - metricString.append(" Max retry count = ").append(maxRetryCount).append('\n'); + metricString.append(" #BandwidthThrottled = ").append(numberOfBandwidthThrottledRequests) + .append(" #IOPSThrottled = ").append(numberOfIOPSThrottledRequests) + .append(" #OtherThrottled = ").append(numberOfOtherThrottledRequests).append('\n'); + metricString.append(" #Percentage of throttled requests = ").append(percentageOfRequestsThrottled).append('\n'); + metricString.append(" #Total number of requests which succeeded without retrying = ").append(numberOfRequestsSucceededWithoutRetrying).append('\n'); + metricString.append(" #Total number of requests which failed = ").append(numberOfRequestsFailed).append('\n'); + metricString.append(" #Total number of requests = ").append(totalNumberOfRequests).append('\n'); + metricString.append(" #Max retry count = ").append(maxRetryCount).append('\n'); return metricString + " "; } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java index 049034fdb5ab9..d9791e288e80c 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java @@ -249,7 +249,7 @@ private void completeExecute(TracingContext tracingContext) LOG.trace("{} REST operation complete", operationType); } - private void updateDriverMetrics(int retryCount, int statusCode){ + private synchronized void updateDriverMetrics(int retryCount, int statusCode){ if (statusCode == HttpURLConnection.HTTP_UNAVAILABLE) { if (retryCount == 30) { abfsDriverMetrics.getNumberOfRequestsFailed().getAndIncrement(); From b9273d3383dc1d65826093170a414d5826e55ed1 Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Tue, 5 Jul 2022 15:44:20 +0530 Subject: [PATCH 10/33] Made the metric enablement configurable --- .../hadoop/fs/azurebfs/AbfsConfiguration.java | 9 ++++ .../fs/azurebfs/AzureBlobFileSystem.java | 48 ++++++++++--------- .../fs/azurebfs/AzureBlobFileSystemStore.java | 2 +- .../azurebfs/constants/ConfigurationKeys.java | 4 +- .../constants/FileSystemConfigurations.java | 1 + .../fs/azurebfs/services/AbfsClient.java | 6 +-- .../fs/azurebfs/utils/TracingContext.java | 19 +++++++- .../azurebfs/utils/TracingHeaderFormat.java | 5 +- .../utils/TracingHeaderValidator.java | 3 ++ 9 files changed, 65 insertions(+), 32 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java index fafc30372b4a5..4ac8a821a2388 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java @@ -112,6 +112,11 @@ public class AbfsConfiguration{ DefaultValue = DEFAULT_READ_SMALL_FILES_COMPLETELY) private boolean readSmallFilesCompletely; + @BooleanConfigurationValidatorAnnotation( + ConfigurationKey = AZURE_ENABLE_METRIC_COLLECTION, + DefaultValue = DEFAULT_AZURE_ENABLE_METRIC_COLLECTION) + private boolean enableMetricCollection; + @BooleanConfigurationValidatorAnnotation( ConfigurationKey = AZURE_READ_OPTIMIZE_FOOTER_READ, DefaultValue = DEFAULT_OPTIMIZE_FOOTER_READ) @@ -689,6 +694,10 @@ public String getAppendBlobDirs() { return this.azureAppendBlobDirs; } + public boolean isMetricCollectionEnabled() { + return enableMetricCollection; + } + public String getAzureInfiniteLeaseDirs() { return this.azureInfiniteLeaseDirs; } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java index 645bd11db354c..e08eda6777e1f 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java @@ -144,6 +144,8 @@ public class AzureBlobFileSystem extends FileSystem private AbfsDelegationTokenManager delegationTokenManager; private AbfsCounters abfsCounters; private String clientCorrelationId; + + private boolean isMetricCollectionEnabled; private TracingHeaderFormat tracingHeaderFormat; private Listener listener; @@ -201,6 +203,7 @@ public void initialize(URI uri, Configuration configuration) .getAbfsConfiguration(); clientCorrelationId = TracingContext.validateClientCorrelationID( abfsConfiguration.getClientCorrelationId()); + isMetricCollectionEnabled = abfsConfiguration.isMetricCollectionEnabled(); tracingHeaderFormat = abfsConfiguration.getTracingHeaderFormat(); this.setWorkingDirectory(this.getHomeDirectory()); @@ -681,27 +684,27 @@ public synchronized void close() throws IOException { if (isClosed) { return; } + if(isMetricCollectionEnabled) { if(abfsCounters.getAbfsDriverMetrics().getTotalNumberOfRequests().get() > 0) { try { - String metric = abfsCounters.getAbfsDriverMetrics().toString(); - Configuration metricConfig = getConf(); - String metricAccountName = getConf().get(FS_AZURE_METRIC_ACCOUNT_NAME); - String metricAccountKey = getConf().get(FS_AZURE_METRIC_ACCOUNT_KEY); - final String abfsMetricUrl = "metrics" + "@" + metricAccountName; - metricConfig.set(FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME, metricAccountKey); - URI metricUri; - try { - metricUri = new URI(getScheme(), abfsMetricUrl, null, null, null); - } catch (URISyntaxException ex) { - throw new AssertionError(ex); - } - AzureBlobFileSystem metricfs - = (AzureBlobFileSystem) FileSystem.newInstance(metricUri, - metricConfig); - metricfs.sentMetric(metric); - }catch (AzureBlobFileSystemException ex) { + String metric = abfsCounters.getAbfsDriverMetrics().toString(); + Configuration metricConfig = getConf(); + String metricAccountName = getConf().get(FS_AZURE_METRIC_ACCOUNT_NAME); + String metricAccountKey = getConf().get(FS_AZURE_METRIC_ACCOUNT_KEY); + final String abfsMetricUrl = "metrics" + "@" + metricAccountName; + metricConfig.set(FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME, metricAccountKey); + URI metricUri; + try { + metricUri = new URI(getScheme(), abfsMetricUrl, null, null, null); + } catch (URISyntaxException ex) { + throw new AssertionError(ex); + } + AzureBlobFileSystem metricFs = (AzureBlobFileSystem) FileSystem.newInstance(metricUri, metricConfig); + metricFs.sentMetric(metric); + } catch (AzureBlobFileSystemException ex) { // do nothing - } + } + } } // does all the delete-on-exit calls, and may be slow. super.close(); @@ -720,10 +723,11 @@ public synchronized void close() throws IOException { } public void sentMetric(String metric) throws AzureBlobFileSystemException { - TracingContext tracingContext = new TracingContext(clientCorrelationId, - fileSystemId, FSOperationType.GET_ATTR, true, tracingHeaderFormat, - listener); - abfsStore.sentMetric(metric, tracingContext); + TracingHeaderFormat tracingHeaderFormatMetric = TracingHeaderFormat.INTERNAL_METRIC_FORMAT; + TracingContext tracingContextMetric = new TracingContext(clientCorrelationId, + fileSystemId, FSOperationType.GET_ATTR, true, tracingHeaderFormatMetric, + listener, metric); + abfsStore.sentMetric(metric, tracingContextMetric); } @Override diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java index 17a2ffbabf5e2..c5fad10f217e9 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java @@ -1975,6 +1975,6 @@ public static String extractEtagHeader(AbfsHttpOperation result) { } public void sentMetric(String metric, TracingContext tracingContext) throws AzureBlobFileSystemException{ - client.getPathStatusMetric("/..$$@@", tracingContext, metric); // Will sent a GFS calls that will fail to register in MDM x-ms-client-metric + client.getPathStatusMetric("/..$$@@", tracingContext); // Will sent a GFS calls that will fail to register in MDM x-ms-client-metric } } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java index 99963fe0cda89..a79d95d72d830 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java @@ -36,14 +36,12 @@ public final class ConfigurationKeys { */ public static final String FS_AZURE_ACCOUNT_IS_HNS_ENABLED = "fs.azure.account.hns.enabled"; public static final String FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME = "fs.azure.account.key"; - public static final String FS_AZURE_METRIC_ACCOUNT_NAME = "fs.azure.metric.account.name"; - public static final String FS_AZURE_METRIC_ACCOUNT_KEY = "fs.azure.metric.account.key"; public static final String FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME_REGX = "fs\\.azure\\.account\\.key\\.(.*)"; public static final String FS_AZURE_SECURE_MODE = "fs.azure.secure.mode"; - + public static final String AZURE_ENABLE_METRIC_COLLECTION = "fs.azure.enable.metric.collection"; // Retry strategy defined by the user public static final String AZURE_MIN_BACKOFF_INTERVAL = "fs.azure.io.retry.min.backoff.interval"; public static final String AZURE_MAX_BACKOFF_INTERVAL = "fs.azure.io.retry.max.backoff.interval"; diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java index 49d0a1fd18917..3d7263790a0fe 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java @@ -60,6 +60,7 @@ public final class FileSystemConfigurations { public static final int DEFAULT_READ_BUFFER_SIZE = 4 * ONE_MB; // 4 MB public static final boolean DEFAULT_READ_SMALL_FILES_COMPLETELY = false; public static final boolean DEFAULT_OPTIMIZE_FOOTER_READ = false; + public static final boolean DEFAULT_AZURE_ENABLE_METRIC_COLLECTION = false; public static final boolean DEFAULT_ALWAYS_READ_BUFFER_SIZE = false; public static final int DEFAULT_READ_AHEAD_BLOCK_SIZE = 4 * ONE_MB; public static final int DEFAULT_READ_AHEAD_RANGE = 64 * ONE_KB; // 64 KB diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java index d6a09c2fdf582..8f3abd00737a4 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java @@ -751,13 +751,11 @@ public AbfsRestOperation setPathProperties(final String path, final String prope } public AbfsRestOperation getPathStatusMetric(final String path, - TracingContext tracingContext, String metric) throws AzureBlobFileSystemException { + TracingContext tracingContext) throws AzureBlobFileSystemException { final List requestHeaders = createDefaultHeaders(); - requestHeaders.add(new AbfsHttpHeader(X_MS_CLIENT_REQUEST_ID, metric)); final AbfsUriQueryBuilder abfsUriQueryBuilder = createDefaultUriQueryBuilder(); - String operation = SASTokenProvider.GET_PROPERTIES_OPERATION; + String operation = SASTokenProvider.GET_STATUS_OPERATION; abfsUriQueryBuilder.addQuery(HttpQueryParams.QUERY_PARAM_ACTION, AbfsHttpConstants.GET_STATUS); - operation = SASTokenProvider.GET_STATUS_OPERATION; abfsUriQueryBuilder.addQuery(HttpQueryParams.QUERY_PARAM_UPN, String.valueOf(abfsConfiguration.isUpnUsed())); appendSASTokenToQuery(path, operation, abfsUriQueryBuilder); final URL url = createRequestUrl(path, abfsUriQueryBuilder.toString()); diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/TracingContext.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/TracingContext.java index 85580919e910c..70d4371daefe4 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/TracingContext.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/TracingContext.java @@ -63,6 +63,8 @@ public class TracingContext { //final concatenated ID list set into x-ms-client-request-id header private String header = EMPTY_STRING; + private String metricResults = EMPTY_STRING; + private static final Logger LOG = LoggerFactory.getLogger(AbfsClient.class); public static final int MAX_CLIENT_CORRELATION_ID_LENGTH = 72; public static final String CLIENT_CORRELATION_ID_PATTERN = "[a-zA-Z0-9-]*"; @@ -101,6 +103,14 @@ public TracingContext(String clientCorrelationID, String fileSystemID, } } + public TracingContext(String clientCorrelationID, String fileSystemID, + FSOperationType opType, boolean needsPrimaryReqId, + TracingHeaderFormat tracingHeaderFormat, Listener listener, + String metricResults) { + this(clientCorrelationID, fileSystemID, opType, needsPrimaryReqId, tracingHeaderFormat, listener); + this.metricResults = metricResults; + } + public TracingContext(TracingContext originalTracingContext) { this.fileSystemID = originalTracingContext.fileSystemID; this.streamID = originalTracingContext.streamID; @@ -123,6 +133,10 @@ public static String validateClientCorrelationID(String clientCorrelationID) { return clientCorrelationID; } + public String getMetricResults() { + return metricResults; + } + public void setPrimaryRequestID() { primaryRequestId = UUID.randomUUID().toString(); if (listener != null) { @@ -164,13 +178,16 @@ public void constructHeader(AbfsHttpOperation httpOperation) { case TWO_ID_FORMAT: header = clientCorrelationID + ":" + clientRequestId; break; + case INTERNAL_METRIC_FORMAT: + header = clientCorrelationID + ":" + clientRequestId + ":" + fileSystemID + ":" + metricResults; + break; default: header = clientRequestId; //case SINGLE_ID_FORMAT } if (listener != null) { //for testing listener.callTracingHeaderValidator(header, format); } - //httpOperation.setRequestProperty(HttpHeaderConfigurations.X_MS_CLIENT_REQUEST_ID, header); + httpOperation.setRequestProperty(HttpHeaderConfigurations.X_MS_CLIENT_REQUEST_ID, header); } /** diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/TracingHeaderFormat.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/TracingHeaderFormat.java index 3f23ae3ed7c14..010308e85b5b7 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/TracingHeaderFormat.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/TracingHeaderFormat.java @@ -23,6 +23,9 @@ public enum TracingHeaderFormat { TWO_ID_FORMAT, // : - ALL_ID_FORMAT; // :: + ALL_ID_FORMAT, // :: // :::: + + INTERNAL_METRIC_FORMAT; // :: + // : } diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/TracingHeaderValidator.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/TracingHeaderValidator.java index e195f1c381a94..4cd21906c077b 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/TracingHeaderValidator.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/TracingHeaderValidator.java @@ -97,6 +97,9 @@ private void validateBasicFormat(String[] idList) { } else if (format == TracingHeaderFormat.TWO_ID_FORMAT) { Assertions.assertThat(idList) .describedAs("header should have 2 elements").hasSize(2); + } else if (format == TracingHeaderFormat.INTERNAL_METRIC_FORMAT) { + Assertions.assertThat(idList) + .describedAs("header should have 4 elements").hasSize(4); } else { Assertions.assertThat(idList).describedAs("header should have 1 element") .hasSize(1); From 5fc14174136dde4dc491f7c12d345fba96ad42fb Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Wed, 6 Jul 2022 14:17:02 +0530 Subject: [PATCH 11/33] Removed hardcoded value for maxIoRetries --- .../hadoop/fs/azurebfs/services/AbfsClient.java | 8 ++++++++ .../fs/azurebfs/services/AbfsRestOperation.java | 11 ++++------- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java index 8f3abd00737a4..27d6e99b4db36 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java @@ -1243,6 +1243,14 @@ protected AbfsCounters getAbfsCounters() { return abfsCounters; } + /** + * Getter for abfsConfiguration from AbfsClient. + * @return AbfsConfiguration instance + */ + protected AbfsConfiguration getAbfsConfiguration() { + return abfsConfiguration; + } + public int getNumLeaseThreads() { return abfsConfiguration.getNumLeaseThreads(); } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java index d9791e288e80c..ec4c8ab7c6b4d 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java @@ -72,6 +72,7 @@ public class AbfsRestOperation { private int retryCount = 0; private boolean isThrottledRequest = false; private long maxRetryCount = 0L; + private int maxIoRetries = 0; private AbfsHttpOperation result; private AbfsCounters abfsCounters; private AbfsDriverMetrics abfsDriverMetrics; @@ -123,8 +124,6 @@ String getSasToken() { final URL url, final List requestHeaders) { this(operationType, client, method, url, requestHeaders, null); - this.abfsDriverMetrics = abfsCounters.getAbfsDriverMetrics(); - this.metricsMap = abfsDriverMetrics.getMetricsMap(); } /** @@ -154,6 +153,7 @@ String getSasToken() { this.abfsCounters = client.getAbfsCounters(); this.abfsDriverMetrics = abfsCounters.getAbfsDriverMetrics(); this.metricsMap = abfsDriverMetrics.getMetricsMap(); + this.maxIoRetries = client.getAbfsConfiguration().getMaxIoRetries(); } /** @@ -183,9 +183,6 @@ String getSasToken() { this.buffer = buffer; this.bufferOffset = bufferOffset; this.bufferLength = bufferLength; - this.abfsCounters = client.getAbfsCounters(); - this.abfsDriverMetrics = abfsCounters.getAbfsDriverMetrics(); - this.metricsMap = abfsDriverMetrics.getMetricsMap(); } /** @@ -251,11 +248,11 @@ private void completeExecute(TracingContext tracingContext) private synchronized void updateDriverMetrics(int retryCount, int statusCode){ if (statusCode == HttpURLConnection.HTTP_UNAVAILABLE) { - if (retryCount == 30) { + if (retryCount >= maxIoRetries) { abfsDriverMetrics.getNumberOfRequestsFailed().getAndIncrement(); } } else { - if (retryCount > 0 && retryCount <= 30) { + if (retryCount > 0 && retryCount <= maxIoRetries) { maxRetryCount = Math.max(abfsDriverMetrics.getMaxRetryCount().get(), retryCount); abfsDriverMetrics.getMaxRetryCount().set(maxRetryCount); updateCount(retryCount); From b185269396001ffa2842d1b18e78da954c719751 Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Thu, 7 Jul 2022 14:23:16 +0530 Subject: [PATCH 12/33] Fix null pointer exceptions --- .../fs/azurebfs/AzureBlobFileSystem.java | 6 +++--- .../azurebfs/services/AbfsRestOperation.java | 19 ++++++++++++++----- .../azurebfs/ITestAbfsReadWriteAndSeek.java | 13 ++++--------- .../fs/azurebfs/services/TestAbfsClient.java | 2 +- 4 files changed, 22 insertions(+), 18 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java index e08eda6777e1f..ffe0e46d16986 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java @@ -689,8 +689,8 @@ public synchronized void close() throws IOException { try { String metric = abfsCounters.getAbfsDriverMetrics().toString(); Configuration metricConfig = getConf(); - String metricAccountName = getConf().get(FS_AZURE_METRIC_ACCOUNT_NAME); - String metricAccountKey = getConf().get(FS_AZURE_METRIC_ACCOUNT_KEY); + String metricAccountName = metricConfig.get(FS_AZURE_METRIC_ACCOUNT_NAME); + String metricAccountKey = metricConfig.get(FS_AZURE_METRIC_ACCOUNT_KEY); final String abfsMetricUrl = "metrics" + "@" + metricAccountName; metricConfig.set(FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME, metricAccountKey); URI metricUri; @@ -702,7 +702,7 @@ public synchronized void close() throws IOException { AzureBlobFileSystem metricFs = (AzureBlobFileSystem) FileSystem.newInstance(metricUri, metricConfig); metricFs.sentMetric(metric); } catch (AzureBlobFileSystemException ex) { - // do nothing + //do nothing } } } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java index ec4c8ab7c6b4d..d68b4093ed8e9 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java @@ -151,8 +151,11 @@ String getSasToken() { || AbfsHttpConstants.HTTP_METHOD_PATCH.equals(method)); this.sasToken = sasToken; this.abfsCounters = client.getAbfsCounters(); - this.abfsDriverMetrics = abfsCounters.getAbfsDriverMetrics(); - this.metricsMap = abfsDriverMetrics.getMetricsMap(); + if(abfsCounters != null) { + this.abfsDriverMetrics = abfsCounters.getAbfsDriverMetrics(); + }if(abfsDriverMetrics != null) { + this.metricsMap = abfsDriverMetrics.getMetricsMap(); + } this.maxIoRetries = client.getAbfsConfiguration().getMaxIoRetries(); } @@ -223,7 +226,9 @@ private void completeExecute(TracingContext tracingContext) retryCount = 0; LOG.debug("First execution of REST operation - {}", operationType); long sleepDuration = 0L; - abfsDriverMetrics.getTotalNumberOfRequests().getAndIncrement(); + if(abfsDriverMetrics != null) { + abfsDriverMetrics.getTotalNumberOfRequests().getAndIncrement(); + } while (!executeHttpOperation(retryCount, tracingContext)) { try { ++retryCount; @@ -231,13 +236,17 @@ private void completeExecute(TracingContext tracingContext) LOG.debug("Retrying REST operation {}. RetryCount = {}", operationType, retryCount); sleepDuration = client.getRetryPolicy().getRetryInterval(retryCount); - updateTimeMetrics(retryCount, sleepDuration); + if(abfsDriverMetrics != null) { + updateTimeMetrics(retryCount, sleepDuration); + } Thread.sleep(sleepDuration); } catch (InterruptedException ex) { Thread.currentThread().interrupt(); } } - updateDriverMetrics(retryCount, result.getStatusCode()); + if(abfsDriverMetrics != null) { + updateDriverMetrics(retryCount, result.getStatusCode()); + } if (result.getStatusCode() >= HttpURLConnection.HTTP_BAD_REQUEST) { throw new AbfsRestOperationException(result.getStatusCode(), result.getStorageErrorCode(), result.getStorageErrorMessage(), null, result); diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsReadWriteAndSeek.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsReadWriteAndSeek.java index ff60306dac9b5..73c8dc8d62068 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsReadWriteAndSeek.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsReadWriteAndSeek.java @@ -51,16 +51,11 @@ public class ITestAbfsReadWriteAndSeek extends AbstractAbfsScaleTest { private static final String TEST_PATH = "/testfile"; // @Parameterized.Parameters(name = "Size={0}") -// public static Iterable sizes() { -// return Arrays.asList(new Object[][]{{MIN_BUFFER_SIZE}, -// {DEFAULT_READ_BUFFER_SIZE}, -// {APPENDBLOB_MAX_WRITE_BUFFER_SIZE}, -// {MAX_BUFFER_SIZE}}); -// } - - @Parameterized.Parameters(name = "Size={0}") public static Iterable sizes() { - return Arrays.asList(new Object[][]{{MIN_BUFFER_SIZE}}); + return Arrays.asList(new Object[][]{{MIN_BUFFER_SIZE}, + {DEFAULT_READ_BUFFER_SIZE}, + {APPENDBLOB_MAX_WRITE_BUFFER_SIZE}, + {MAX_BUFFER_SIZE}}); } private final int size; diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsClient.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsClient.java index a725bf3175a5c..68907d20b02fb 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsClient.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsClient.java @@ -306,7 +306,7 @@ public static AbfsClient getMockAbfsClient(AbfsClient baseAbfsClientInstance, when(client.getAccessToken()).thenCallRealMethod(); when(client.getSharedKeyCredentials()).thenCallRealMethod(); when(client.createDefaultHeaders()).thenCallRealMethod(); - + when(client.getAbfsConfiguration()).thenReturn(abfsConfig); // override baseurl client = TestAbfsClient.setAbfsClientField(client, "abfsConfiguration", abfsConfig); From ef9691a29d3889b7d89c6a884fb0e13a38490efe Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Fri, 8 Jul 2022 17:02:55 +0530 Subject: [PATCH 13/33] Fix for acronyms --- .../hadoop/fs/azurebfs/AbfsDriverMetrics.java | 44 ++++++++++++------- .../azurebfs/services/AbfsRestOperation.java | 2 +- .../azurebfs/ITestAbfsReadWriteAndSeek.java | 2 +- 3 files changed, 31 insertions(+), 17 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java index 91cdbbceab28a..6d95d61c7b11f 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java @@ -59,7 +59,7 @@ public AbfsDriverMetrics(String retryCount) { } private void initializeMap() { - ArrayList retryCountList = new ArrayList(Arrays.asList("1","2","3","4","5_15","15_25","25_30")); + ArrayList retryCountList = new ArrayList(Arrays.asList("1","2","3","4","5_15","15_25","25andabove")); for (String s : retryCountList) { metricsMap.put(s, new AbfsDriverMetrics(s)); } @@ -121,6 +121,20 @@ public AtomicLong getNumberOfRequestsFailed() { return numberOfRequestsFailed; } + /* + Acronyms :- + 1.RCTSI :- Request count that succeeded in x retries + 2.MMA :- Min Max Average (This refers to the backoff or sleep time between 2 requests) + 3.s :- seconds + 4.BWT :- Number of Bandwidth throttled requests + 5.IT :- Number of IOPS throttled requests + 6.OT :- Number of Other throttled requests + 7.%RT :- Percentage of requests that are throttled + 8.TRNR :- Total number of requests which succeeded without retrying + 9.TRF :- Total number of requests which failed + 10.TR :- Total number of requests which were made + 11.MRC :- Max retry count across all requests + */ @Override public String toString() { StringBuilder metricString = new StringBuilder(); @@ -132,27 +146,27 @@ public String toString() { .append(entry.getValue().getNumberOfRequestsSucceeded()).append(" "); long totalRequests = entry.getValue().getTotalRequests().get(); if(totalRequests > 0) { - metricString.append(" #MinMaxAvg#_").append(entry.getKey()) + metricString.append(" #MMA#_").append(entry.getKey()) .append("R_").append("=") .append(String.format("%.5f", (double) entry.getValue().getMinBackoff().get() / 1000L)) - .append(" seconds ") + .append(" s ") .append(String.format("%.5f", (double) entry.getValue().getMaxBackoff().get() / 1000L)) - .append(" seconds ") + .append(" s ") .append(String.format("%.5f", (double) ((entry.getValue().getTotalBackoff().get() / totalRequests) / 1000L))) - .append(" seconds "); + .append(" s "); }else { - metricString.append(" #MinMaxAvg#_").append(entry.getKey()) - .append("R_").append("= 0 seconds "); + metricString.append(" #MMA#_").append(entry.getKey()) + .append("R_").append("= 0 s "); } } - metricString.append(" #BandwidthThrottled = ").append(numberOfBandwidthThrottledRequests) - .append(" #IOPSThrottled = ").append(numberOfIOPSThrottledRequests) - .append(" #OtherThrottled = ").append(numberOfOtherThrottledRequests).append('\n'); - metricString.append(" #Percentage of throttled requests = ").append(percentageOfRequestsThrottled).append('\n'); - metricString.append(" #Total number of requests which succeeded without retrying = ").append(numberOfRequestsSucceededWithoutRetrying).append('\n'); - metricString.append(" #Total number of requests which failed = ").append(numberOfRequestsFailed).append('\n'); - metricString.append(" #Total number of requests = ").append(totalNumberOfRequests).append('\n'); - metricString.append(" #Max retry count = ").append(maxRetryCount).append('\n'); + metricString.append(" #BWT=").append(numberOfBandwidthThrottledRequests) + .append(" #IT=").append(numberOfIOPSThrottledRequests) + .append(" #OT=").append(numberOfOtherThrottledRequests).append('\n'); + metricString.append(" #%RT=").append(percentageOfRequestsThrottled).append('\n'); + metricString.append(" #TRNR=").append(numberOfRequestsSucceededWithoutRetrying).append('\n'); + metricString.append(" #TRF=").append(numberOfRequestsFailed).append('\n'); + metricString.append(" #TR=").append(totalNumberOfRequests).append('\n'); + metricString.append(" #MRC = ").append(maxRetryCount).append('\n'); return metricString + " "; } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java index d68b4093ed8e9..bd6abe8dc2532 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java @@ -423,7 +423,7 @@ private String getKey(int retryCount) { }else if(retryCount >= 15 && retryCount < 25){ retryCounter = "15_25"; }else{ - retryCounter = "25_30"; + retryCounter = "25andabove"; } return retryCounter; } diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsReadWriteAndSeek.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsReadWriteAndSeek.java index 73c8dc8d62068..5bd6eaff42e84 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsReadWriteAndSeek.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsReadWriteAndSeek.java @@ -50,7 +50,7 @@ public class ITestAbfsReadWriteAndSeek extends AbstractAbfsScaleTest { private static final String TEST_PATH = "/testfile"; -// @Parameterized.Parameters(name = "Size={0}") + @Parameterized.Parameters(name = "Size={0}") public static Iterable sizes() { return Arrays.asList(new Object[][]{{MIN_BUFFER_SIZE}, {DEFAULT_READ_BUFFER_SIZE}, From 3bddb11b6dbe58f5065973e25d8dc3ac677da135 Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Mon, 11 Jul 2022 12:51:31 +0530 Subject: [PATCH 14/33] Fix for spacing issues --- .../hadoop/fs/azurebfs/AbfsDriverMetrics.java | 22 +++++++++---------- .../azurebfs/services/AbfsRestOperation.java | 2 +- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java index 6d95d61c7b11f..3558f7cae3d05 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java @@ -149,24 +149,24 @@ public String toString() { metricString.append(" #MMA#_").append(entry.getKey()) .append("R_").append("=") .append(String.format("%.5f", (double) entry.getValue().getMinBackoff().get() / 1000L)) - .append(" s ") + .append("s ") .append(String.format("%.5f", (double) entry.getValue().getMaxBackoff().get() / 1000L)) - .append(" s ") + .append("s ") .append(String.format("%.5f", (double) ((entry.getValue().getTotalBackoff().get() / totalRequests) / 1000L))) - .append(" s "); + .append("s "); }else { metricString.append(" #MMA#_").append(entry.getKey()) - .append("R_").append("= 0 s "); + .append("R_").append("=0s "); } } - metricString.append(" #BWT=").append(numberOfBandwidthThrottledRequests) + metricString.append("#BWT=").append(numberOfBandwidthThrottledRequests) .append(" #IT=").append(numberOfIOPSThrottledRequests) - .append(" #OT=").append(numberOfOtherThrottledRequests).append('\n'); - metricString.append(" #%RT=").append(percentageOfRequestsThrottled).append('\n'); - metricString.append(" #TRNR=").append(numberOfRequestsSucceededWithoutRetrying).append('\n'); - metricString.append(" #TRF=").append(numberOfRequestsFailed).append('\n'); - metricString.append(" #TR=").append(totalNumberOfRequests).append('\n'); - metricString.append(" #MRC = ").append(maxRetryCount).append('\n'); + .append(" #OT=").append(numberOfOtherThrottledRequests); + metricString.append(" #%RT=").append(String.format("%.3f",percentageOfRequestsThrottled)); + metricString.append(" #TRNR=").append(numberOfRequestsSucceededWithoutRetrying); + metricString.append(" #TRF=").append(numberOfRequestsFailed); + metricString.append(" #TR=").append(totalNumberOfRequests); + metricString.append(" #MRC=").append(maxRetryCount); return metricString + " "; } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java index bd6abe8dc2532..d8ca9db178edb 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java @@ -74,7 +74,7 @@ public class AbfsRestOperation { private long maxRetryCount = 0L; private int maxIoRetries = 0; private AbfsHttpOperation result; - private AbfsCounters abfsCounters; + private final AbfsCounters abfsCounters; private AbfsDriverMetrics abfsDriverMetrics; private Map metricsMap; /** From 4afba696af035940577f99a136af75af7ce25787 Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Tue, 12 Jul 2022 09:52:34 +0530 Subject: [PATCH 15/33] Added documentation for configs --- .../apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java | 12 ++++++------ .../hadoop/fs/azurebfs/AzureBlobFileSystem.java | 10 +++++----- .../hadoop/fs/azurebfs/AzureBlobFileSystemStore.java | 2 +- hadoop-tools/hadoop-azure/src/site/markdown/abfs.md | 12 ++++++++++++ 4 files changed, 24 insertions(+), 12 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java index 3558f7cae3d05..029b0eff56ea1 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java @@ -141,21 +141,21 @@ public String toString() { long totalRequestsThrottled = numberOfBandwidthThrottledRequests.get() + numberOfIOPSThrottledRequests.get() + numberOfOtherThrottledRequests.get(); double percentageOfRequestsThrottled = ((double)totalRequestsThrottled/totalNumberOfRequests.get())*100; for (Map.Entry entry : metricsMap.entrySet()) { - metricString.append(" #RCTSI#_").append(entry.getKey()) + metricString.append("#RCTSI#_").append(entry.getKey()) .append("R_").append("=") .append(entry.getValue().getNumberOfRequestsSucceeded()).append(" "); long totalRequests = entry.getValue().getTotalRequests().get(); if(totalRequests > 0) { - metricString.append(" #MMA#_").append(entry.getKey()) + metricString.append("#MMA#_").append(entry.getKey()) .append("R_").append("=") - .append(String.format("%.5f", (double) entry.getValue().getMinBackoff().get() / 1000L)) + .append(String.format("%.3f", (double) entry.getValue().getMinBackoff().get() / 1000L)) .append("s ") - .append(String.format("%.5f", (double) entry.getValue().getMaxBackoff().get() / 1000L)) + .append(String.format("%.3f", (double) entry.getValue().getMaxBackoff().get() / 1000L)) .append("s ") - .append(String.format("%.5f", (double) ((entry.getValue().getTotalBackoff().get() / totalRequests) / 1000L))) + .append(String.format("%.3f", (double) ((entry.getValue().getTotalBackoff().get() / totalRequests) / 1000L))) .append("s "); }else { - metricString.append(" #MMA#_").append(entry.getKey()) + metricString.append("#MMA#_").append(entry.getKey()) .append("R_").append("=0s "); } } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java index ffe0e46d16986..0918e25886a6c 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java @@ -144,7 +144,6 @@ public class AzureBlobFileSystem extends FileSystem private AbfsDelegationTokenManager delegationTokenManager; private AbfsCounters abfsCounters; private String clientCorrelationId; - private boolean isMetricCollectionEnabled; private TracingHeaderFormat tracingHeaderFormat; private Listener listener; @@ -684,10 +683,11 @@ public synchronized void close() throws IOException { if (isClosed) { return; } + String metric = abfsCounters.getAbfsDriverMetrics().toString(); + LOG.debug("The metrics collected over this instance are " + metric); if(isMetricCollectionEnabled) { if(abfsCounters.getAbfsDriverMetrics().getTotalNumberOfRequests().get() > 0) { try { - String metric = abfsCounters.getAbfsDriverMetrics().toString(); Configuration metricConfig = getConf(); String metricAccountName = metricConfig.get(FS_AZURE_METRIC_ACCOUNT_NAME); String metricAccountKey = metricConfig.get(FS_AZURE_METRIC_ACCOUNT_KEY); @@ -700,7 +700,7 @@ public synchronized void close() throws IOException { throw new AssertionError(ex); } AzureBlobFileSystem metricFs = (AzureBlobFileSystem) FileSystem.newInstance(metricUri, metricConfig); - metricFs.sentMetric(metric); + metricFs.sendMetric(metric); } catch (AzureBlobFileSystemException ex) { //do nothing } @@ -722,12 +722,12 @@ public synchronized void close() throws IOException { } } - public void sentMetric(String metric) throws AzureBlobFileSystemException { + public void sendMetric(String metric) throws AzureBlobFileSystemException { TracingHeaderFormat tracingHeaderFormatMetric = TracingHeaderFormat.INTERNAL_METRIC_FORMAT; TracingContext tracingContextMetric = new TracingContext(clientCorrelationId, fileSystemId, FSOperationType.GET_ATTR, true, tracingHeaderFormatMetric, listener, metric); - abfsStore.sentMetric(metric, tracingContextMetric); + abfsStore.sendMetric(metric, tracingContextMetric); } @Override diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java index c5fad10f217e9..e45fa3141604d 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java @@ -1974,7 +1974,7 @@ public static String extractEtagHeader(AbfsHttpOperation result) { return etag; } - public void sentMetric(String metric, TracingContext tracingContext) throws AzureBlobFileSystemException{ + public void sendMetric(String metric, TracingContext tracingContext) throws AzureBlobFileSystemException{ client.getPathStatusMetric("/..$$@@", tracingContext); // Will sent a GFS calls that will fail to register in MDM x-ms-client-metric } } diff --git a/hadoop-tools/hadoop-azure/src/site/markdown/abfs.md b/hadoop-tools/hadoop-azure/src/site/markdown/abfs.md index 35d360556047e..46bef8546ae98 100644 --- a/hadoop-tools/hadoop-azure/src/site/markdown/abfs.md +++ b/hadoop-tools/hadoop-azure/src/site/markdown/abfs.md @@ -991,6 +991,18 @@ Note that these performance numbers are also sent back to the ADLS Gen 2 API end in the `x-ms-abfs-client-latency` HTTP headers in subsequent requests. Azure uses these settings to track their end-to-end latency. +### Driver Metric Options + +`fs.azure.metric.account.name`: This configuration parameter is used to specify +the name of the account which will be used to push a failed(404) GetPathStatus request to the +backend. We can configure a separate account to push metrics to the store or use the same for as the existing account on which other requests are made. + +`fs.azure.metric.account.key`: This is the access key for the storage account +used for pushing metrics to the store. + +`fs.azure.enable.metric.collection`: This configuration provides an option to specify whether we want to push the metrics or not. +By default, this config will be set to false. + ## Troubleshooting The problems associated with the connector usually come down to, in order From 0edb3687cf8c943a44ccf572f951f2d0d0a15e96 Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Tue, 12 Jul 2022 09:59:33 +0530 Subject: [PATCH 16/33] Update variable name --- .../org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java index 0918e25886a6c..c19754b4746a4 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java @@ -144,7 +144,7 @@ public class AzureBlobFileSystem extends FileSystem private AbfsDelegationTokenManager delegationTokenManager; private AbfsCounters abfsCounters; private String clientCorrelationId; - private boolean isMetricCollectionEnabled; + private boolean sendMetricsToStore; private TracingHeaderFormat tracingHeaderFormat; private Listener listener; @@ -202,7 +202,7 @@ public void initialize(URI uri, Configuration configuration) .getAbfsConfiguration(); clientCorrelationId = TracingContext.validateClientCorrelationID( abfsConfiguration.getClientCorrelationId()); - isMetricCollectionEnabled = abfsConfiguration.isMetricCollectionEnabled(); + sendMetricsToStore = abfsConfiguration.isMetricCollectionEnabled(); tracingHeaderFormat = abfsConfiguration.getTracingHeaderFormat(); this.setWorkingDirectory(this.getHomeDirectory()); @@ -685,7 +685,7 @@ public synchronized void close() throws IOException { } String metric = abfsCounters.getAbfsDriverMetrics().toString(); LOG.debug("The metrics collected over this instance are " + metric); - if(isMetricCollectionEnabled) { + if(sendMetricsToStore) { if(abfsCounters.getAbfsDriverMetrics().getTotalNumberOfRequests().get() > 0) { try { Configuration metricConfig = getConf(); From 791b84e60254286b72bd8539fc8f76df486ef814 Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Tue, 12 Jul 2022 12:42:53 +0530 Subject: [PATCH 17/33] Added config for URI --- .../fs/azurebfs/AzureBlobFileSystem.java | 11 ++++++++--- .../fs/azurebfs/AzureBlobFileSystemStore.java | 2 +- .../azurebfs/constants/ConfigurationKeys.java | 1 + .../fs/azurebfs/services/AbfsClient.java | 19 ------------------- .../hadoop-azure/src/site/markdown/abfs.md | 3 +++ 5 files changed, 13 insertions(+), 23 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java index c19754b4746a4..9b3e458656796 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java @@ -49,7 +49,6 @@ import org.apache.hadoop.util.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.concurrent.atomic.AtomicLong; import org.apache.commons.lang3.ArrayUtils; import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.classification.InterfaceAudience; @@ -113,14 +112,17 @@ import static org.apache.hadoop.fs.CommonConfigurationKeys.IOSTATISTICS_LOGGING_LEVEL; import static org.apache.hadoop.fs.CommonConfigurationKeys.IOSTATISTICS_LOGGING_LEVEL_DEFAULT; import static org.apache.hadoop.fs.azurebfs.AbfsStatistic.*; +import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.DATA_BLOCKS_BUFFER; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_BLOCK_UPLOAD_ACTIVE_BLOCKS; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_BLOCK_UPLOAD_BUFFER_DIR; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_METRIC_ACCOUNT_KEY; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_METRIC_ACCOUNT_NAME; +import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_METRIC_URI; import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.BLOCK_UPLOAD_ACTIVE_BLOCKS_DEFAULT; import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DATA_BLOCKS_BUFFER_DEFAULT; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import static org.apache.hadoop.fs.impl.PathCapabilitiesSupport.validatePathCapabilityArgs; import static org.apache.hadoop.fs.statistics.IOStatisticsLogging.logIOStatisticsAtLevel; import static org.apache.hadoop.util.functional.RemoteIterators.filteringRemoteIterator; @@ -689,10 +691,13 @@ public synchronized void close() throws IOException { if(abfsCounters.getAbfsDriverMetrics().getTotalNumberOfRequests().get() > 0) { try { Configuration metricConfig = getConf(); - String metricAccountName = metricConfig.get(FS_AZURE_METRIC_ACCOUNT_NAME); String metricAccountKey = metricConfig.get(FS_AZURE_METRIC_ACCOUNT_KEY); - final String abfsMetricUrl = "metrics" + "@" + metricAccountName; + final String abfsMetricUrl = metricConfig.get(FS_AZURE_METRIC_URI); + if(abfsMetricUrl == null){ + return; + } metricConfig.set(FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME, metricAccountKey); + metricConfig.set(AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION, "false"); URI metricUri; try { metricUri = new URI(getScheme(), abfsMetricUrl, null, null, null); diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java index e45fa3141604d..658291e83e135 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java @@ -1975,6 +1975,6 @@ public static String extractEtagHeader(AbfsHttpOperation result) { } public void sendMetric(String metric, TracingContext tracingContext) throws AzureBlobFileSystemException{ - client.getPathStatusMetric("/..$$@@", tracingContext); // Will sent a GFS calls that will fail to register in MDM x-ms-client-metric + client.getPathStatus("/..$$@@", true, tracingContext); // Will sent a GFS calls that will fail to register in MDM x-ms-client-metric } } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java index a79d95d72d830..eddb0ef9a5391 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java @@ -38,6 +38,7 @@ public final class ConfigurationKeys { public static final String FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME = "fs.azure.account.key"; public static final String FS_AZURE_METRIC_ACCOUNT_NAME = "fs.azure.metric.account.name"; public static final String FS_AZURE_METRIC_ACCOUNT_KEY = "fs.azure.metric.account.key"; + public static final String FS_AZURE_METRIC_URI = "fs.azure.metric.uri"; public static final String FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME_REGX = "fs\\.azure\\.account\\.key\\.(.*)"; public static final String FS_AZURE_SECURE_MODE = "fs.azure.secure.mode"; diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java index 27d6e99b4db36..3b351a53205c5 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java @@ -750,25 +750,6 @@ public AbfsRestOperation setPathProperties(final String path, final String prope return op; } - public AbfsRestOperation getPathStatusMetric(final String path, - TracingContext tracingContext) throws AzureBlobFileSystemException { - final List requestHeaders = createDefaultHeaders(); - final AbfsUriQueryBuilder abfsUriQueryBuilder = createDefaultUriQueryBuilder(); - String operation = SASTokenProvider.GET_STATUS_OPERATION; - abfsUriQueryBuilder.addQuery(HttpQueryParams.QUERY_PARAM_ACTION, AbfsHttpConstants.GET_STATUS); - abfsUriQueryBuilder.addQuery(HttpQueryParams.QUERY_PARAM_UPN, String.valueOf(abfsConfiguration.isUpnUsed())); - appendSASTokenToQuery(path, operation, abfsUriQueryBuilder); - final URL url = createRequestUrl(path, abfsUriQueryBuilder.toString()); - final AbfsRestOperation op = new AbfsRestOperation( - AbfsRestOperationType.GetPathStatus, - this, - HTTP_METHOD_HEAD, - url, - requestHeaders); - op.execute(tracingContext); - return op; - } - public AbfsRestOperation getPathStatus(final String path, final boolean includeProperties, TracingContext tracingContext) throws AzureBlobFileSystemException { final List requestHeaders = createDefaultHeaders(); diff --git a/hadoop-tools/hadoop-azure/src/site/markdown/abfs.md b/hadoop-tools/hadoop-azure/src/site/markdown/abfs.md index 46bef8546ae98..12e2bc6a5d7c6 100644 --- a/hadoop-tools/hadoop-azure/src/site/markdown/abfs.md +++ b/hadoop-tools/hadoop-azure/src/site/markdown/abfs.md @@ -1003,6 +1003,9 @@ used for pushing metrics to the store. `fs.azure.enable.metric.collection`: This configuration provides an option to specify whether we want to push the metrics or not. By default, this config will be set to false. +`fs.azure.metric.uri`: This configuration provides the uri in the format of containername@accountname.dfs.core.windows.net. +This should be a part of the config in order to prevent extra calls to create the filesystem. We use an existing filsystem to push the metrics. + ## Troubleshooting The problems associated with the connector usually come down to, in order From f6bad55ee40526b658934f8191d578c6691a4de0 Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Wed, 13 Jul 2022 10:11:53 +0530 Subject: [PATCH 18/33] Test for max retries --- .../ITestAzureBlobFileSystemDelete.java | 35 +++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelete.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelete.java index db181fb5dd660..fc4059d3d7908 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelete.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelete.java @@ -45,10 +45,12 @@ import org.apache.hadoop.fs.Path; import static java.net.HttpURLConnection.HTTP_BAD_REQUEST; +import static java.net.HttpURLConnection.HTTP_INTERNAL_ERROR; import static java.net.HttpURLConnection.HTTP_NOT_FOUND; import static java.net.HttpURLConnection.HTTP_OK; - +import org.mockito.Mockito; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.doCallRealMethod; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; @@ -61,7 +63,6 @@ import static org.apache.hadoop.fs.contract.ContractTestUtils.assertPathDoesNotExist; import static org.apache.hadoop.test.LambdaTestUtils.intercept; - /** * Test delete operation. */ @@ -219,6 +220,36 @@ public void testDeleteIdempotency() throws Exception { } + @Test + public void testTryStatus() throws java.io.IOException { + AbfsConfiguration abfsConfig + = TestAbfsConfigurationFieldsValidation.updateRetryConfigs( + getConfiguration(), + 5, 5000); + + final AzureBlobFileSystem fs = this.getFileSystem(); + AbfsClient abfsClient = fs.getAbfsStore().getClient(); + AbfsClient testClient = TestAbfsClient.createTestClientFromCurrentContext( + abfsClient, + abfsConfig); + + final AbfsRestOperation mockRestOperation = mock(AbfsRestOperation.class); + when(mockRestOperation.isARetriedRequest()).thenReturn(true); + + // Case 1: Mock instance of Http Operation response. This will return + // HTTP:Not Found + AbfsHttpOperation httpOperation = mock(AbfsHttpOperation.class); + when(httpOperation.getStatusCode()).thenReturn(HTTP_INTERNAL_ERROR); + + // Mock delete response to 404 + when(mockRestOperation.getResult()).thenReturn(httpOperation); + when(mockRestOperation.hasResult()).thenReturn(true); + TracingContext tracingContext = new TracingContext("abcd", + fs.getFileSystemId(), FSOperationType.TEST_OP, + org.apache.hadoop.fs.azurebfs.utils.TracingHeaderFormat.ALL_ID_FORMAT, null); + fs.close(); + } + @Test public void testDeleteIdempotencyTriggerHttp404() throws Exception { From 9230bd8792968a732a8baabc4a650a27c0dce475 Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Wed, 13 Jul 2022 16:28:55 +0530 Subject: [PATCH 19/33] Add test case for retry --- .../hadoop/fs/azurebfs/AbfsDriverMetrics.java | 48 +++++++++++-------- .../fs/azurebfs/AzureBlobFileSystem.java | 2 - .../fs/azurebfs/services/AbfsClient.java | 2 +- .../azurebfs/services/AbfsRestOperation.java | 10 ++-- .../ITestAzureBlobFileSystemDelete.java | 35 +++++++++++++- .../fs/azurebfs/TestAbfsRestOperation.java | 41 ++++++++++++++++ 6 files changed, 109 insertions(+), 29 deletions(-) create mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestAbfsRestOperation.java diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java index 029b0eff56ea1..1826f1bd0f210 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java @@ -33,6 +33,7 @@ public class AbfsDriverMetrics { private AtomicLong numberOfIOPSThrottledRequests; private AtomicLong numberOfBandwidthThrottledRequests; private AtomicLong numberOfOtherThrottledRequests; + private AtomicLong numberOfNetworkFailedRequests; private AtomicLong maxRetryCount; private AtomicLong totalNumberOfRequests; private AtomicLong numberOfRequestsSucceededWithoutRetrying; @@ -48,6 +49,7 @@ public AbfsDriverMetrics() { this.maxRetryCount = new AtomicLong(); this.numberOfRequestsSucceededWithoutRetrying = new AtomicLong(); this.numberOfRequestsFailed = new AtomicLong(); + this.numberOfNetworkFailedRequests = new AtomicLong(); } public AbfsDriverMetrics(String retryCount) { this.retryCount = retryCount; @@ -59,7 +61,7 @@ public AbfsDriverMetrics(String retryCount) { } private void initializeMap() { - ArrayList retryCountList = new ArrayList(Arrays.asList("1","2","3","4","5_15","15_25","25andabove")); + ArrayList retryCountList = new ArrayList(Arrays.asList("1","2","3","4","5_15","15_25","25AndAbove")); for (String s : retryCountList) { metricsMap.put(s, new AbfsDriverMetrics(s)); } @@ -121,20 +123,25 @@ public AtomicLong getNumberOfRequestsFailed() { return numberOfRequestsFailed; } + public AtomicLong getNumberOfNetworkFailedRequests() { + return numberOfNetworkFailedRequests; + } + /* - Acronyms :- - 1.RCTSI :- Request count that succeeded in x retries - 2.MMA :- Min Max Average (This refers to the backoff or sleep time between 2 requests) - 3.s :- seconds - 4.BWT :- Number of Bandwidth throttled requests - 5.IT :- Number of IOPS throttled requests - 6.OT :- Number of Other throttled requests - 7.%RT :- Percentage of requests that are throttled - 8.TRNR :- Total number of requests which succeeded without retrying - 9.TRF :- Total number of requests which failed - 10.TR :- Total number of requests which were made - 11.MRC :- Max retry count across all requests - */ + Acronyms :- + 1.RCTSI :- Request count that succeeded in x retries + 2.MMA :- Min Max Average (This refers to the backoff or sleep time between 2 requests) + 3.s :- seconds + 4.BWT :- Number of Bandwidth throttled requests + 5.IT :- Number of IOPS throttled requests + 6.OT :- Number of Other throttled requests + 7.NFR :- Number of requests which failed due to network errors + 7.%RT :- Percentage of requests that are throttled + 8.TRNR :- Total number of requests which succeeded without retrying + 9.TRF :- Total number of requests which failed + 10.TR :- Total number of requests which were made + 11.MRC :- Max retry count across all requests + */ @Override public String toString() { StringBuilder metricString = new StringBuilder(); @@ -161,12 +168,13 @@ public String toString() { } metricString.append("#BWT=").append(numberOfBandwidthThrottledRequests) .append(" #IT=").append(numberOfIOPSThrottledRequests) - .append(" #OT=").append(numberOfOtherThrottledRequests); - metricString.append(" #%RT=").append(String.format("%.3f",percentageOfRequestsThrottled)); - metricString.append(" #TRNR=").append(numberOfRequestsSucceededWithoutRetrying); - metricString.append(" #TRF=").append(numberOfRequestsFailed); - metricString.append(" #TR=").append(totalNumberOfRequests); - metricString.append(" #MRC=").append(maxRetryCount); + .append(" #OT=").append(numberOfOtherThrottledRequests) + .append(" #%RT=").append(String.format("%.3f",percentageOfRequestsThrottled)) + .append( "#NFR").append(numberOfNetworkFailedRequests) + .append(" #TRNR=").append(numberOfRequestsSucceededWithoutRetrying) + .append(" #TRF=").append(numberOfRequestsFailed) + .append(" #TR=").append(totalNumberOfRequests) + .append(" #MRC=").append(maxRetryCount); return metricString + " "; } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java index 9b3e458656796..52ca49213addb 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java @@ -118,11 +118,9 @@ import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_BLOCK_UPLOAD_ACTIVE_BLOCKS; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_BLOCK_UPLOAD_BUFFER_DIR; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_METRIC_ACCOUNT_KEY; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_METRIC_ACCOUNT_NAME; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_METRIC_URI; import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.BLOCK_UPLOAD_ACTIVE_BLOCKS_DEFAULT; import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DATA_BLOCKS_BUFFER_DEFAULT; -import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import static org.apache.hadoop.fs.impl.PathCapabilitiesSupport.validatePathCapabilityArgs; import static org.apache.hadoop.fs.statistics.IOStatisticsLogging.logIOStatisticsAtLevel; import static org.apache.hadoop.util.functional.RemoteIterators.filteringRemoteIterator; diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java index 3b351a53205c5..c2e698cb63790 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java @@ -1220,7 +1220,7 @@ public SASTokenProvider getSasTokenProvider() { * Getter for abfsCounters from AbfsClient. * @return AbfsCounters instance. */ - protected AbfsCounters getAbfsCounters() { + public AbfsCounters getAbfsCounters() { return abfsCounters; } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java index d8ca9db178edb..580b09e0ceb15 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java @@ -256,7 +256,8 @@ private void completeExecute(TracingContext tracingContext) } private synchronized void updateDriverMetrics(int retryCount, int statusCode){ - if (statusCode == HttpURLConnection.HTTP_UNAVAILABLE) { + if(statusCode < HttpURLConnection.HTTP_OK + || statusCode >= HttpURLConnection.HTTP_INTERNAL_ERROR) { if (retryCount >= maxIoRetries) { abfsDriverMetrics.getNumberOfRequestsFailed().getAndIncrement(); } @@ -325,7 +326,7 @@ private boolean executeHttpOperation(final int retryCount, } httpOperation.processResponse(buffer, bufferOffset, bufferLength); - if(!isThrottledRequest && httpOperation.getStatusCode() == HttpURLConnection.HTTP_UNAVAILABLE){ + if(!isThrottledRequest && httpOperation.getStatusCode() >= HttpURLConnection.HTTP_INTERNAL_ERROR){ isThrottledRequest = true; AzureServiceErrorCode serviceErrorCode = AzureServiceErrorCode.getAzureServiceCode(httpOperation.getStatusCode(), httpOperation.getStorageErrorCode(), httpOperation.getStorageErrorMessage()); @@ -354,6 +355,7 @@ private boolean executeHttpOperation(final int retryCount, hostname = httpOperation.getHost(); LOG.warn("Unknown host name: {}. Retrying to resolve the host name...", hostname); + abfsDriverMetrics.getNumberOfRequestsFailed().getAndIncrement(); if (!client.getRetryPolicy().shouldRetry(retryCount, -1)) { updateDriverMetrics(retryCount, httpOperation.getStatusCode()); throw new InvalidAbfsRestOperationException(ex); @@ -363,7 +365,7 @@ private boolean executeHttpOperation(final int retryCount, if (LOG.isDebugEnabled()) { LOG.debug("HttpRequestFailure: {}, {}", httpOperation, ex); } - + abfsDriverMetrics.getNumberOfNetworkFailedRequests().getAndIncrement(); if (!client.getRetryPolicy().shouldRetry(retryCount, -1)) { updateDriverMetrics(retryCount, httpOperation.getStatusCode()); throw new InvalidAbfsRestOperationException(ex); @@ -423,7 +425,7 @@ private String getKey(int retryCount) { }else if(retryCount >= 15 && retryCount < 25){ retryCounter = "15_25"; }else{ - retryCounter = "25andabove"; + retryCounter = "25AndAbove"; } return retryCounter; } diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelete.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelete.java index db181fb5dd660..fc4059d3d7908 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelete.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelete.java @@ -45,10 +45,12 @@ import org.apache.hadoop.fs.Path; import static java.net.HttpURLConnection.HTTP_BAD_REQUEST; +import static java.net.HttpURLConnection.HTTP_INTERNAL_ERROR; import static java.net.HttpURLConnection.HTTP_NOT_FOUND; import static java.net.HttpURLConnection.HTTP_OK; - +import org.mockito.Mockito; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.doCallRealMethod; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; @@ -61,7 +63,6 @@ import static org.apache.hadoop.fs.contract.ContractTestUtils.assertPathDoesNotExist; import static org.apache.hadoop.test.LambdaTestUtils.intercept; - /** * Test delete operation. */ @@ -219,6 +220,36 @@ public void testDeleteIdempotency() throws Exception { } + @Test + public void testTryStatus() throws java.io.IOException { + AbfsConfiguration abfsConfig + = TestAbfsConfigurationFieldsValidation.updateRetryConfigs( + getConfiguration(), + 5, 5000); + + final AzureBlobFileSystem fs = this.getFileSystem(); + AbfsClient abfsClient = fs.getAbfsStore().getClient(); + AbfsClient testClient = TestAbfsClient.createTestClientFromCurrentContext( + abfsClient, + abfsConfig); + + final AbfsRestOperation mockRestOperation = mock(AbfsRestOperation.class); + when(mockRestOperation.isARetriedRequest()).thenReturn(true); + + // Case 1: Mock instance of Http Operation response. This will return + // HTTP:Not Found + AbfsHttpOperation httpOperation = mock(AbfsHttpOperation.class); + when(httpOperation.getStatusCode()).thenReturn(HTTP_INTERNAL_ERROR); + + // Mock delete response to 404 + when(mockRestOperation.getResult()).thenReturn(httpOperation); + when(mockRestOperation.hasResult()).thenReturn(true); + TracingContext tracingContext = new TracingContext("abcd", + fs.getFileSystemId(), FSOperationType.TEST_OP, + org.apache.hadoop.fs.azurebfs.utils.TracingHeaderFormat.ALL_ID_FORMAT, null); + fs.close(); + } + @Test public void testDeleteIdempotencyTriggerHttp404() throws Exception { diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestAbfsRestOperation.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestAbfsRestOperation.java new file mode 100644 index 0000000000000..39017b5770500 --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestAbfsRestOperation.java @@ -0,0 +1,41 @@ +package org.apache.hadoop.fs.azurebfs; + +import org.junit.Test; +import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.HTTP_METHOD_DELETE; +import static org.apache.hadoop.fs.azurebfs.services.AbfsRestOperationType.DeletePath; +import org.apache.hadoop.fs.azurebfs.services.AbfsClient; +import org.apache.hadoop.fs.azurebfs.services.TestAbfsClient; +import java.lang.reflect.Method; +import org.apache.hadoop.fs.azurebfs.services.AbfsRestOperation; +import java.util.ArrayList; +import java.util.Arrays; +import org.junit.Assert; +import java.net.HttpURLConnection; + +public class TestAbfsRestOperation extends + AbstractAbfsIntegrationTest { + + public TestAbfsRestOperation() throws Exception { + } + + @Test + public void testDriverRetryMetrics() throws Exception { + final AzureBlobFileSystem fs = getFileSystem(); + AbfsClient client = fs.getAbfsClient(); + + // Mock instance of AbfsRestOperation + AbfsRestOperation op = TestAbfsClient.getRestOp( + DeletePath, client, HTTP_METHOD_DELETE, + TestAbfsClient.getTestUrl(client, "/NonExistingPath"), TestAbfsClient.getTestRequestHeaders(client)); + + ArrayList retryCounts = new ArrayList<>(Arrays.asList(35, 28, 31, 45, 10, 2, 9)); + int statusCode = HttpURLConnection.HTTP_UNAVAILABLE; + Method getMetrics = AbfsRestOperation.class.getDeclaredMethod("updateDriverMetrics", int.class, int.class); + getMetrics.setAccessible(true); + for(int retryCount: retryCounts) { + getMetrics.invoke(op, retryCount, statusCode); + } + Assert.assertEquals(client.getAbfsCounters().getAbfsDriverMetrics().getNumberOfRequestsFailed().toString(), "3"); + fs.close(); + } +} From e1c4c618bde21627e31f29d0bdd2e652a2c9459d Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Thu, 14 Jul 2022 09:31:04 +0530 Subject: [PATCH 20/33] Refactored the test class --- .../hadoop/fs/azurebfs/services/AbfsClient.java | 2 +- .../{ => services}/TestAbfsRestOperation.java | 15 +++++++-------- 2 files changed, 8 insertions(+), 9 deletions(-) rename hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/{ => services}/TestAbfsRestOperation.java (67%) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java index c2e698cb63790..3b351a53205c5 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java @@ -1220,7 +1220,7 @@ public SASTokenProvider getSasTokenProvider() { * Getter for abfsCounters from AbfsClient. * @return AbfsCounters instance. */ - public AbfsCounters getAbfsCounters() { + protected AbfsCounters getAbfsCounters() { return abfsCounters; } diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestAbfsRestOperation.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsRestOperation.java similarity index 67% rename from hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestAbfsRestOperation.java rename to hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsRestOperation.java index 39017b5770500..dea6c3f4edd92 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestAbfsRestOperation.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsRestOperation.java @@ -1,12 +1,11 @@ -package org.apache.hadoop.fs.azurebfs; +package org.apache.hadoop.fs.azurebfs.services; import org.junit.Test; import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.HTTP_METHOD_DELETE; import static org.apache.hadoop.fs.azurebfs.services.AbfsRestOperationType.DeletePath; -import org.apache.hadoop.fs.azurebfs.services.AbfsClient; -import org.apache.hadoop.fs.azurebfs.services.TestAbfsClient; import java.lang.reflect.Method; -import org.apache.hadoop.fs.azurebfs.services.AbfsRestOperation; +import org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem; +import org.apache.hadoop.fs.azurebfs.AbstractAbfsIntegrationTest; import java.util.ArrayList; import java.util.Arrays; import org.junit.Assert; @@ -21,12 +20,12 @@ public TestAbfsRestOperation() throws Exception { @Test public void testDriverRetryMetrics() throws Exception { final AzureBlobFileSystem fs = getFileSystem(); - AbfsClient client = fs.getAbfsClient(); + AbfsClient testClient = super.getAbfsClient(super.getAbfsStore(getFileSystem())); // Mock instance of AbfsRestOperation AbfsRestOperation op = TestAbfsClient.getRestOp( - DeletePath, client, HTTP_METHOD_DELETE, - TestAbfsClient.getTestUrl(client, "/NonExistingPath"), TestAbfsClient.getTestRequestHeaders(client)); + DeletePath, testClient, HTTP_METHOD_DELETE, + TestAbfsClient.getTestUrl(testClient, "/NonExistingPath"), TestAbfsClient.getTestRequestHeaders(testClient)); ArrayList retryCounts = new ArrayList<>(Arrays.asList(35, 28, 31, 45, 10, 2, 9)); int statusCode = HttpURLConnection.HTTP_UNAVAILABLE; @@ -35,7 +34,7 @@ public void testDriverRetryMetrics() throws Exception { for(int retryCount: retryCounts) { getMetrics.invoke(op, retryCount, statusCode); } - Assert.assertEquals(client.getAbfsCounters().getAbfsDriverMetrics().getNumberOfRequestsFailed().toString(), "3"); + Assert.assertEquals(testClient.getAbfsCounters().getAbfsDriverMetrics().getNumberOfRequestsFailed().toString(), "3"); fs.close(); } } From 7686ccee13024ba47638a26847d9e8caa2777ff8 Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Thu, 14 Jul 2022 11:42:23 +0530 Subject: [PATCH 21/33] Added comments --- .../hadoop/fs/azurebfs/services/TestAbfsRestOperation.java | 1 + 1 file changed, 1 insertion(+) diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsRestOperation.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsRestOperation.java index dea6c3f4edd92..54c72f966e29c 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsRestOperation.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsRestOperation.java @@ -34,6 +34,7 @@ public void testDriverRetryMetrics() throws Exception { for(int retryCount: retryCounts) { getMetrics.invoke(op, retryCount, statusCode); } + //For retry count greater than max configured value, the request should fail Assert.assertEquals(testClient.getAbfsCounters().getAbfsDriverMetrics().getNumberOfRequestsFailed().toString(), "3"); fs.close(); } From f66bfe396ccd10ade811d44db15919899a132f1b Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Thu, 14 Jul 2022 12:16:24 +0530 Subject: [PATCH 22/33] Back to previous configs --- .../hadoop/fs/azurebfs/constants/FileSystemConfigurations.java | 2 +- .../apache/hadoop/fs/azurebfs/services/AbfsHttpOperation.java | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java index 3d7263790a0fe..227e1d6810b8b 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java @@ -38,7 +38,7 @@ public final class FileSystemConfigurations { private static final int SIXTY_SECONDS = 60 * 1000; // Retry parameter defaults. - public static final int DEFAULT_MIN_BACKOFF_INTERVAL = 25 * 10; // 3s + public static final int DEFAULT_MIN_BACKOFF_INTERVAL = 3 * 1000; // 3s public static final int DEFAULT_MAX_BACKOFF_INTERVAL = 30 * 1000; // 30s public static final int DEFAULT_BACKOFF_INTERVAL = 3 * 1000; // 3s public static final int DEFAULT_MAX_RETRY_ATTEMPTS = 30; diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsHttpOperation.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsHttpOperation.java index 0cfbf03106227..6d9110dca03aa 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsHttpOperation.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsHttpOperation.java @@ -345,7 +345,6 @@ public void processResponse(final byte[] buffer, final int offset, final int len } this.statusCode = this.connection.getResponseCode(); - //this.statusCode = 503; if (this.isTraceEnabled) { this.recvResponseTimeMs = elapsedTimeMs(startTime); } From d0c0f68007d851b345af61864ebf3482e7197b69 Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Thu, 14 Jul 2022 12:22:51 +0530 Subject: [PATCH 23/33] remove test case --- .../ITestAzureBlobFileSystemDelete.java | 30 ------------------- 1 file changed, 30 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelete.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelete.java index fc4059d3d7908..5562f08a7bcd0 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelete.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelete.java @@ -220,36 +220,6 @@ public void testDeleteIdempotency() throws Exception { } - @Test - public void testTryStatus() throws java.io.IOException { - AbfsConfiguration abfsConfig - = TestAbfsConfigurationFieldsValidation.updateRetryConfigs( - getConfiguration(), - 5, 5000); - - final AzureBlobFileSystem fs = this.getFileSystem(); - AbfsClient abfsClient = fs.getAbfsStore().getClient(); - AbfsClient testClient = TestAbfsClient.createTestClientFromCurrentContext( - abfsClient, - abfsConfig); - - final AbfsRestOperation mockRestOperation = mock(AbfsRestOperation.class); - when(mockRestOperation.isARetriedRequest()).thenReturn(true); - - // Case 1: Mock instance of Http Operation response. This will return - // HTTP:Not Found - AbfsHttpOperation httpOperation = mock(AbfsHttpOperation.class); - when(httpOperation.getStatusCode()).thenReturn(HTTP_INTERNAL_ERROR); - - // Mock delete response to 404 - when(mockRestOperation.getResult()).thenReturn(httpOperation); - when(mockRestOperation.hasResult()).thenReturn(true); - TracingContext tracingContext = new TracingContext("abcd", - fs.getFileSystemId(), FSOperationType.TEST_OP, - org.apache.hadoop.fs.azurebfs.utils.TracingHeaderFormat.ALL_ID_FORMAT, null); - fs.close(); - } - @Test public void testDeleteIdempotencyTriggerHttp404() throws Exception { From 637060f986966bd0e2f41280cd99311d27294e7b Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Thu, 14 Jul 2022 12:24:11 +0530 Subject: [PATCH 24/33] remove test case --- .../hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelete.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelete.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelete.java index 5562f08a7bcd0..e3adf1f98ef77 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelete.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelete.java @@ -45,12 +45,9 @@ import org.apache.hadoop.fs.Path; import static java.net.HttpURLConnection.HTTP_BAD_REQUEST; -import static java.net.HttpURLConnection.HTTP_INTERNAL_ERROR; import static java.net.HttpURLConnection.HTTP_NOT_FOUND; import static java.net.HttpURLConnection.HTTP_OK; -import org.mockito.Mockito; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.doCallRealMethod; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; From a4774007b88ed879a257ed95e048398ec17883f9 Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Thu, 14 Jul 2022 12:26:10 +0530 Subject: [PATCH 25/33] remove test case --- .../hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelete.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelete.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelete.java index e3adf1f98ef77..db181fb5dd660 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelete.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelete.java @@ -47,6 +47,7 @@ import static java.net.HttpURLConnection.HTTP_BAD_REQUEST; import static java.net.HttpURLConnection.HTTP_NOT_FOUND; import static java.net.HttpURLConnection.HTTP_OK; + import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doCallRealMethod; import static org.mockito.Mockito.doReturn; @@ -60,6 +61,7 @@ import static org.apache.hadoop.fs.contract.ContractTestUtils.assertPathDoesNotExist; import static org.apache.hadoop.test.LambdaTestUtils.intercept; + /** * Test delete operation. */ From 0f44a9bf4c38d473accb3a4e188f24063bdea831 Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Fri, 15 Jul 2022 09:57:05 +0530 Subject: [PATCH 26/33] Fix for checkstyle errors --- .../hadoop/fs/azurebfs/AbfsDriverMetrics.java | 71 ++++++++++++++----- .../fs/azurebfs/AzureBlobFileSystem.java | 20 ++++-- .../services/AzureServiceErrorCode.java | 7 +- .../azurebfs/services/AbfsRestOperation.java | 57 +++++++++------ .../hadoop-azure/src/site/markdown/abfs.md | 19 +++-- .../services/TestAbfsRestOperation.java | 20 +++++- 6 files changed, 137 insertions(+), 57 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java index 1826f1bd0f210..19c9f40bdcf33 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java @@ -23,22 +23,39 @@ import java.util.Map; import java.util.ArrayList; import java.util.Arrays; + public class AbfsDriverMetrics { + private AtomicLong numberOfRequestsSucceeded; + private AtomicLong minBackoff; + private AtomicLong maxBackoff; + private AtomicLong totalRequests; + private AtomicLong totalBackoff; + private String retryCount; + private AtomicLong numberOfIOPSThrottledRequests; + private AtomicLong numberOfBandwidthThrottledRequests; + private AtomicLong numberOfOtherThrottledRequests; + private AtomicLong numberOfNetworkFailedRequests; + private AtomicLong maxRetryCount; + private AtomicLong totalNumberOfRequests; + private AtomicLong numberOfRequestsSucceededWithoutRetrying; + private AtomicLong numberOfRequestsFailed; - private final Map metricsMap = new ConcurrentHashMap<>(); + + private final Map metricsMap + = new ConcurrentHashMap<>(); public AbfsDriverMetrics() { initializeMap(); @@ -51,6 +68,7 @@ public AbfsDriverMetrics() { this.numberOfRequestsFailed = new AtomicLong(); this.numberOfNetworkFailedRequests = new AtomicLong(); } + public AbfsDriverMetrics(String retryCount) { this.retryCount = retryCount; this.numberOfRequestsSucceeded = new AtomicLong(); @@ -61,7 +79,8 @@ public AbfsDriverMetrics(String retryCount) { } private void initializeMap() { - ArrayList retryCountList = new ArrayList(Arrays.asList("1","2","3","4","5_15","15_25","25AndAbove")); + ArrayList retryCountList = new ArrayList( + Arrays.asList("1", "2", "3", "4", "5_15", "15_25", "25AndAbove")); for (String s : retryCountList) { metricsMap.put(s, new AbfsDriverMetrics(s)); } @@ -145,36 +164,52 @@ public AtomicLong getNumberOfNetworkFailedRequests() { @Override public String toString() { StringBuilder metricString = new StringBuilder(); - long totalRequestsThrottled = numberOfBandwidthThrottledRequests.get() + numberOfIOPSThrottledRequests.get() + numberOfOtherThrottledRequests.get(); - double percentageOfRequestsThrottled = ((double)totalRequestsThrottled/totalNumberOfRequests.get())*100; + long totalRequestsThrottled = numberOfBandwidthThrottledRequests.get() + + numberOfIOPSThrottledRequests.get() + + numberOfOtherThrottledRequests.get(); + double percentageOfRequestsThrottled = + ((double) totalRequestsThrottled / totalNumberOfRequests.get()) * 100; for (Map.Entry entry : metricsMap.entrySet()) { metricString.append("#RCTSI#_").append(entry.getKey()) .append("R_").append("=") .append(entry.getValue().getNumberOfRequestsSucceeded()).append(" "); long totalRequests = entry.getValue().getTotalRequests().get(); - if(totalRequests > 0) { + if (totalRequests > 0) { metricString.append("#MMA#_").append(entry.getKey()) .append("R_").append("=") - .append(String.format("%.3f", (double) entry.getValue().getMinBackoff().get() / 1000L)) + .append(String.format("%.3f", + (double) entry.getValue().getMinBackoff().get() / 1000L)) .append("s ") - .append(String.format("%.3f", (double) entry.getValue().getMaxBackoff().get() / 1000L)) + .append(String.format("%.3f", + (double) entry.getValue().getMaxBackoff().get() / 1000L)) .append("s ") - .append(String.format("%.3f", (double) ((entry.getValue().getTotalBackoff().get() / totalRequests) / 1000L))) + .append(String.format("%.3f", (double) ( + (entry.getValue().getTotalBackoff().get() / totalRequests) + / 1000L))) .append("s "); - }else { + } else { metricString.append("#MMA#_").append(entry.getKey()) .append("R_").append("=0s "); } } - metricString.append("#BWT=").append(numberOfBandwidthThrottledRequests) - .append(" #IT=").append(numberOfIOPSThrottledRequests) - .append(" #OT=").append(numberOfOtherThrottledRequests) - .append(" #%RT=").append(String.format("%.3f",percentageOfRequestsThrottled)) - .append( "#NFR").append(numberOfNetworkFailedRequests) - .append(" #TRNR=").append(numberOfRequestsSucceededWithoutRetrying) - .append(" #TRF=").append(numberOfRequestsFailed) - .append(" #TR=").append(totalNumberOfRequests) - .append(" #MRC=").append(maxRetryCount); + metricString.append("#BWT=") + .append(numberOfBandwidthThrottledRequests) + .append(" #IT=") + .append(numberOfIOPSThrottledRequests) + .append(" #OT=") + .append(numberOfOtherThrottledRequests) + .append(" #%RT=") + .append(String.format("%.3f", percentageOfRequestsThrottled)) + .append("#NFR") + .append(numberOfNetworkFailedRequests) + .append(" #TRNR=") + .append(numberOfRequestsSucceededWithoutRetrying) + .append(" #TRF=") + .append(numberOfRequestsFailed) + .append(" #TR=") + .append(totalNumberOfRequests) + .append(" #MRC=") + .append(maxRetryCount); return metricString + " "; } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java index 87da314329448..2ff453f8d8780 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java @@ -685,24 +685,30 @@ public synchronized void close() throws IOException { } String metric = abfsCounters.getAbfsDriverMetrics().toString(); LOG.debug("The metrics collected over this instance are " + metric); - if(sendMetricsToStore) { - if(abfsCounters.getAbfsDriverMetrics().getTotalNumberOfRequests().get() > 0) { + if (sendMetricsToStore) { + if (abfsCounters.getAbfsDriverMetrics().getTotalNumberOfRequests().get() + > 0) { try { Configuration metricConfig = getConf(); - String metricAccountKey = metricConfig.get(FS_AZURE_METRIC_ACCOUNT_KEY); + String metricAccountKey = metricConfig.get( + FS_AZURE_METRIC_ACCOUNT_KEY); final String abfsMetricUrl = metricConfig.get(FS_AZURE_METRIC_URI); - if(abfsMetricUrl == null){ + if (abfsMetricUrl == null) { return; } - metricConfig.set(FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME, metricAccountKey); - metricConfig.set(AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION, "false"); + metricConfig.set(FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME, + metricAccountKey); + metricConfig.set(AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION, + "false"); URI metricUri; try { metricUri = new URI(getScheme(), abfsMetricUrl, null, null, null); } catch (URISyntaxException ex) { throw new AssertionError(ex); } - AzureBlobFileSystem metricFs = (AzureBlobFileSystem) FileSystem.newInstance(metricUri, metricConfig); + AzureBlobFileSystem metricFs + = (AzureBlobFileSystem) FileSystem.newInstance(metricUri, + metricConfig); metricFs.sendMetric(metric); } catch (AzureBlobFileSystemException ex) { //do nothing diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/services/AzureServiceErrorCode.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/services/AzureServiceErrorCode.java index ce53b1fd1bfe1..262d37ec63d4e 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/services/AzureServiceErrorCode.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/services/AzureServiceErrorCode.java @@ -113,10 +113,11 @@ public static AzureServiceErrorCode getAzureServiceCode(int httpStatusCode, Stri for (AzureServiceErrorCode azureServiceErrorCode : AzureServiceErrorCode.values()) { if (azureServiceErrorCode.getStatusCode() == httpStatusCode && azureServiceErrorCode.getErrorCode().equalsIgnoreCase(errorCode) && - azureServiceErrorCode.getErrorMessage().equalsIgnoreCase(errorMessages[0])) { - return azureServiceErrorCode; - } + azureServiceErrorCode.getErrorMessage() + .equalsIgnoreCase(errorMessages[0])) { + return azureServiceErrorCode; } + } return UNKNOWN; } } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java index 580b09e0ceb15..0c4eb1f0b19bd 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java @@ -151,9 +151,10 @@ String getSasToken() { || AbfsHttpConstants.HTTP_METHOD_PATCH.equals(method)); this.sasToken = sasToken; this.abfsCounters = client.getAbfsCounters(); - if(abfsCounters != null) { + if (abfsCounters != null) { this.abfsDriverMetrics = abfsCounters.getAbfsDriverMetrics(); - }if(abfsDriverMetrics != null) { + } + if (abfsDriverMetrics != null) { this.metricsMap = abfsDriverMetrics.getMetricsMap(); } this.maxIoRetries = client.getAbfsConfiguration().getMaxIoRetries(); @@ -226,7 +227,7 @@ private void completeExecute(TracingContext tracingContext) retryCount = 0; LOG.debug("First execution of REST operation - {}", operationType); long sleepDuration = 0L; - if(abfsDriverMetrics != null) { + if (abfsDriverMetrics != null) { abfsDriverMetrics.getTotalNumberOfRequests().getAndIncrement(); } while (!executeHttpOperation(retryCount, tracingContext)) { @@ -244,7 +245,7 @@ private void completeExecute(TracingContext tracingContext) Thread.currentThread().interrupt(); } } - if(abfsDriverMetrics != null) { + if (abfsDriverMetrics != null) { updateDriverMetrics(retryCount, result.getStatusCode()); } if (result.getStatusCode() >= HttpURLConnection.HTTP_BAD_REQUEST) { @@ -256,18 +257,20 @@ private void completeExecute(TracingContext tracingContext) } private synchronized void updateDriverMetrics(int retryCount, int statusCode){ - if(statusCode < HttpURLConnection.HTTP_OK + if (statusCode < HttpURLConnection.HTTP_OK || statusCode >= HttpURLConnection.HTTP_INTERNAL_ERROR) { if (retryCount >= maxIoRetries) { abfsDriverMetrics.getNumberOfRequestsFailed().getAndIncrement(); } } else { if (retryCount > 0 && retryCount <= maxIoRetries) { - maxRetryCount = Math.max(abfsDriverMetrics.getMaxRetryCount().get(), retryCount); + maxRetryCount = Math.max(abfsDriverMetrics.getMaxRetryCount().get(), + retryCount); abfsDriverMetrics.getMaxRetryCount().set(maxRetryCount); updateCount(retryCount); } else { - abfsDriverMetrics.getNumberOfRequestsSucceededWithoutRetrying().getAndIncrement(); + abfsDriverMetrics.getNumberOfRequestsSucceededWithoutRetrying() + .getAndIncrement(); } } } @@ -326,19 +329,31 @@ private boolean executeHttpOperation(final int retryCount, } httpOperation.processResponse(buffer, bufferOffset, bufferLength); - if(!isThrottledRequest && httpOperation.getStatusCode() >= HttpURLConnection.HTTP_INTERNAL_ERROR){ + if (!isThrottledRequest && httpOperation.getStatusCode() + >= HttpURLConnection.HTTP_INTERNAL_ERROR) { isThrottledRequest = true; AzureServiceErrorCode serviceErrorCode = - AzureServiceErrorCode.getAzureServiceCode(httpOperation.getStatusCode(), httpOperation.getStorageErrorCode(), httpOperation.getStorageErrorMessage()); - LOG1.trace("Service code is " + serviceErrorCode + " status code is " + httpOperation.getStatusCode() + " error code is " + httpOperation.getStorageErrorCode() + AzureServiceErrorCode.getAzureServiceCode( + httpOperation.getStatusCode(), + httpOperation.getStorageErrorCode(), + httpOperation.getStorageErrorMessage()); + LOG1.trace("Service code is " + serviceErrorCode + " status code is " + + httpOperation.getStatusCode() + " error code is " + + httpOperation.getStorageErrorCode() + " error message is " + httpOperation.getStorageErrorMessage()); - if(serviceErrorCode.equals(AzureServiceErrorCode.INGRESS_OVER_ACCOUNT_LIMIT) || - serviceErrorCode.equals(AzureServiceErrorCode.EGRESS_OVER_ACCOUNT_LIMIT)){ - abfsDriverMetrics.getNumberOfBandwidthThrottledRequests().getAndIncrement(); - }else if(serviceErrorCode.equals(AzureServiceErrorCode.REQUEST_OVER_ACCOUNT_LIMIT)){ - abfsDriverMetrics.getNumberOfIOPSThrottledRequests().getAndIncrement(); - }else{ - abfsDriverMetrics.getNumberOfOtherThrottledRequests().getAndIncrement(); + if (serviceErrorCode.equals( + AzureServiceErrorCode.INGRESS_OVER_ACCOUNT_LIMIT) || + serviceErrorCode.equals( + AzureServiceErrorCode.EGRESS_OVER_ACCOUNT_LIMIT)) { + abfsDriverMetrics.getNumberOfBandwidthThrottledRequests() + .getAndIncrement(); + } else if (serviceErrorCode.equals( + AzureServiceErrorCode.REQUEST_OVER_ACCOUNT_LIMIT)) { + abfsDriverMetrics.getNumberOfIOPSThrottledRequests() + .getAndIncrement(); + } else { + abfsDriverMetrics.getNumberOfOtherThrottledRequests() + .getAndIncrement(); } } incrementCounter(AbfsStatistic.GET_RESPONSES, 1); @@ -418,13 +433,13 @@ private void updateTimeMetrics(int retryCount, long sleepDuration){ private String getKey(int retryCount) { String retryCounter; - if(retryCount >= 1 && retryCount <= 4){ + if (retryCount >= 1 && retryCount <= 4) { retryCounter = Integer.toString(retryCount); - }else if(retryCount >= 5 && retryCount < 15){ + } else if (retryCount >= 5 && retryCount < 15) { retryCounter = "5_15"; - }else if(retryCount >= 15 && retryCount < 25){ + } else if (retryCount >= 15 && retryCount < 25) { retryCounter = "15_25"; - }else{ + } else { retryCounter = "25AndAbove"; } return retryCounter; diff --git a/hadoop-tools/hadoop-azure/src/site/markdown/abfs.md b/hadoop-tools/hadoop-azure/src/site/markdown/abfs.md index 12e2bc6a5d7c6..7250e6893a0ef 100644 --- a/hadoop-tools/hadoop-azure/src/site/markdown/abfs.md +++ b/hadoop-tools/hadoop-azure/src/site/markdown/abfs.md @@ -993,18 +993,23 @@ settings to track their end-to-end latency. ### Driver Metric Options -`fs.azure.metric.account.name`: This configuration parameter is used to specify -the name of the account which will be used to push a failed(404) GetPathStatus request to the -backend. We can configure a separate account to push metrics to the store or use the same for as the existing account on which other requests are made. +`fs.azure.metric.account.name`: This configuration parameter is used to specify +the name of the account which will be used to push a failed(404) GetPathStatus +request to the +backend. We can configure a separate account to push metrics to the store or use +the same for as the existing account on which other requests are made. -`fs.azure.metric.account.key`: This is the access key for the storage account +`fs.azure.metric.account.key`: This is the access key for the storage account used for pushing metrics to the store. -`fs.azure.enable.metric.collection`: This configuration provides an option to specify whether we want to push the metrics or not. +`fs.azure.enable.metric.collection`: This configuration provides an option to +specify whether we want to push the metrics or not. By default, this config will be set to false. -`fs.azure.metric.uri`: This configuration provides the uri in the format of containername@accountname.dfs.core.windows.net. -This should be a part of the config in order to prevent extra calls to create the filesystem. We use an existing filsystem to push the metrics. +`fs.azure.metric.uri`: This configuration provides the uri in the format of +containername@accountname.dfs.core.windows.net. +This should be a part of the config in order to prevent extra calls to create +the filesystem. We use an existing filsystem to push the metrics. ## Troubleshooting diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsRestOperation.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsRestOperation.java index 54c72f966e29c..1de3d97d0aaaf 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsRestOperation.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsRestOperation.java @@ -1,3 +1,21 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.hadoop.fs.azurebfs.services; import org.junit.Test; @@ -31,7 +49,7 @@ public void testDriverRetryMetrics() throws Exception { int statusCode = HttpURLConnection.HTTP_UNAVAILABLE; Method getMetrics = AbfsRestOperation.class.getDeclaredMethod("updateDriverMetrics", int.class, int.class); getMetrics.setAccessible(true); - for(int retryCount: retryCounts) { + for (int retryCount : retryCounts) { getMetrics.invoke(op, retryCount, statusCode); } //For retry count greater than max configured value, the request should fail From 312853cb4395e27b6112f8002943395e97765dd5 Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Mon, 18 Jul 2022 10:34:26 +0530 Subject: [PATCH 27/33] Checkstyle error fixes --- .../contracts/services/AzureServiceErrorCode.java | 4 ++-- .../fs/azurebfs/services/AbfsRestOperation.java | 11 ++++------- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/services/AzureServiceErrorCode.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/services/AzureServiceErrorCode.java index 262d37ec63d4e..7a4ffaca4ead7 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/services/AzureServiceErrorCode.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/services/AzureServiceErrorCode.java @@ -112,8 +112,8 @@ public static AzureServiceErrorCode getAzureServiceCode(int httpStatusCode, Stri String[] errorMessages = errorMessage.split(System.lineSeparator(), 2); for (AzureServiceErrorCode azureServiceErrorCode : AzureServiceErrorCode.values()) { if (azureServiceErrorCode.getStatusCode() == httpStatusCode - && azureServiceErrorCode.getErrorCode().equalsIgnoreCase(errorCode) && - azureServiceErrorCode.getErrorMessage() + && azureServiceErrorCode.getErrorCode().equalsIgnoreCase(errorCode) + && azureServiceErrorCode.getErrorMessage() .equalsIgnoreCase(errorMessages[0])) { return azureServiceErrorCode; } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java index 0c4eb1f0b19bd..be611e14fc011 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java @@ -237,7 +237,7 @@ private void completeExecute(TracingContext tracingContext) LOG.debug("Retrying REST operation {}. RetryCount = {}", operationType, retryCount); sleepDuration = client.getRetryPolicy().getRetryInterval(retryCount); - if(abfsDriverMetrics != null) { + if (abfsDriverMetrics != null) { updateTimeMetrics(retryCount, sleepDuration); } Thread.sleep(sleepDuration); @@ -341,14 +341,11 @@ private boolean executeHttpOperation(final int retryCount, + httpOperation.getStatusCode() + " error code is " + httpOperation.getStorageErrorCode() + " error message is " + httpOperation.getStorageErrorMessage()); - if (serviceErrorCode.equals( - AzureServiceErrorCode.INGRESS_OVER_ACCOUNT_LIMIT) || - serviceErrorCode.equals( - AzureServiceErrorCode.EGRESS_OVER_ACCOUNT_LIMIT)) { + if (serviceErrorCode.equals(AzureServiceErrorCode.INGRESS_OVER_ACCOUNT_LIMIT) + || serviceErrorCode.equals(AzureServiceErrorCode.EGRESS_OVER_ACCOUNT_LIMIT)) { abfsDriverMetrics.getNumberOfBandwidthThrottledRequests() .getAndIncrement(); - } else if (serviceErrorCode.equals( - AzureServiceErrorCode.REQUEST_OVER_ACCOUNT_LIMIT)) { + } else if (serviceErrorCode.equals(AzureServiceErrorCode.REQUEST_OVER_ACCOUNT_LIMIT)) { abfsDriverMetrics.getNumberOfIOPSThrottledRequests() .getAndIncrement(); } else { From bca9c4e900d70d446dbacb932ea9f713ac3eaf53 Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Mon, 18 Jul 2022 12:34:21 +0530 Subject: [PATCH 28/33] Spotbugs fixes --- .../org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java index 19c9f40bdcf33..3caecb5295f88 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java @@ -183,9 +183,9 @@ public String toString() { .append(String.format("%.3f", (double) entry.getValue().getMaxBackoff().get() / 1000L)) .append("s ") - .append(String.format("%.3f", (double) ( - (entry.getValue().getTotalBackoff().get() / totalRequests) - / 1000L))) + .append(String.format("%.3f", + ((double)entry.getValue().getTotalBackoff().get() / totalRequests) + / 1000L)) .append("s "); } else { metricString.append("#MMA#_").append(entry.getKey()) From a5ca55f594e1054d356d332185f172a01d981fe6 Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Tue, 26 Jul 2022 18:49:15 +0530 Subject: [PATCH 29/33] Added metrics for input stream --- .../hadoop/fs/azurebfs/AbfsCountersImpl.java | 12 +++- .../fs/azurebfs/AzureBlobFileSystem.java | 57 ++++++++++++++----- .../fs/azurebfs/AzureBlobFileSystemStore.java | 3 +- .../fs/azurebfs/services/AbfsCounters.java | 4 +- .../fs/azurebfs/services/AbfsInputStream.java | 28 ++++++++- .../services/AbfsInputStreamContext.java | 11 ++++ .../services/AbfsInputStreamMetrics.java | 43 ++++++++++++++ .../fs/azurebfs/utils/TracingContext.java | 6 +- 8 files changed, 139 insertions(+), 25 deletions(-) create mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStreamMetrics.java diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsCountersImpl.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsCountersImpl.java index 1367855cb9d97..f7b25ac4ac76a 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsCountersImpl.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsCountersImpl.java @@ -21,7 +21,8 @@ import java.net.URI; import java.util.Map; import java.util.UUID; - +import java.util.List; +import java.util.ArrayList; import org.apache.hadoop.classification.VisibleForTesting; import org.apache.hadoop.fs.azurebfs.services.AbfsCounters; @@ -33,7 +34,7 @@ import org.apache.hadoop.metrics2.lib.MetricsRegistry; import org.apache.hadoop.metrics2.lib.MutableCounterLong; import org.apache.hadoop.metrics2.lib.MutableMetric; - +import org.apache.hadoop.fs.azurebfs.services.AbfsInputStreamMetrics; import static org.apache.hadoop.fs.azurebfs.AbfsStatistic.*; import static org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.iostatisticsStore; import java.util.concurrent.atomic.AtomicReference; @@ -66,6 +67,8 @@ public class AbfsCountersImpl implements AbfsCounters { private AtomicReference abfsDriverMetrics = null; + private List inputStreamMetricsList; + private static final AbfsStatistic[] STATISTIC_LIST = { CALL_CREATE, CALL_OPEN, @@ -125,6 +128,7 @@ public AbfsCountersImpl(URI uri) { } ioStatisticsStore = ioStatisticsStoreBuilder.build(); abfsDriverMetrics = new AtomicReference<>(new AbfsDriverMetrics()); + inputStreamMetricsList = new ArrayList<>(); } /** @@ -196,6 +200,10 @@ public AbfsDriverMetrics getAbfsDriverMetrics() { return abfsDriverMetrics.get(); } + public List getAbfsInputStreamMetrics() { + return inputStreamMetricsList; + } + /** * {@inheritDoc} * diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java index 2ff453f8d8780..4fa3c5080e3bb 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java @@ -42,7 +42,10 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; - +import java.util.Set; +import java.util.LinkedHashSet; +import java.util.Iterator; +import java.util.concurrent.atomic.AtomicLong; import javax.annotation.Nullable; import org.apache.hadoop.classification.VisibleForTesting; @@ -108,7 +111,7 @@ import org.apache.hadoop.util.DurationInfo; import org.apache.hadoop.util.LambdaUtils; import org.apache.hadoop.util.Progressable; - +import org.apache.hadoop.fs.azurebfs.services.AbfsInputStreamMetrics; import static org.apache.hadoop.fs.CommonConfigurationKeys.IOSTATISTICS_LOGGING_LEVEL; import static org.apache.hadoop.fs.CommonConfigurationKeys.IOSTATISTICS_LOGGING_LEVEL_DEFAULT; import static org.apache.hadoop.fs.azurebfs.AbfsStatistic.*; @@ -157,7 +160,6 @@ public class AzureBlobFileSystem extends FileSystem /** Rate limiting for operations which use it to throttle their IO. */ private RateLimiting rateLimiting; - @Override public void initialize(URI uri, Configuration configuration) throws IOException { @@ -683,32 +685,35 @@ public synchronized void close() throws IOException { if (isClosed) { return; } - String metric = abfsCounters.getAbfsDriverMetrics().toString(); + List inputStreamMetricsList = abfsCounters.getAbfsInputStreamMetrics(); + ArrayList avgStreamMetrics = new ArrayList<>(); + String inputStreamMetric = ""; + if (!inputStreamMetricsList.isEmpty()){ + avgStreamMetrics = getStreamMetricsAverage(inputStreamMetricsList); + } + if(avgStreamMetrics.size() > 0) { + inputStreamMetric = getStreamMetrics(avgStreamMetrics); + } + String metric = abfsCounters.getAbfsDriverMetrics().toString() + inputStreamMetric; LOG.debug("The metrics collected over this instance are " + metric); if (sendMetricsToStore) { - if (abfsCounters.getAbfsDriverMetrics().getTotalNumberOfRequests().get() - > 0) { + if (abfsCounters.getAbfsDriverMetrics().getTotalNumberOfRequests().get() > 0) { try { Configuration metricConfig = getConf(); - String metricAccountKey = metricConfig.get( - FS_AZURE_METRIC_ACCOUNT_KEY); + String metricAccountKey = metricConfig.get(FS_AZURE_METRIC_ACCOUNT_KEY); final String abfsMetricUrl = metricConfig.get(FS_AZURE_METRIC_URI); if (abfsMetricUrl == null) { return; } - metricConfig.set(FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME, - metricAccountKey); - metricConfig.set(AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION, - "false"); + metricConfig.set(FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME, metricAccountKey); + metricConfig.set(AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION, "false"); URI metricUri; try { metricUri = new URI(getScheme(), abfsMetricUrl, null, null, null); } catch (URISyntaxException ex) { throw new AssertionError(ex); } - AzureBlobFileSystem metricFs - = (AzureBlobFileSystem) FileSystem.newInstance(metricUri, - metricConfig); + AzureBlobFileSystem metricFs = (AzureBlobFileSystem) FileSystem.newInstance(metricUri, metricConfig); metricFs.sendMetric(metric); } catch (AzureBlobFileSystemException ex) { //do nothing @@ -731,6 +736,28 @@ public synchronized void close() throws IOException { } } + private ArrayList getStreamMetricsAverage(List inputStreamMetricsList){ + ArrayList avgInputStreamMetrics = new ArrayList<>(); + avgInputStreamMetrics.add(inputStreamMetricsList.stream() + .map(AbfsInputStreamMetrics::getSizeReadByFirstRead).mapToDouble(AtomicLong::get) + .average().orElse(0.0)); + avgInputStreamMetrics.add(inputStreamMetricsList.stream() + .map(AbfsInputStreamMetrics::getOffsetDiffBetweenFirstAndSecondRead) + .mapToDouble(AtomicLong::get).average().orElse(0.0)); + avgInputStreamMetrics.add(inputStreamMetricsList.stream() + .map(AbfsInputStreamMetrics::getFileLength) + .mapToDouble(AtomicLong::get).average().orElse(0.0)); + return avgInputStreamMetrics; + } + + private String getStreamMetrics(ArrayList avgInputStreamMetrics) { + String inputStreamMetric = " #%SRFR=" + String.format("%.3f", + avgInputStreamMetrics.get(0)) + + " #%DOFS=" + String.format("%.3f", avgInputStreamMetrics.get(1)) + + " #%FL=" + String.format("%.3f", avgInputStreamMetrics.get(2)); + return inputStreamMetric + " "; + } + public void sendMetric(String metric) throws AzureBlobFileSystemException { TracingHeaderFormat tracingHeaderFormatMetric = TracingHeaderFormat.INTERNAL_METRIC_FORMAT; TracingContext tracingContextMetric = new TracingContext(clientCorrelationId, diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java index 4a8da9ac4453f..68063818297b2 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java @@ -131,7 +131,7 @@ import org.apache.hadoop.util.SemaphoredDelegatingExecutor; import org.apache.hadoop.util.concurrent.HadoopExecutors; import org.apache.http.client.utils.URIBuilder; - +import org.apache.hadoop.fs.azurebfs.services.AbfsInputStreamMetrics; import static org.apache.hadoop.fs.azurebfs.AbfsStatistic.METADATA_INCOMPLETE_RENAME_FAILURES; import static org.apache.hadoop.fs.azurebfs.AbfsStatistic.RENAME_RECOVERY; import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.CHAR_EQUALS; @@ -812,6 +812,7 @@ private AbfsInputStreamContext populateAbfsInputStreamContext( .withOptimizeFooterRead(abfsConfiguration.optimizeFooterRead()) .withReadAheadRange(abfsConfiguration.getReadAheadRange()) .withStreamStatistics(new AbfsInputStreamStatisticsImpl()) + .withStreamMetrics(new AbfsInputStreamMetrics()) .withShouldReadBufferSizeAlways( abfsConfiguration.shouldReadBufferSizeAlways()) .withReadAheadBlockSize(abfsConfiguration.getReadAheadBlockSize()) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsCounters.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsCounters.java index 91eb03c821566..756e6e173e36b 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsCounters.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsCounters.java @@ -19,7 +19,7 @@ package org.apache.hadoop.fs.azurebfs.services; import java.util.Map; - +import java.util.List; import org.apache.hadoop.classification.VisibleForTesting; import org.apache.hadoop.classification.InterfaceAudience; @@ -76,4 +76,6 @@ String formString(String prefix, String separator, String suffix, DurationTracker trackDuration(String key); AbfsDriverMetrics getAbfsDriverMetrics(); + + List getAbfsInputStreamMetrics(); } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java index 553ccdcbc0a43..01b97b210d7d5 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java @@ -86,6 +86,7 @@ public class AbfsInputStream extends FSInputStream implements CanUnbuffer, private final int readAheadRange; private boolean firstRead = true; + private long offsetOfFirstRead = 0; // SAS tokens can be re-used until they expire private CachedSASToken cachedSasToken; private byte[] buffer = null; // will be initialized on first use @@ -105,13 +106,15 @@ public class AbfsInputStream extends FSInputStream implements CanUnbuffer, private int bCursorBkp; private long fCursorBkp; private long fCursorAfterLastReadBkp; - + private final AbfsInputStreamMetrics inputStreamMetrics; /** Stream statistics. */ private final AbfsInputStreamStatistics streamStatistics; private long bytesFromReadAhead; // bytes read from readAhead; for testing private long bytesFromRemoteRead; // bytes read remotely; for testing private Listener listener; + private boolean collectMetricsForNextRead = false; + private boolean collectStreamMetrics = false; private final AbfsInputStreamContext context; private IOStatistics ioStatistics; /** @@ -145,6 +148,7 @@ public AbfsInputStream( this.cachedSasToken = new CachedSASToken( abfsInputStreamContext.getSasTokenRenewPeriodForStreamsInSeconds()); this.streamStatistics = abfsInputStreamContext.getStreamStatistics(); + this.inputStreamMetrics = abfsInputStreamContext.getAbfsInputStreamMetrics(); this.inputStreamId = createInputStreamId(); this.tracingContext = new TracingContext(tracingContext); this.tracingContext.setOperation(FSOperationType.READ); @@ -239,6 +243,17 @@ public synchronized int read(final byte[] b, final int off, final int len) throw // go back and read from buffer is fCursor - limit. // There maybe case that we read less than requested data. long filePosAtStartOfBuffer = fCursor - limit; + if (firstRead && nextReadPos >= contentLength - 16 * ONE_KB) { + this.collectStreamMetrics = true; + this.collectMetricsForNextRead = true; + this.offsetOfFirstRead = nextReadPos; + this.inputStreamMetrics.getSizeReadByFirstRead().set(len); + this.inputStreamMetrics.getFileLength().set(contentLength); + } + if (!firstRead && collectMetricsForNextRead){ + this.inputStreamMetrics.getOffsetDiffBetweenFirstAndSecondRead().set(off - offsetOfFirstRead); + this.collectMetricsForNextRead = false; + } if (nextReadPos >= filePosAtStartOfBuffer && nextReadPos <= fCursor) { // Determining position in buffer from where data is to be read. bCursor = (int) (nextReadPos - filePosAtStartOfBuffer); @@ -325,7 +340,6 @@ private int readOneBlock(final byte[] b, final int off, final int len) throws IO if (firstRead) { firstRead = false; } - if (bytesRead == -1) { return -1; } @@ -606,6 +620,11 @@ public synchronized void seek(long n) throws IOException { LOG.debug("set nextReadPos to {}", nextReadPos); } + private void updateInputStreamMetrics(long bytesRead) { + this.inputStreamMetrics.getSizeReadByFirstRead().set(bytesRead); + this.inputStreamMetrics.getFileLength().set(contentLength); + } + @Override public synchronized long skip(long n) throws IOException { if (closed) { @@ -696,6 +715,11 @@ public synchronized void close() throws IOException { LOG.debug("Closing {}", this); closed = true; buffer = null; // de-reference the buffer so it can be GC'ed sooner + if(this.collectStreamMetrics) { + this.client.getAbfsCounters() + .getAbfsInputStreamMetrics() + .add(inputStreamMetrics); + } ReadBufferManager.getBufferManager().purgeBuffersForStream(this); } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStreamContext.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStreamContext.java index ae69cde6efac1..d555cd847fb66 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStreamContext.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStreamContext.java @@ -49,6 +49,8 @@ public class AbfsInputStreamContext extends AbfsStreamContext { private boolean bufferedPreadDisabled; + private AbfsInputStreamMetrics abfsInputStreamMetrics; + public AbfsInputStreamContext(final long sasTokenRenewPeriodForStreamsInSeconds) { super(sasTokenRenewPeriodForStreamsInSeconds); } @@ -84,6 +86,12 @@ public AbfsInputStreamContext withStreamStatistics( return this; } + public AbfsInputStreamContext withStreamMetrics( + final AbfsInputStreamMetrics inputStreamMetrics) { + this.abfsInputStreamMetrics = inputStreamMetrics; + return this; + } + public AbfsInputStreamContext withReadSmallFilesCompletely( final boolean readSmallFilesCompletely) { this.readSmallFilesCompletely = readSmallFilesCompletely; @@ -148,6 +156,9 @@ public int getReadAheadRange() { public AbfsInputStreamStatistics getStreamStatistics() { return streamStatistics; } + public AbfsInputStreamMetrics getAbfsInputStreamMetrics() { + return abfsInputStreamMetrics; + } public boolean readSmallFilesCompletely() { return this.readSmallFilesCompletely; diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStreamMetrics.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStreamMetrics.java new file mode 100644 index 0000000000000..596b8602ae7a4 --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStreamMetrics.java @@ -0,0 +1,43 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs.azurebfs.services; + +import java.util.concurrent.atomic.AtomicLong; + +public class AbfsInputStreamMetrics { + private AtomicLong sizeReadByFirstRead; + private AtomicLong offsetDiffBetweenFirstAndSecondRead; + private AtomicLong fileLength; + + public AbfsInputStreamMetrics() { + this.sizeReadByFirstRead = new AtomicLong(); + this.offsetDiffBetweenFirstAndSecondRead = new AtomicLong(); + this.fileLength = new AtomicLong(); + } + public AtomicLong getSizeReadByFirstRead() { + return sizeReadByFirstRead; + } + + public AtomicLong getOffsetDiffBetweenFirstAndSecondRead() { + return offsetDiffBetweenFirstAndSecondRead; + } + + public AtomicLong getFileLength() { + return fileLength; + } +} diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/TracingContext.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/TracingContext.java index 70d4371daefe4..67c9789457707 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/TracingContext.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/TracingContext.java @@ -19,10 +19,10 @@ package org.apache.hadoop.fs.azurebfs.utils; import java.util.UUID; - +import java.util.Set; import org.slf4j.Logger; import org.slf4j.LoggerFactory; - +import java.util.LinkedHashSet; import org.apache.hadoop.fs.azurebfs.constants.FSOperationType; import org.apache.hadoop.fs.azurebfs.constants.HttpHeaderConfigurations; import org.apache.hadoop.fs.azurebfs.services.AbfsClient; @@ -62,9 +62,7 @@ public class TracingContext { private Listener listener = null; // null except when testing //final concatenated ID list set into x-ms-client-request-id header private String header = EMPTY_STRING; - private String metricResults = EMPTY_STRING; - private static final Logger LOG = LoggerFactory.getLogger(AbfsClient.class); public static final int MAX_CLIENT_CORRELATION_ID_LENGTH = 72; public static final String CLIENT_CORRELATION_ID_PATTERN = "[a-zA-Z0-9-]*"; From 1d99d7b97ab994127d35ee410161b0dd6fbf517b Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Tue, 26 Jul 2022 21:35:34 +0530 Subject: [PATCH 30/33] Fix for input stream metrics --- .../org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java | 2 +- .../org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java | 6 +++--- .../apache/hadoop/fs/azurebfs/services/AbfsInputStream.java | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java index 3caecb5295f88..a827b083d8f57 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java @@ -200,7 +200,7 @@ public String toString() { .append(numberOfOtherThrottledRequests) .append(" #%RT=") .append(String.format("%.3f", percentageOfRequestsThrottled)) - .append("#NFR") + .append(" #NFR=") .append(numberOfNetworkFailedRequests) .append(" #TRNR=") .append(numberOfRequestsSucceededWithoutRetrying) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java index 4fa3c5080e3bb..f68cafd4a1525 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java @@ -751,10 +751,10 @@ private ArrayList getStreamMetricsAverage(List i } private String getStreamMetrics(ArrayList avgInputStreamMetrics) { - String inputStreamMetric = " #%SRFR=" + String.format("%.3f", + String inputStreamMetric = " #SRFR=" + String.format("%.3f", avgInputStreamMetrics.get(0)) - + " #%DOFS=" + String.format("%.3f", avgInputStreamMetrics.get(1)) - + " #%FL=" + String.format("%.3f", avgInputStreamMetrics.get(2)); + + " #DOFS=" + String.format("%.3f", avgInputStreamMetrics.get(1)) + + " #FL=" + String.format("%.3f", avgInputStreamMetrics.get(2)); return inputStreamMetric + " "; } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java index 01b97b210d7d5..f0efa29441576 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java @@ -251,7 +251,7 @@ public synchronized int read(final byte[] b, final int off, final int len) throw this.inputStreamMetrics.getFileLength().set(contentLength); } if (!firstRead && collectMetricsForNextRead){ - this.inputStreamMetrics.getOffsetDiffBetweenFirstAndSecondRead().set(off - offsetOfFirstRead); + this.inputStreamMetrics.getOffsetDiffBetweenFirstAndSecondRead().set(Math.abs(nextReadPos - offsetOfFirstRead)); this.collectMetricsForNextRead = false; } if (nextReadPos >= filePosAtStartOfBuffer && nextReadPos <= fCursor) { From 3b5ba520114d6dc9e596ad42fd88e09849d2e3bf Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Tue, 26 Jul 2022 21:41:47 +0530 Subject: [PATCH 31/33] Fix for input stream metrics --- .../apache/hadoop/fs/azurebfs/services/AbfsInputStream.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java index f0efa29441576..ef66fac1390a1 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java @@ -620,11 +620,6 @@ public synchronized void seek(long n) throws IOException { LOG.debug("set nextReadPos to {}", nextReadPos); } - private void updateInputStreamMetrics(long bytesRead) { - this.inputStreamMetrics.getSizeReadByFirstRead().set(bytesRead); - this.inputStreamMetrics.getFileLength().set(contentLength); - } - @Override public synchronized long skip(long n) throws IOException { if (closed) { From 139b52839a7afcbcd9abdedd34b7ba8d6e8f9fba Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Tue, 26 Jul 2022 21:51:25 +0530 Subject: [PATCH 32/33] variable name fix --- .../java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java index f68cafd4a1525..8c9deb84a59b7 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java @@ -753,7 +753,7 @@ private ArrayList getStreamMetricsAverage(List i private String getStreamMetrics(ArrayList avgInputStreamMetrics) { String inputStreamMetric = " #SRFR=" + String.format("%.3f", avgInputStreamMetrics.get(0)) - + " #DOFS=" + String.format("%.3f", avgInputStreamMetrics.get(1)) + + " #DOFSR=" + String.format("%.3f", avgInputStreamMetrics.get(1)) + " #FL=" + String.format("%.3f", avgInputStreamMetrics.get(2)); return inputStreamMetric + " "; } From 4e9cd936bf21b9f0e9f36c91db997972425c1e36 Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Thu, 28 Jul 2022 12:32:59 +0530 Subject: [PATCH 33/33] Added changes for first read and second read --- .../hadoop/fs/azurebfs/AbfsCountersImpl.java | 10 +- .../fs/azurebfs/AzureBlobFileSystem.java | 103 +++++++++++++----- .../fs/azurebfs/AzureBlobFileSystemStore.java | 5 +- .../fs/azurebfs/services/AbfsCounters.java | 2 +- .../fs/azurebfs/services/AbfsInputStream.java | 25 +++-- .../services/AbfsInputStreamContext.java | 12 +- ...etrics.java => AbfsReadFooterMetrics.java} | 37 +++++-- 7 files changed, 136 insertions(+), 58 deletions(-) rename hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/{AbfsInputStreamMetrics.java => AbfsReadFooterMetrics.java} (56%) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsCountersImpl.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsCountersImpl.java index f7b25ac4ac76a..bad4905a1de99 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsCountersImpl.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsCountersImpl.java @@ -34,7 +34,7 @@ import org.apache.hadoop.metrics2.lib.MetricsRegistry; import org.apache.hadoop.metrics2.lib.MutableCounterLong; import org.apache.hadoop.metrics2.lib.MutableMetric; -import org.apache.hadoop.fs.azurebfs.services.AbfsInputStreamMetrics; +import org.apache.hadoop.fs.azurebfs.services.AbfsReadFooterMetrics; import static org.apache.hadoop.fs.azurebfs.AbfsStatistic.*; import static org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.iostatisticsStore; import java.util.concurrent.atomic.AtomicReference; @@ -67,7 +67,7 @@ public class AbfsCountersImpl implements AbfsCounters { private AtomicReference abfsDriverMetrics = null; - private List inputStreamMetricsList; + private List readFooterMetricsList; private static final AbfsStatistic[] STATISTIC_LIST = { CALL_CREATE, @@ -128,7 +128,7 @@ public AbfsCountersImpl(URI uri) { } ioStatisticsStore = ioStatisticsStoreBuilder.build(); abfsDriverMetrics = new AtomicReference<>(new AbfsDriverMetrics()); - inputStreamMetricsList = new ArrayList<>(); + readFooterMetricsList = new ArrayList<>(); } /** @@ -200,8 +200,8 @@ public AbfsDriverMetrics getAbfsDriverMetrics() { return abfsDriverMetrics.get(); } - public List getAbfsInputStreamMetrics() { - return inputStreamMetricsList; + public List getAbfsReadFooterMetrics() { + return readFooterMetricsList; } /** diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java index 8c9deb84a59b7..f5579ea6195d4 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java @@ -34,6 +34,7 @@ import java.util.Collections; import java.util.EnumSet; import java.util.Map; +import java.util.HashMap; import java.util.Optional; import java.util.UUID; import java.util.concurrent.Callable; @@ -42,9 +43,6 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; -import java.util.Set; -import java.util.LinkedHashSet; -import java.util.Iterator; import java.util.concurrent.atomic.AtomicLong; import javax.annotation.Nullable; @@ -111,7 +109,7 @@ import org.apache.hadoop.util.DurationInfo; import org.apache.hadoop.util.LambdaUtils; import org.apache.hadoop.util.Progressable; -import org.apache.hadoop.fs.azurebfs.services.AbfsInputStreamMetrics; +import org.apache.hadoop.fs.azurebfs.services.AbfsReadFooterMetrics; import static org.apache.hadoop.fs.CommonConfigurationKeys.IOSTATISTICS_LOGGING_LEVEL; import static org.apache.hadoop.fs.CommonConfigurationKeys.IOSTATISTICS_LOGGING_LEVEL_DEFAULT; import static org.apache.hadoop.fs.azurebfs.AbfsStatistic.*; @@ -685,16 +683,28 @@ public synchronized void close() throws IOException { if (isClosed) { return; } - List inputStreamMetricsList = abfsCounters.getAbfsInputStreamMetrics(); - ArrayList avgStreamMetrics = new ArrayList<>(); - String inputStreamMetric = ""; - if (!inputStreamMetricsList.isEmpty()){ - avgStreamMetrics = getStreamMetricsAverage(inputStreamMetricsList); + List readFooterMetricsList = abfsCounters.getAbfsReadFooterMetrics(); + List isParquetList = new ArrayList<>(); + List isNonParquetList = new ArrayList<>(); + for (AbfsReadFooterMetrics abfsReadFooterMetrics : readFooterMetricsList) { + if (abfsReadFooterMetrics.getIsParquetFile()) { + isParquetList.add(abfsReadFooterMetrics); + } else { + isNonParquetList.add(abfsReadFooterMetrics); + } + } + List avgParquetReadFooterMetrics = new ArrayList<>(); + List avgNonparquetReadFooterMetrics = new ArrayList<>(); + String readFooterMetric = ""; + if (!isParquetList.isEmpty()){ + getParquetReadFooterMetricsAverage(isParquetList, avgParquetReadFooterMetrics); + readFooterMetric += getParquetReadFooterMetrics(avgParquetReadFooterMetrics); } - if(avgStreamMetrics.size() > 0) { - inputStreamMetric = getStreamMetrics(avgStreamMetrics); + if(!isNonParquetList.isEmpty()) { + getNonParquetReadFooterMetricsAverage(isNonParquetList, avgNonparquetReadFooterMetrics); + readFooterMetric += getNonParquetReadFooterMetrics(avgNonparquetReadFooterMetrics); } - String metric = abfsCounters.getAbfsDriverMetrics().toString() + inputStreamMetric; + String metric = abfsCounters.getAbfsDriverMetrics().toString() + readFooterMetric; LOG.debug("The metrics collected over this instance are " + metric); if (sendMetricsToStore) { if (abfsCounters.getAbfsDriverMetrics().getTotalNumberOfRequests().get() > 0) { @@ -736,26 +746,61 @@ public synchronized void close() throws IOException { } } - private ArrayList getStreamMetricsAverage(List inputStreamMetricsList){ - ArrayList avgInputStreamMetrics = new ArrayList<>(); - avgInputStreamMetrics.add(inputStreamMetricsList.stream() - .map(AbfsInputStreamMetrics::getSizeReadByFirstRead).mapToDouble(AtomicLong::get) - .average().orElse(0.0)); - avgInputStreamMetrics.add(inputStreamMetricsList.stream() - .map(AbfsInputStreamMetrics::getOffsetDiffBetweenFirstAndSecondRead) + private void getParquetReadFooterMetricsAverage(List isParquetList, + List avgParquetReadFooterMetrics){ + avgParquetReadFooterMetrics.add(isParquetList.stream() + .map(AbfsReadFooterMetrics::getSizeReadByFirstRead).mapToDouble( + Double::parseDouble).average().orElse(0.0)); + avgParquetReadFooterMetrics.add(isParquetList.stream() + .map(AbfsReadFooterMetrics::getOffsetDiffBetweenFirstAndSecondRead) + .mapToDouble(Double::parseDouble).average().orElse(0.0)); + avgParquetReadFooterMetrics.add(isParquetList.stream() + .map(AbfsReadFooterMetrics::getFileLength) .mapToDouble(AtomicLong::get).average().orElse(0.0)); - avgInputStreamMetrics.add(inputStreamMetricsList.stream() - .map(AbfsInputStreamMetrics::getFileLength) - .mapToDouble(AtomicLong::get).average().orElse(0.0)); - return avgInputStreamMetrics; } - private String getStreamMetrics(ArrayList avgInputStreamMetrics) { - String inputStreamMetric = " #SRFR=" + String.format("%.3f", - avgInputStreamMetrics.get(0)) - + " #DOFSR=" + String.format("%.3f", avgInputStreamMetrics.get(1)) - + " #FL=" + String.format("%.3f", avgInputStreamMetrics.get(2)); - return inputStreamMetric + " "; + private void getNonParquetReadFooterMetricsAverage(List isNonParquetList, + List avgNonParquetReadFooterMetrics){ + int size = isNonParquetList.get(0).getSizeReadByFirstRead().split("_").length; + double[] store = new double[2*size]; + for (AbfsReadFooterMetrics abfsReadFooterMetrics : isNonParquetList) { + String[] firstReadSize = abfsReadFooterMetrics.getSizeReadByFirstRead().split("_"); + String[] offDiffFirstSecondRead = abfsReadFooterMetrics.getOffsetDiffBetweenFirstAndSecondRead().split("_"); + for(int i=0; i avgParquetReadFooterMetrics) { + String parquetReadFooterMetric = "Parquet:" + " #FR=" + String.format("%.3f", + avgParquetReadFooterMetrics.get(0)) + + " #SR=" + String.format("%.3f", avgParquetReadFooterMetrics.get(1)) + + " #FL=" + String.format("%.3f", avgParquetReadFooterMetrics.get(2)); + return parquetReadFooterMetric + " "; + } + + private String getNonParquetReadFooterMetrics(List avgNonParquetReadFooterMetrics) { + String nonParquetReadFooterMetric = "NonParquet:" + " #FR=" + + avgNonParquetReadFooterMetrics.get(0) + + " #SR=" + avgNonParquetReadFooterMetrics.get(1) + + " #FL=" + avgNonParquetReadFooterMetrics.get(2); + return nonParquetReadFooterMetric + " "; } public void sendMetric(String metric) throws AzureBlobFileSystemException { diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java index 68063818297b2..a37c65d25f44f 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java @@ -100,6 +100,7 @@ import org.apache.hadoop.fs.azurebfs.services.AbfsCounters; import org.apache.hadoop.fs.azurebfs.services.AbfsHttpOperation; import org.apache.hadoop.fs.azurebfs.services.AbfsInputStream; +import org.apache.hadoop.fs.azurebfs.services.AbfsReadFooterMetrics; import org.apache.hadoop.fs.azurebfs.services.AbfsInputStreamContext; import org.apache.hadoop.fs.azurebfs.services.AbfsInputStreamStatisticsImpl; import org.apache.hadoop.fs.azurebfs.services.AbfsOutputStream; @@ -131,7 +132,7 @@ import org.apache.hadoop.util.SemaphoredDelegatingExecutor; import org.apache.hadoop.util.concurrent.HadoopExecutors; import org.apache.http.client.utils.URIBuilder; -import org.apache.hadoop.fs.azurebfs.services.AbfsInputStreamMetrics; + import static org.apache.hadoop.fs.azurebfs.AbfsStatistic.METADATA_INCOMPLETE_RENAME_FAILURES; import static org.apache.hadoop.fs.azurebfs.AbfsStatistic.RENAME_RECOVERY; import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.CHAR_EQUALS; @@ -812,7 +813,7 @@ private AbfsInputStreamContext populateAbfsInputStreamContext( .withOptimizeFooterRead(abfsConfiguration.optimizeFooterRead()) .withReadAheadRange(abfsConfiguration.getReadAheadRange()) .withStreamStatistics(new AbfsInputStreamStatisticsImpl()) - .withStreamMetrics(new AbfsInputStreamMetrics()) + .withReadFooterMetrics(new AbfsReadFooterMetrics()) .withShouldReadBufferSizeAlways( abfsConfiguration.shouldReadBufferSizeAlways()) .withReadAheadBlockSize(abfsConfiguration.getReadAheadBlockSize()) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsCounters.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsCounters.java index 756e6e173e36b..c65640c1080cc 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsCounters.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsCounters.java @@ -77,5 +77,5 @@ String formString(String prefix, String separator, String suffix, AbfsDriverMetrics getAbfsDriverMetrics(); - List getAbfsInputStreamMetrics(); + List getAbfsReadFooterMetrics(); } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java index ef66fac1390a1..f85deed6854cf 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java @@ -106,7 +106,7 @@ public class AbfsInputStream extends FSInputStream implements CanUnbuffer, private int bCursorBkp; private long fCursorBkp; private long fCursorAfterLastReadBkp; - private final AbfsInputStreamMetrics inputStreamMetrics; + private final AbfsReadFooterMetrics abfsReadFooterMetrics; /** Stream statistics. */ private final AbfsInputStreamStatistics streamStatistics; private long bytesFromReadAhead; // bytes read from readAhead; for testing @@ -148,7 +148,7 @@ public AbfsInputStream( this.cachedSasToken = new CachedSASToken( abfsInputStreamContext.getSasTokenRenewPeriodForStreamsInSeconds()); this.streamStatistics = abfsInputStreamContext.getStreamStatistics(); - this.inputStreamMetrics = abfsInputStreamContext.getAbfsInputStreamMetrics(); + this.abfsReadFooterMetrics = abfsInputStreamContext.getAbfsReadFooterMetrics(); this.inputStreamId = createInputStreamId(); this.tracingContext = new TracingContext(tracingContext); this.tracingContext.setOperation(FSOperationType.READ); @@ -247,11 +247,11 @@ public synchronized int read(final byte[] b, final int off, final int len) throw this.collectStreamMetrics = true; this.collectMetricsForNextRead = true; this.offsetOfFirstRead = nextReadPos; - this.inputStreamMetrics.getSizeReadByFirstRead().set(len); - this.inputStreamMetrics.getFileLength().set(contentLength); + this.abfsReadFooterMetrics.setSizeReadByFirstRead(len+"_"+(Math.abs(contentLength - nextReadPos))); + this.abfsReadFooterMetrics.getFileLength().set(contentLength); } if (!firstRead && collectMetricsForNextRead){ - this.inputStreamMetrics.getOffsetDiffBetweenFirstAndSecondRead().set(Math.abs(nextReadPos - offsetOfFirstRead)); + this.abfsReadFooterMetrics.setOffsetDiffBetweenFirstAndSecondRead(len+"_"+(Math.abs(nextReadPos - offsetOfFirstRead))); this.collectMetricsForNextRead = false; } if (nextReadPos >= filePosAtStartOfBuffer && nextReadPos <= fCursor) { @@ -711,13 +711,24 @@ public synchronized void close() throws IOException { closed = true; buffer = null; // de-reference the buffer so it can be GC'ed sooner if(this.collectStreamMetrics) { + checkIsParquet(abfsReadFooterMetrics); this.client.getAbfsCounters() - .getAbfsInputStreamMetrics() - .add(inputStreamMetrics); + .getAbfsReadFooterMetrics() + .add(abfsReadFooterMetrics); } ReadBufferManager.getBufferManager().purgeBuffersForStream(this); } + private void checkIsParquet(AbfsReadFooterMetrics abfsReadFooterMetrics) { + String[] firstReadSize = abfsReadFooterMetrics.getSizeReadByFirstRead().split("_"); + String[] offDiffFirstSecondRead = abfsReadFooterMetrics.getOffsetDiffBetweenFirstAndSecondRead().split("_"); + if((firstReadSize[0].equals(firstReadSize[1])) && (offDiffFirstSecondRead[0].equals(offDiffFirstSecondRead[1]))){ + abfsReadFooterMetrics.setParquetFile(true); + abfsReadFooterMetrics.setSizeReadByFirstRead(firstReadSize[0]); + abfsReadFooterMetrics.setOffsetDiffBetweenFirstAndSecondRead(offDiffFirstSecondRead[0]); + } + } + /** * Not supported by this stream. Throws {@link UnsupportedOperationException} * @param readlimit ignored diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStreamContext.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStreamContext.java index d555cd847fb66..5dc3bec2eb87a 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStreamContext.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStreamContext.java @@ -49,7 +49,7 @@ public class AbfsInputStreamContext extends AbfsStreamContext { private boolean bufferedPreadDisabled; - private AbfsInputStreamMetrics abfsInputStreamMetrics; + private AbfsReadFooterMetrics abfsReadFooterMetrics; public AbfsInputStreamContext(final long sasTokenRenewPeriodForStreamsInSeconds) { super(sasTokenRenewPeriodForStreamsInSeconds); @@ -86,9 +86,9 @@ public AbfsInputStreamContext withStreamStatistics( return this; } - public AbfsInputStreamContext withStreamMetrics( - final AbfsInputStreamMetrics inputStreamMetrics) { - this.abfsInputStreamMetrics = inputStreamMetrics; + public AbfsInputStreamContext withReadFooterMetrics( + final AbfsReadFooterMetrics abfsReadFooterMetrics) { + this.abfsReadFooterMetrics = abfsReadFooterMetrics; return this; } @@ -156,8 +156,8 @@ public int getReadAheadRange() { public AbfsInputStreamStatistics getStreamStatistics() { return streamStatistics; } - public AbfsInputStreamMetrics getAbfsInputStreamMetrics() { - return abfsInputStreamMetrics; + public AbfsReadFooterMetrics getAbfsReadFooterMetrics() { + return abfsReadFooterMetrics; } public boolean readSmallFilesCompletely() { diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStreamMetrics.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsReadFooterMetrics.java similarity index 56% rename from hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStreamMetrics.java rename to hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsReadFooterMetrics.java index 596b8602ae7a4..20281579be639 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStreamMetrics.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsReadFooterMetrics.java @@ -19,25 +19,46 @@ import java.util.concurrent.atomic.AtomicLong; -public class AbfsInputStreamMetrics { - private AtomicLong sizeReadByFirstRead; - private AtomicLong offsetDiffBetweenFirstAndSecondRead; +public class AbfsReadFooterMetrics { + private boolean isParquetFile; + private String sizeReadByFirstRead; + private String offsetDiffBetweenFirstAndSecondRead; private AtomicLong fileLength; - public AbfsInputStreamMetrics() { - this.sizeReadByFirstRead = new AtomicLong(); - this.offsetDiffBetweenFirstAndSecondRead = new AtomicLong(); + public AbfsReadFooterMetrics() { this.fileLength = new AtomicLong(); } - public AtomicLong getSizeReadByFirstRead() { + + public boolean getIsParquetFile() { + return isParquetFile; + } + + public void setParquetFile(final boolean parquetFile) { + isParquetFile = parquetFile; + } + + public String getSizeReadByFirstRead() { return sizeReadByFirstRead; } - public AtomicLong getOffsetDiffBetweenFirstAndSecondRead() { + public void setSizeReadByFirstRead(final String sizeReadByFirstRead) { + this.sizeReadByFirstRead = sizeReadByFirstRead; + } + + public String getOffsetDiffBetweenFirstAndSecondRead() { return offsetDiffBetweenFirstAndSecondRead; } + public void setOffsetDiffBetweenFirstAndSecondRead(final String offsetDiffBetweenFirstAndSecondRead) { + this.offsetDiffBetweenFirstAndSecondRead + = offsetDiffBetweenFirstAndSecondRead; + } + public AtomicLong getFileLength() { return fileLength; } + + public void setFileLength(final AtomicLong fileLength) { + this.fileLength = fileLength; + } }