From 98d2901f1ba79a2483e890676e5eed869514caad Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Thu, 12 May 2022 15:26:11 +0530 Subject: [PATCH 01/42] Metric --- .../hadoop/fs/azurebfs/AbfsCountersImpl.java | 8 +++++++ .../hadoop/fs/azurebfs/AbfsDriverMetrics.java | 23 +++++++++++++++++++ .../fs/azurebfs/AzureBlobFileSystem.java | 2 +- .../fs/azurebfs/services/AbfsClient.java | 5 ++-- .../azurebfs/services/AbfsRestOperation.java | 7 ++++-- 5 files changed, 39 insertions(+), 6 deletions(-) create mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsCountersImpl.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsCountersImpl.java index c478256e706bc..d44e70f16eceb 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsCountersImpl.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsCountersImpl.java @@ -36,6 +36,7 @@ import static org.apache.hadoop.fs.azurebfs.AbfsStatistic.*; import static org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.iostatisticsStore; +import java.util.concurrent.atomic.AtomicReference; /** * Instrumentation of Abfs counters. @@ -63,6 +64,8 @@ public class AbfsCountersImpl implements AbfsCounters { private final IOStatisticsStore ioStatisticsStore; + private AtomicReference abfsDriverMetrics = null; + private static final AbfsStatistic[] STATISTIC_LIST = { CALL_CREATE, CALL_OPEN, @@ -117,6 +120,7 @@ public AbfsCountersImpl(URI uri) { ioStatisticsStoreBuilder.withDurationTracking(durationStats.getStatName()); } ioStatisticsStore = ioStatisticsStoreBuilder.build(); + abfsDriverMetrics = new AtomicReference(new AbfsDriverMetrics()); } /** @@ -184,6 +188,10 @@ private MetricsRegistry getRegistry() { return registry; } + public AbfsDriverMetrics getAbfsDriverMetrics() { + return abfsDriverMetrics.get(); + } + /** * {@inheritDoc} * diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java new file mode 100644 index 0000000000000..01393e92cf3cf --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java @@ -0,0 +1,23 @@ +package org.apache.hadoop.fs.azurebfs; + +import java.util.concurrent.atomic.AtomicLong; + +public class AbfsDriverMetrics { + private AtomicLong numberOfRequestsSucceededInFirstRetry; + private AtomicLong numberOfRequestsSucceededInSecondRetry; + private AtomicLong numberOfRequestsSucceededInThirdRetry; + private AtomicLong numberOfRequestsSucceededInFourthRetry; + private AtomicLong numberOfRequestsSucceededInFiveToFifteenRetry; + private AtomicLong numberOfRequestsSucceededInFifteenToTwentyFiveRetry; + private AtomicLong numberOfRequestsSucceededInTwentyFiveToThirtyRetry; + + public AbfsDriverMetrics() { + this.numberOfRequestsSucceededInFirstRetry = new AtomicLong(); + this.numberOfRequestsSucceededInSecondRetry = new AtomicLong(); + this.numberOfRequestsSucceededInThirdRetry = new AtomicLong(); + this.numberOfRequestsSucceededInFourthRetry = new AtomicLong(); + this.numberOfRequestsSucceededInFiveToFifteenRetry = new AtomicLong(); + this.numberOfRequestsSucceededInFifteenToTwentyFiveRetry = new AtomicLong(); + this.numberOfRequestsSucceededInTwentyFiveToThirtyRetry = new AtomicLong(); + } +} diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java index 46141e7c4a838..1b36c0103703d 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java @@ -49,7 +49,7 @@ import org.apache.hadoop.util.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; - +import java.util.concurrent.atomic.AtomicLong; import org.apache.commons.lang3.ArrayUtils; import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.classification.InterfaceAudience; diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java index b701037d0fc41..fcb4d615476c0 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java @@ -308,7 +308,6 @@ public AbfsRestOperation listPath(final String relativePath, final boolean recur public AbfsRestOperation getFilesystemProperties(TracingContext tracingContext) throws AzureBlobFileSystemException { final List requestHeaders = createDefaultHeaders(); - final AbfsUriQueryBuilder abfsUriQueryBuilder = createDefaultUriQueryBuilder(); abfsUriQueryBuilder.addQuery(QUERY_PARAM_RESOURCE, FILESYSTEM); @@ -754,7 +753,8 @@ public AbfsRestOperation setPathProperties(final String path, final String prope public AbfsRestOperation getPathStatus(final String path, final boolean includeProperties, TracingContext tracingContext) throws AzureBlobFileSystemException { final List requestHeaders = createDefaultHeaders(); - + String headerValue = "Path=%2Fsomepath%2Ffile.txt,Operation=ReadFile,ReadAheadHit=10,ReadAheadMiss=5"; + requestHeaders.add(new AbfsHttpHeader(X_MS_CLIENT_METRICS, headerValue)); final AbfsUriQueryBuilder abfsUriQueryBuilder = createDefaultUriQueryBuilder(); String operation = SASTokenProvider.GET_PROPERTIES_OPERATION; if (!includeProperties) { @@ -984,7 +984,6 @@ public AbfsRestOperation getAclStatus(final String path, TracingContext tracingC public AbfsRestOperation getAclStatus(final String path, final boolean useUPN, TracingContext tracingContext) throws AzureBlobFileSystemException { final List requestHeaders = createDefaultHeaders(); - final AbfsUriQueryBuilder abfsUriQueryBuilder = createDefaultUriQueryBuilder(); abfsUriQueryBuilder.addQuery(HttpQueryParams.QUERY_PARAM_ACTION, AbfsHttpConstants.GET_ACCESS_CONTROL); abfsUriQueryBuilder.addQuery(HttpQueryParams.QUERY_PARAM_UPN, String.valueOf(useUPN)); diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java index 74b267d563eb2..e67b57f05a735 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java @@ -36,7 +36,7 @@ import org.apache.hadoop.fs.azurebfs.constants.HttpHeaderConfigurations; import org.apache.hadoop.fs.azurebfs.utils.TracingContext; import org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding; - +import org.apache.hadoop.fs.azurebfs.AbfsDriverMetrics; /** * The AbfsRestOperation for Rest AbfsClient. */ @@ -71,6 +71,8 @@ public class AbfsRestOperation { private AbfsHttpOperation result; private AbfsCounters abfsCounters; + private AbfsDriverMetrics abfsDriverMetrics; + /** * Checks if there is non-null HTTP response. * @return true if there is a non-null HTTP response from the ABFS call. @@ -145,6 +147,7 @@ String getSasToken() { || AbfsHttpConstants.HTTP_METHOD_PATCH.equals(method)); this.sasToken = sasToken; this.abfsCounters = client.getAbfsCounters(); + this.abfsDriverMetrics = abfsCounters.getAbfsDriverMetrics(); } /** @@ -175,6 +178,7 @@ String getSasToken() { this.bufferOffset = bufferOffset; this.bufferLength = bufferLength; this.abfsCounters = client.getAbfsCounters(); + this.abfsDriverMetrics = abfsCounters.getAbfsDriverMetrics(); } /** @@ -225,7 +229,6 @@ private void completeExecute(TracingContext tracingContext) Thread.currentThread().interrupt(); } } - if (result.getStatusCode() >= HttpURLConnection.HTTP_BAD_REQUEST) { throw new AbfsRestOperationException(result.getStatusCode(), result.getStorageErrorCode(), result.getStorageErrorMessage(), null, result); From 16612d30d19550780fd90cf02abde98293525770 Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Mon, 16 May 2022 12:25:12 +0530 Subject: [PATCH 02/42] Metric changes --- .../hadoop/fs/azurebfs/AbfsDriverMetrics.java | 35 ++++++++++++++ .../fs/azurebfs/AzureBlobFileSystem.java | 18 ++++++++ .../fs/azurebfs/AzureBlobFileSystemStore.java | 4 ++ .../constants/FileSystemConfigurations.java | 2 +- .../fs/azurebfs/services/AbfsClient.java | 24 +++++++++- .../fs/azurebfs/services/AbfsCounters.java | 4 +- .../azurebfs/services/AbfsHttpOperation.java | 2 +- .../azurebfs/services/AbfsRestOperation.java | 46 +++++++++++++++++++ .../fs/azurebfs/utils/TracingContext.java | 2 +- 9 files changed, 131 insertions(+), 6 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java index 01393e92cf3cf..7ba28c6c5df04 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java @@ -20,4 +20,39 @@ public AbfsDriverMetrics() { this.numberOfRequestsSucceededInFifteenToTwentyFiveRetry = new AtomicLong(); this.numberOfRequestsSucceededInTwentyFiveToThirtyRetry = new AtomicLong(); } + + public AtomicLong getNumberOfRequestsSucceededInFirstRetry() { + return numberOfRequestsSucceededInFirstRetry; + } + + public AtomicLong getNumberOfRequestsSucceededInSecondRetry() { + return numberOfRequestsSucceededInSecondRetry; + } + + public AtomicLong getNumberOfRequestsSucceededInThirdRetry() { + return numberOfRequestsSucceededInThirdRetry; + } + + public AtomicLong getNumberOfRequestsSucceededInFourthRetry() { + return numberOfRequestsSucceededInFourthRetry; + } + + public AtomicLong getNumberOfRequestsSucceededInFiveToFifteenRetry() { + return numberOfRequestsSucceededInFiveToFifteenRetry; + } + + public AtomicLong getNumberOfRequestsSucceededInFifteenToTwentyFiveRetry() { + return numberOfRequestsSucceededInFifteenToTwentyFiveRetry; + } + + public AtomicLong getNumberOfRequestsSucceededInTwentyFiveToThirtyRetry() { + return numberOfRequestsSucceededInTwentyFiveToThirtyRetry; + } + + @Override + public String toString() { + return "#RCTSI_1R_ " + numberOfRequestsSucceededInFirstRetry + "#RCTSI_2R_ " + numberOfRequestsSucceededInSecondRetry + "#RCTSI_3R_ " + + numberOfRequestsSucceededInThirdRetry + "#RCTSI_4R_ " + numberOfRequestsSucceededInFourthRetry + "#RCTSI_5-15R_ " + numberOfRequestsSucceededInFiveToFifteenRetry + + "#RCTSI_15-25R_ " + numberOfRequestsSucceededInFifteenToTwentyFiveRetry + "#RCTSI_25-30R_ " + numberOfRequestsSucceededInTwentyFiveToThirtyRetry; + } } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java index 1b36c0103703d..6853c2a45e80a 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java @@ -678,6 +678,17 @@ public synchronized void close() throws IOException { if (isClosed) { return; } + try { + //String metric = abfsCounters.getAbfsDriverMetrics().toString(); + String metric = "abcd"; + URI metricUri = new URI( + "abfs://metric@snvarmacanary.dfs.core.windows.net"); + AzureBlobFileSystem metricfs = (AzureBlobFileSystem) FileSystem.get( + metricUri, getConf()); + metricfs.sentMetric(metric); + } catch (Exception ex) { + // do nothing + } // does all the delete-on-exit calls, and may be slow. super.close(); LOG.debug("AzureBlobFileSystem.close"); @@ -694,6 +705,13 @@ public synchronized void close() throws IOException { } } + public void sentMetric(String metric) throws AzureBlobFileSystemException { + TracingContext tracingContext = new TracingContext(clientCorrelationId, + fileSystemId, FSOperationType.SET_ATTR, true, tracingHeaderFormat, + listener); + abfsStore.sentMetric(metric, tracingContext); + } + @Override public FileStatus getFileStatus(final Path f) throws IOException { TracingContext tracingContext = new TracingContext(clientCorrelationId, diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java index 09b48a855f00b..681f991facbe2 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java @@ -1973,4 +1973,8 @@ public static String extractEtagHeader(AbfsHttpOperation result) { } return etag; } + + public void sentMetric(String metric, TracingContext tracingContext) throws AzureBlobFileSystemException{ + AbfsRestOperation op = client.getPathStatusMetric("/..$$@@", tracingContext, metric); // Will sent a GFS calls that will fail to register in MDM x-ms-client-metric + } } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java index 63d62a33b1819..f510824041bce 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java @@ -41,7 +41,7 @@ public final class FileSystemConfigurations { public static final int DEFAULT_MIN_BACKOFF_INTERVAL = 3 * 1000; // 3s public static final int DEFAULT_MAX_BACKOFF_INTERVAL = 30 * 1000; // 30s public static final int DEFAULT_BACKOFF_INTERVAL = 3 * 1000; // 3s - public static final int DEFAULT_MAX_RETRY_ATTEMPTS = 30; + public static final int DEFAULT_MAX_RETRY_ATTEMPTS = 5; public static final int DEFAULT_CUSTOM_TOKEN_FETCH_RETRY_COUNT = 3; // Retry parameter defaults. diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java index fcb4d615476c0..aaa6b384486af 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java @@ -750,11 +750,31 @@ public AbfsRestOperation setPathProperties(final String path, final String prope return op; } + public AbfsRestOperation getPathStatusMetric(final String path, + TracingContext tracingContext, String metric) throws AzureBlobFileSystemException { + final List requestHeaders = createDefaultHeaders(); + requestHeaders.add(new AbfsHttpHeader(X_MS_CLIENT_REQUEST_ID, metric)); + final AbfsUriQueryBuilder abfsUriQueryBuilder = createDefaultUriQueryBuilder(); + String operation = SASTokenProvider.GET_PROPERTIES_OPERATION; + abfsUriQueryBuilder.addQuery(HttpQueryParams.QUERY_PARAM_ACTION, AbfsHttpConstants.GET_STATUS); + operation = SASTokenProvider.GET_STATUS_OPERATION; + abfsUriQueryBuilder.addQuery(HttpQueryParams.QUERY_PARAM_UPN, String.valueOf(abfsConfiguration.isUpnUsed())); + appendSASTokenToQuery(path, operation, abfsUriQueryBuilder); + + final URL url = createRequestUrl(path, abfsUriQueryBuilder.toString()); + final AbfsRestOperation op = new AbfsRestOperation( + AbfsRestOperationType.GetPathStatus, + this, + HTTP_METHOD_HEAD, + url, + requestHeaders); + op.execute(tracingContext); + return op; + } + public AbfsRestOperation getPathStatus(final String path, final boolean includeProperties, TracingContext tracingContext) throws AzureBlobFileSystemException { final List requestHeaders = createDefaultHeaders(); - String headerValue = "Path=%2Fsomepath%2Ffile.txt,Operation=ReadFile,ReadAheadHit=10,ReadAheadMiss=5"; - requestHeaders.add(new AbfsHttpHeader(X_MS_CLIENT_METRICS, headerValue)); final AbfsUriQueryBuilder abfsUriQueryBuilder = createDefaultUriQueryBuilder(); String operation = SASTokenProvider.GET_PROPERTIES_OPERATION; if (!includeProperties) { diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsCounters.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsCounters.java index d01a3598afcf8..91eb03c821566 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsCounters.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsCounters.java @@ -28,7 +28,7 @@ import org.apache.hadoop.fs.statistics.DurationTracker; import org.apache.hadoop.fs.statistics.DurationTrackerFactory; import org.apache.hadoop.fs.statistics.IOStatisticsSource; - +import org.apache.hadoop.fs.azurebfs.AbfsDriverMetrics; /** * An interface for Abfs counters. */ @@ -74,4 +74,6 @@ String formString(String prefix, String separator, String suffix, */ @Override DurationTracker trackDuration(String key); + + AbfsDriverMetrics getAbfsDriverMetrics(); } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsHttpOperation.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsHttpOperation.java index 413bf3686898b..0cfbf03106227 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsHttpOperation.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsHttpOperation.java @@ -345,7 +345,7 @@ public void processResponse(final byte[] buffer, final int offset, final int len } this.statusCode = this.connection.getResponseCode(); - + //this.statusCode = 503; if (this.isTraceEnabled) { this.recvResponseTimeMs = elapsedTimeMs(startTime); } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java index e67b57f05a735..7b3a8aba132d4 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java @@ -37,6 +37,7 @@ import org.apache.hadoop.fs.azurebfs.utils.TracingContext; import org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding; import org.apache.hadoop.fs.azurebfs.AbfsDriverMetrics; +import java.util.concurrent.atomic.AtomicLong; /** * The AbfsRestOperation for Rest AbfsClient. */ @@ -229,6 +230,51 @@ private void completeExecute(TracingContext tracingContext) Thread.currentThread().interrupt(); } } + if(retryCount > 0) { + if (retryCount == 1) { + long noOfRequests1Retry + = abfsDriverMetrics.getNumberOfRequestsSucceededInFirstRetry() + .incrementAndGet(); + abfsDriverMetrics.getNumberOfRequestsSucceededInFirstRetry() + .set(noOfRequests1Retry); + } else if (retryCount == 2) { + long noOfRequests2Retry + = abfsDriverMetrics.getNumberOfRequestsSucceededInSecondRetry() + .incrementAndGet(); + abfsDriverMetrics.getNumberOfRequestsSucceededInSecondRetry() + .set(noOfRequests2Retry); + } else if (retryCount == 3) { + long noOfRequests3Retry + = abfsDriverMetrics.getNumberOfRequestsSucceededInThirdRetry() + .incrementAndGet(); + abfsDriverMetrics.getNumberOfRequestsSucceededInThirdRetry() + .set(noOfRequests3Retry); + } else if (retryCount == 4) { + long noOfRequests4Retry + = abfsDriverMetrics.getNumberOfRequestsSucceededInFourthRetry() + .incrementAndGet(); + abfsDriverMetrics.getNumberOfRequestsSucceededInFourthRetry() + .set(noOfRequests4Retry); + } else if (retryCount >= 5 && retryCount < 15) { + long noOfRequests5_15Retry + = abfsDriverMetrics.getNumberOfRequestsSucceededInFiveToFifteenRetry() + .incrementAndGet(); + abfsDriverMetrics.getNumberOfRequestsSucceededInFiveToFifteenRetry() + .set(noOfRequests5_15Retry); + } else if (retryCount >= 15 && retryCount < 25) { + long noOfRequests15_25Retry + = abfsDriverMetrics.getNumberOfRequestsSucceededInFifteenToTwentyFiveRetry() + .incrementAndGet(); + abfsDriverMetrics.getNumberOfRequestsSucceededInFifteenToTwentyFiveRetry() + .set(noOfRequests15_25Retry); + } else { + long noOfRequests25_30Retry + = abfsDriverMetrics.getNumberOfRequestsSucceededInTwentyFiveToThirtyRetry() + .incrementAndGet(); + abfsDriverMetrics.getNumberOfRequestsSucceededInTwentyFiveToThirtyRetry() + .set(noOfRequests25_30Retry); + } + } if (result.getStatusCode() >= HttpURLConnection.HTTP_BAD_REQUEST) { throw new AbfsRestOperationException(result.getStatusCode(), result.getStorageErrorCode(), result.getStorageErrorMessage(), null, result); diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/TracingContext.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/TracingContext.java index 5a115451df159..05fe33b7a19da 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/TracingContext.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/TracingContext.java @@ -171,7 +171,7 @@ public void constructHeader(AbfsHttpOperation httpOperation) { if (listener != null) { //for testing listener.callTracingHeaderValidator(header, format); } - httpOperation.setRequestProperty(HttpHeaderConfigurations.X_MS_CLIENT_REQUEST_ID, header); + //httpOperation.setRequestProperty(HttpHeaderConfigurations.X_MS_CLIENT_REQUEST_ID, header); } /** From fb2295ea13f10c2ce911137466a9e9c02e68282e Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Fri, 20 May 2022 15:03:14 +0530 Subject: [PATCH 03/42] Metric changes --- .../hadoop/fs/azurebfs/AbfsDriverMetrics.java | 14 ++++++++--- .../fs/azurebfs/AzureBlobFileSystem.java | 23 ++++++++++++++----- .../azurebfs/constants/ConfigurationKeys.java | 5 ++++ .../azurebfs/services/AbfsRestOperation.java | 6 ++++- 4 files changed, 38 insertions(+), 10 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java index 7ba28c6c5df04..69d57c5ac1eb3 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java @@ -11,6 +11,8 @@ public class AbfsDriverMetrics { private AtomicLong numberOfRequestsSucceededInFifteenToTwentyFiveRetry; private AtomicLong numberOfRequestsSucceededInTwentyFiveToThirtyRetry; + private AtomicLong minBackoffForFirstRetry; + public AbfsDriverMetrics() { this.numberOfRequestsSucceededInFirstRetry = new AtomicLong(); this.numberOfRequestsSucceededInSecondRetry = new AtomicLong(); @@ -19,6 +21,7 @@ public AbfsDriverMetrics() { this.numberOfRequestsSucceededInFiveToFifteenRetry = new AtomicLong(); this.numberOfRequestsSucceededInFifteenToTwentyFiveRetry = new AtomicLong(); this.numberOfRequestsSucceededInTwentyFiveToThirtyRetry = new AtomicLong(); + this.minBackoffForFirstRetry = new AtomicLong(Long.MAX_VALUE); } public AtomicLong getNumberOfRequestsSucceededInFirstRetry() { @@ -49,10 +52,15 @@ public AtomicLong getNumberOfRequestsSucceededInTwentyFiveToThirtyRetry() { return numberOfRequestsSucceededInTwentyFiveToThirtyRetry; } + public AtomicLong getMinBackoffForFirstRetry() { + return minBackoffForFirstRetry; + } + @Override public String toString() { - return "#RCTSI_1R_ " + numberOfRequestsSucceededInFirstRetry + "#RCTSI_2R_ " + numberOfRequestsSucceededInSecondRetry + "#RCTSI_3R_ " + - numberOfRequestsSucceededInThirdRetry + "#RCTSI_4R_ " + numberOfRequestsSucceededInFourthRetry + "#RCTSI_5-15R_ " + numberOfRequestsSucceededInFiveToFifteenRetry + - "#RCTSI_15-25R_ " + numberOfRequestsSucceededInFifteenToTwentyFiveRetry + "#RCTSI_25-30R_ " + numberOfRequestsSucceededInTwentyFiveToThirtyRetry; + return "#RCTSI_1R_" + numberOfRequestsSucceededInFirstRetry + " #RCTSI_2R_" + numberOfRequestsSucceededInSecondRetry + " #RCTSI_3R_" + + numberOfRequestsSucceededInThirdRetry + " #RCTSI_4R_" + numberOfRequestsSucceededInFourthRetry + " #RCTSI_5-15R_" + numberOfRequestsSucceededInFiveToFifteenRetry + + " #RCTSI_15-25R_" + numberOfRequestsSucceededInFifteenToTwentyFiveRetry + " #RCTSI_25-30R_" + numberOfRequestsSucceededInTwentyFiveToThirtyRetry + + " #Min_1R_ " + minBackoffForFirstRetry; } } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java index 6853c2a45e80a..8e67f36b368d0 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java @@ -114,8 +114,11 @@ import static org.apache.hadoop.fs.CommonConfigurationKeys.IOSTATISTICS_LOGGING_LEVEL_DEFAULT; import static org.apache.hadoop.fs.azurebfs.AbfsStatistic.*; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.DATA_BLOCKS_BUFFER; +import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_BLOCK_UPLOAD_ACTIVE_BLOCKS; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_BLOCK_UPLOAD_BUFFER_DIR; +import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_METRIC_ACCOUNT_KEY; +import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_METRIC_ACCOUNT_NAME; import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.BLOCK_UPLOAD_ACTIVE_BLOCKS_DEFAULT; import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DATA_BLOCKS_BUFFER_DEFAULT; import static org.apache.hadoop.fs.impl.PathCapabilitiesSupport.validatePathCapabilityArgs; @@ -679,12 +682,20 @@ public synchronized void close() throws IOException { return; } try { - //String metric = abfsCounters.getAbfsDriverMetrics().toString(); - String metric = "abcd"; - URI metricUri = new URI( - "abfs://metric@snvarmacanary.dfs.core.windows.net"); - AzureBlobFileSystem metricfs = (AzureBlobFileSystem) FileSystem.get( - metricUri, getConf()); + String metric = abfsCounters.getAbfsDriverMetrics().toString(); + Configuration metricConfig = getConf(); + String metricAccountName = getConf().get(FS_AZURE_METRIC_ACCOUNT_NAME); + String metricAccountKey = getConf().get(FS_AZURE_METRIC_ACCOUNT_KEY); + final String abfsMetricUrl = "metrics" + "@" + metricAccountName; + metricConfig.set(FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME, metricAccountKey); + URI metricUri = null; + try { + metricUri = new URI(getScheme(), abfsMetricUrl, null, null, null); + } catch (Exception ex) { + throw new AssertionError(ex); + } + AzureBlobFileSystem metricfs = (AzureBlobFileSystem) FileSystem.newInstance( + metricUri, metricConfig); metricfs.sentMetric(metric); } catch (Exception ex) { // do nothing diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java index 9d3b2d5e82c6e..99963fe0cda89 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java @@ -36,6 +36,11 @@ public final class ConfigurationKeys { */ public static final String FS_AZURE_ACCOUNT_IS_HNS_ENABLED = "fs.azure.account.hns.enabled"; public static final String FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME = "fs.azure.account.key"; + + public static final String FS_AZURE_METRIC_ACCOUNT_NAME = "fs.azure.metric.account.name"; + + public static final String FS_AZURE_METRIC_ACCOUNT_KEY = "fs.azure.metric.account.key"; + public static final String FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME_REGX = "fs\\.azure\\.account\\.key\\.(.*)"; public static final String FS_AZURE_SECURE_MODE = "fs.azure.secure.mode"; diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java index 7b3a8aba132d4..8ff8a281eabdf 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java @@ -219,24 +219,28 @@ private void completeExecute(TracingContext tracingContext) retryCount = 0; LOG.debug("First execution of REST operation - {}", operationType); + long sleepDuration = 0L; while (!executeHttpOperation(retryCount, tracingContext)) { try { ++retryCount; tracingContext.setRetryCount(retryCount); LOG.debug("Retrying REST operation {}. RetryCount = {}", operationType, retryCount); - Thread.sleep(client.getRetryPolicy().getRetryInterval(retryCount)); + sleepDuration = client.getRetryPolicy().getRetryInterval(retryCount); + Thread.sleep(sleepDuration); } catch (InterruptedException ex) { Thread.currentThread().interrupt(); } } if(retryCount > 0) { if (retryCount == 1) { + long minSleepForFirstRetry = Math.min(abfsDriverMetrics.getMinBackoffForFirstRetry().get(), sleepDuration); long noOfRequests1Retry = abfsDriverMetrics.getNumberOfRequestsSucceededInFirstRetry() .incrementAndGet(); abfsDriverMetrics.getNumberOfRequestsSucceededInFirstRetry() .set(noOfRequests1Retry); + abfsDriverMetrics.getMinBackoffForFirstRetry().set(minSleepForFirstRetry); } else if (retryCount == 2) { long noOfRequests2Retry = abfsDriverMetrics.getNumberOfRequestsSucceededInSecondRetry() From 986d2e6ff24078a32a8aedd5fa66482295eb2720 Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Fri, 27 May 2022 11:37:36 +0530 Subject: [PATCH 04/42] Metrics --- .../hadoop/fs/azurebfs/AbfsCountersImpl.java | 2 +- .../hadoop/fs/azurebfs/AbfsDriverMetrics.java | 127 ++++++++++++------ .../fs/azurebfs/AzureBlobFileSystem.java | 5 +- .../constants/FileSystemConfigurations.java | 2 +- .../services/AzureServiceErrorCode.java | 1 + .../azurebfs/services/AbfsRestOperation.java | 98 +++++++------- .../azurebfs/ITestAbfsReadWriteAndSeek.java | 13 +- 7 files changed, 153 insertions(+), 95 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsCountersImpl.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsCountersImpl.java index d44e70f16eceb..1371cee6f3c64 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsCountersImpl.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsCountersImpl.java @@ -120,7 +120,7 @@ public AbfsCountersImpl(URI uri) { ioStatisticsStoreBuilder.withDurationTracking(durationStats.getStatName()); } ioStatisticsStore = ioStatisticsStoreBuilder.build(); - abfsDriverMetrics = new AtomicReference(new AbfsDriverMetrics()); + abfsDriverMetrics = new AtomicReference<>(new AbfsDriverMetrics()); } /** diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java index 69d57c5ac1eb3..ee38e151016b0 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java @@ -1,66 +1,115 @@ package org.apache.hadoop.fs.azurebfs; -import java.util.concurrent.atomic.AtomicLong; - +import java.util.concurrent.atomic.AtomicLong; +import java.util.Map; +import java.util.LinkedHashMap; +import java.util.ArrayList; +import java.util.Arrays; public class AbfsDriverMetrics { - private AtomicLong numberOfRequestsSucceededInFirstRetry; - private AtomicLong numberOfRequestsSucceededInSecondRetry; - private AtomicLong numberOfRequestsSucceededInThirdRetry; - private AtomicLong numberOfRequestsSucceededInFourthRetry; - private AtomicLong numberOfRequestsSucceededInFiveToFifteenRetry; - private AtomicLong numberOfRequestsSucceededInFifteenToTwentyFiveRetry; - private AtomicLong numberOfRequestsSucceededInTwentyFiveToThirtyRetry; + private AtomicLong numberOfRequestsSucceeded; + private AtomicLong minBackoff; + private AtomicLong maxBackoff; + private AtomicLong totalRequests; + private AtomicLong totalBackoff; + private String retryCount; + private AtomicLong numberOfIOPSThrottledRequests; + private AtomicLong numberOfBandwidthThrottledRequests; + + private AtomicLong numberOfOtherThrottledRequests; - private AtomicLong minBackoffForFirstRetry; + private static final Map metricsMap = new LinkedHashMap<>(); public AbfsDriverMetrics() { - this.numberOfRequestsSucceededInFirstRetry = new AtomicLong(); - this.numberOfRequestsSucceededInSecondRetry = new AtomicLong(); - this.numberOfRequestsSucceededInThirdRetry = new AtomicLong(); - this.numberOfRequestsSucceededInFourthRetry = new AtomicLong(); - this.numberOfRequestsSucceededInFiveToFifteenRetry = new AtomicLong(); - this.numberOfRequestsSucceededInFifteenToTwentyFiveRetry = new AtomicLong(); - this.numberOfRequestsSucceededInTwentyFiveToThirtyRetry = new AtomicLong(); - this.minBackoffForFirstRetry = new AtomicLong(Long.MAX_VALUE); + initializeMap(); + this.numberOfIOPSThrottledRequests = new AtomicLong(); + this.numberOfBandwidthThrottledRequests = new AtomicLong(); + this.numberOfOtherThrottledRequests = new AtomicLong(); + } + public AbfsDriverMetrics(String retryCount) { + this.retryCount = retryCount; + this.numberOfRequestsSucceeded = new AtomicLong(); + this.minBackoff = new AtomicLong(Long.MAX_VALUE); + this.maxBackoff = new AtomicLong(); + this.totalRequests = new AtomicLong(); + this.totalBackoff = new AtomicLong(); + } + + private void initializeMap() { + ArrayList retryCountList = new ArrayList(Arrays.asList("1","2","3","4","5_15","15_25","25_30")); + for (String s : retryCountList) { + metricsMap.put(s, new AbfsDriverMetrics(s)); + } + } + + public AtomicLong getNumberOfRequestsSucceeded() { + return numberOfRequestsSucceeded; + } + + public AtomicLong getMinBackoff() { + return minBackoff; } - public AtomicLong getNumberOfRequestsSucceededInFirstRetry() { - return numberOfRequestsSucceededInFirstRetry; + public AtomicLong getMaxBackoff() { + return maxBackoff; } - public AtomicLong getNumberOfRequestsSucceededInSecondRetry() { - return numberOfRequestsSucceededInSecondRetry; + public AtomicLong getTotalRequests() { + return totalRequests; } - public AtomicLong getNumberOfRequestsSucceededInThirdRetry() { - return numberOfRequestsSucceededInThirdRetry; + public AtomicLong getTotalBackoff() { + return totalBackoff; } - public AtomicLong getNumberOfRequestsSucceededInFourthRetry() { - return numberOfRequestsSucceededInFourthRetry; + public String getRetryCount() { + return retryCount; } - public AtomicLong getNumberOfRequestsSucceededInFiveToFifteenRetry() { - return numberOfRequestsSucceededInFiveToFifteenRetry; + public AtomicLong getNumberOfIOPSThrottledRequests() { + return numberOfIOPSThrottledRequests; } - public AtomicLong getNumberOfRequestsSucceededInFifteenToTwentyFiveRetry() { - return numberOfRequestsSucceededInFifteenToTwentyFiveRetry; + public AtomicLong getNumberOfBandwidthThrottledRequests() { + return numberOfBandwidthThrottledRequests; } - public AtomicLong getNumberOfRequestsSucceededInTwentyFiveToThirtyRetry() { - return numberOfRequestsSucceededInTwentyFiveToThirtyRetry; + public AtomicLong getNumberOfOtherThrottledRequests() { + return numberOfOtherThrottledRequests; } - public AtomicLong getMinBackoffForFirstRetry() { - return minBackoffForFirstRetry; + public Map getMetricsMap() { + return metricsMap; } @Override public String toString() { - return "#RCTSI_1R_" + numberOfRequestsSucceededInFirstRetry + " #RCTSI_2R_" + numberOfRequestsSucceededInSecondRetry + " #RCTSI_3R_" + - numberOfRequestsSucceededInThirdRetry + " #RCTSI_4R_" + numberOfRequestsSucceededInFourthRetry + " #RCTSI_5-15R_" + numberOfRequestsSucceededInFiveToFifteenRetry + - " #RCTSI_15-25R_" + numberOfRequestsSucceededInFifteenToTwentyFiveRetry + " #RCTSI_25-30R_" + numberOfRequestsSucceededInTwentyFiveToThirtyRetry + - " #Min_1R_ " + minBackoffForFirstRetry; - } + StringBuilder requestsSucceeded = new StringBuilder(); + StringBuilder minMaxAvg = new StringBuilder(); + StringBuilder throttledRequests = new StringBuilder(); + for (Map.Entry entry : metricsMap.entrySet()) { + requestsSucceeded.append("#RCTSI#_").append(entry.getKey()) + .append("R_").append("=") + .append(entry.getValue().getNumberOfRequestsSucceeded()).append(" "); + long totalRequests = entry.getValue().getTotalRequests().get(); + if(totalRequests > 0) { + minMaxAvg.append("MinMaxAvg#_").append(entry.getKey()) + .append("R_").append("=") + .append(String.format("%.5f", (double) entry.getValue().getMinBackoff().get() / 1000L)) + .append(" seconds ") + .append(String.format("%.5f", (double) entry.getValue().getMaxBackoff().get() / 1000L)) + .append(" seconds ") + .append(String.format("%.5f", (double) ((entry.getValue().getTotalBackoff().get() / totalRequests) / 1000L))) + .append(" seconds "); + }else { + minMaxAvg.append("MinMaxAvg#_").append(entry.getKey()) + .append("R_").append("= 0 seconds "); + } + } + throttledRequests.append("BandwidthThrottled = ").append(numberOfBandwidthThrottledRequests) + .append("IOPSThrottled = ").append(numberOfIOPSThrottledRequests) + .append("OtherThrottled = ").append(numberOfOtherThrottledRequests); + + return requestsSucceeded + " " + minMaxAvg + " " + throttledRequests; + } } + diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java index 8e67f36b368d0..345d3d03d642a 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java @@ -688,14 +688,13 @@ public synchronized void close() throws IOException { String metricAccountKey = getConf().get(FS_AZURE_METRIC_ACCOUNT_KEY); final String abfsMetricUrl = "metrics" + "@" + metricAccountName; metricConfig.set(FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME, metricAccountKey); - URI metricUri = null; + URI metricUri; try { metricUri = new URI(getScheme(), abfsMetricUrl, null, null, null); } catch (Exception ex) { throw new AssertionError(ex); } - AzureBlobFileSystem metricfs = (AzureBlobFileSystem) FileSystem.newInstance( - metricUri, metricConfig); + AzureBlobFileSystem metricfs = (AzureBlobFileSystem) FileSystem.newInstance(metricUri, metricConfig); metricfs.sentMetric(metric); } catch (Exception ex) { // do nothing diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java index f510824041bce..63d62a33b1819 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java @@ -41,7 +41,7 @@ public final class FileSystemConfigurations { public static final int DEFAULT_MIN_BACKOFF_INTERVAL = 3 * 1000; // 3s public static final int DEFAULT_MAX_BACKOFF_INTERVAL = 30 * 1000; // 30s public static final int DEFAULT_BACKOFF_INTERVAL = 3 * 1000; // 3s - public static final int DEFAULT_MAX_RETRY_ATTEMPTS = 5; + public static final int DEFAULT_MAX_RETRY_ATTEMPTS = 30; public static final int DEFAULT_CUSTOM_TOKEN_FETCH_RETRY_COUNT = 3; // Retry parameter defaults. diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/services/AzureServiceErrorCode.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/services/AzureServiceErrorCode.java index 8bc31c4f92b2a..5c4b01b12dfac 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/services/AzureServiceErrorCode.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/services/AzureServiceErrorCode.java @@ -44,6 +44,7 @@ public enum AzureServiceErrorCode { INVALID_RENAME_SOURCE_PATH("InvalidRenameSourcePath", HttpURLConnection.HTTP_CONFLICT, null), INGRESS_OVER_ACCOUNT_LIMIT(null, HttpURLConnection.HTTP_UNAVAILABLE, "Ingress is over the account limit."), EGRESS_OVER_ACCOUNT_LIMIT(null, HttpURLConnection.HTTP_UNAVAILABLE, "Egress is over the account limit."), + REQUEST_OVER_ACCOUNT_LIMIT(null, HttpURLConnection.HTTP_UNAVAILABLE, "Operations per second is over the account limit."), INVALID_QUERY_PARAMETER_VALUE("InvalidQueryParameterValue", HttpURLConnection.HTTP_BAD_REQUEST, null), AUTHORIZATION_PERMISSION_MISS_MATCH("AuthorizationPermissionMismatch", HttpURLConnection.HTTP_FORBIDDEN, null), ACCOUNT_REQUIRES_HTTPS("AccountRequiresHttps", HttpURLConnection.HTTP_BAD_REQUEST, null), diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java index 8ff8a281eabdf..ea8dcdfed2d6a 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java @@ -36,8 +36,10 @@ import org.apache.hadoop.fs.azurebfs.constants.HttpHeaderConfigurations; import org.apache.hadoop.fs.azurebfs.utils.TracingContext; import org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding; +import org.apache.hadoop.fs.azurebfs.contracts.services.AzureServiceErrorCode; +import java.util.Map; import org.apache.hadoop.fs.azurebfs.AbfsDriverMetrics; -import java.util.concurrent.atomic.AtomicLong; + /** * The AbfsRestOperation for Rest AbfsClient. */ @@ -73,7 +75,7 @@ public class AbfsRestOperation { private AbfsCounters abfsCounters; private AbfsDriverMetrics abfsDriverMetrics; - + private Map metricsMap; /** * Checks if there is non-null HTTP response. * @return true if there is a non-null HTTP response from the ABFS call. @@ -149,6 +151,7 @@ String getSasToken() { this.sasToken = sasToken; this.abfsCounters = client.getAbfsCounters(); this.abfsDriverMetrics = abfsCounters.getAbfsDriverMetrics(); + this.metricsMap = abfsDriverMetrics.getMetricsMap(); } /** @@ -180,6 +183,7 @@ String getSasToken() { this.bufferLength = bufferLength; this.abfsCounters = client.getAbfsCounters(); this.abfsDriverMetrics = abfsCounters.getAbfsDriverMetrics(); + this.metricsMap = abfsDriverMetrics.getMetricsMap(); } /** @@ -227,56 +231,25 @@ private void completeExecute(TracingContext tracingContext) LOG.debug("Retrying REST operation {}. RetryCount = {}", operationType, retryCount); sleepDuration = client.getRetryPolicy().getRetryInterval(retryCount); + updateTimeMetrics(retryCount, sleepDuration); Thread.sleep(sleepDuration); } catch (InterruptedException ex) { Thread.currentThread().interrupt(); } } if(retryCount > 0) { - if (retryCount == 1) { - long minSleepForFirstRetry = Math.min(abfsDriverMetrics.getMinBackoffForFirstRetry().get(), sleepDuration); - long noOfRequests1Retry - = abfsDriverMetrics.getNumberOfRequestsSucceededInFirstRetry() - .incrementAndGet(); - abfsDriverMetrics.getNumberOfRequestsSucceededInFirstRetry() - .set(noOfRequests1Retry); - abfsDriverMetrics.getMinBackoffForFirstRetry().set(minSleepForFirstRetry); - } else if (retryCount == 2) { - long noOfRequests2Retry - = abfsDriverMetrics.getNumberOfRequestsSucceededInSecondRetry() - .incrementAndGet(); - abfsDriverMetrics.getNumberOfRequestsSucceededInSecondRetry() - .set(noOfRequests2Retry); - } else if (retryCount == 3) { - long noOfRequests3Retry - = abfsDriverMetrics.getNumberOfRequestsSucceededInThirdRetry() - .incrementAndGet(); - abfsDriverMetrics.getNumberOfRequestsSucceededInThirdRetry() - .set(noOfRequests3Retry); - } else if (retryCount == 4) { - long noOfRequests4Retry - = abfsDriverMetrics.getNumberOfRequestsSucceededInFourthRetry() - .incrementAndGet(); - abfsDriverMetrics.getNumberOfRequestsSucceededInFourthRetry() - .set(noOfRequests4Retry); - } else if (retryCount >= 5 && retryCount < 15) { - long noOfRequests5_15Retry - = abfsDriverMetrics.getNumberOfRequestsSucceededInFiveToFifteenRetry() - .incrementAndGet(); - abfsDriverMetrics.getNumberOfRequestsSucceededInFiveToFifteenRetry() - .set(noOfRequests5_15Retry); - } else if (retryCount >= 15 && retryCount < 25) { - long noOfRequests15_25Retry - = abfsDriverMetrics.getNumberOfRequestsSucceededInFifteenToTwentyFiveRetry() - .incrementAndGet(); - abfsDriverMetrics.getNumberOfRequestsSucceededInFifteenToTwentyFiveRetry() - .set(noOfRequests15_25Retry); - } else { - long noOfRequests25_30Retry - = abfsDriverMetrics.getNumberOfRequestsSucceededInTwentyFiveToThirtyRetry() - .incrementAndGet(); - abfsDriverMetrics.getNumberOfRequestsSucceededInTwentyFiveToThirtyRetry() - .set(noOfRequests25_30Retry); + updateCount(retryCount); + } + if(result.getStatusCode() == HttpURLConnection.HTTP_UNAVAILABLE){ + AzureServiceErrorCode serviceErrorCode = + AzureServiceErrorCode.getAzureServiceCode(result.getStatusCode(), result.getStorageErrorCode(), result.getStorageErrorMessage()); + if(serviceErrorCode.equals(AzureServiceErrorCode.INGRESS_OVER_ACCOUNT_LIMIT) || + serviceErrorCode.equals(AzureServiceErrorCode.EGRESS_OVER_ACCOUNT_LIMIT)){ + abfsDriverMetrics.getNumberOfBandwidthThrottledRequests().getAndIncrement(); + }else if(serviceErrorCode.equals(AzureServiceErrorCode.REQUEST_OVER_ACCOUNT_LIMIT)){ + abfsDriverMetrics.getNumberOfIOPSThrottledRequests().getAndIncrement(); + }else{ + abfsDriverMetrics.getNumberOfOtherThrottledRequests().getAndIncrement(); } } if (result.getStatusCode() >= HttpURLConnection.HTTP_BAD_REQUEST) { @@ -387,7 +360,7 @@ private boolean executeHttpOperation(final int retryCount, /** * Incrementing Abfs counters with a long value. * - * @param statistic the Abfs statistic that needs to be incremented. + * @param statistic the Abfs statistic that needs to be incremented.f * @param value the value to be incremented by. */ private void incrementCounter(AbfsStatistic statistic, long value) { @@ -395,4 +368,35 @@ private void incrementCounter(AbfsStatistic statistic, long value) { abfsCounters.incrementCounter(statistic, value); } } + + private void updateCount(int retryCount){ + String retryCounter = getKey(retryCount); + metricsMap.get(retryCounter).getNumberOfRequestsSucceeded().getAndIncrement(); + } + + private void updateTimeMetrics(int retryCount, long sleepDuration){ + String retryCounter = getKey(retryCount); + long minBackoffTime = Math.min(metricsMap.get(retryCounter).getMinBackoff().get(), sleepDuration); + long maxBackoffForTime = Math.max(metricsMap.get(retryCounter).getMaxBackoff().get(), sleepDuration); + long totalBackoffTime = metricsMap.get(retryCounter).getTotalBackoff().get() + sleepDuration; + long totalRequests = metricsMap.get(retryCounter).getTotalRequests().incrementAndGet(); + metricsMap.get(retryCounter).getMinBackoff().set(minBackoffTime); + metricsMap.get(retryCounter).getMaxBackoff().set(maxBackoffForTime); + metricsMap.get(retryCounter).getTotalBackoff().set(totalBackoffTime); + metricsMap.get(retryCounter).getTotalRequests().set(totalRequests); + } + + private String getKey(int retryCount) { + String retryCounter; + if(retryCount >= 1 && retryCount <= 4){ + retryCounter = Integer.toString(retryCount); + }else if(retryCount >= 5 && retryCount < 15){ + retryCounter = "5_15"; + }else if(retryCount >= 15 && retryCount < 25){ + retryCounter = "15_25"; + }else{ + retryCounter = "25_30"; + } + return retryCounter; + } } diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsReadWriteAndSeek.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsReadWriteAndSeek.java index 5bd6eaff42e84..ff60306dac9b5 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsReadWriteAndSeek.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsReadWriteAndSeek.java @@ -50,12 +50,17 @@ public class ITestAbfsReadWriteAndSeek extends AbstractAbfsScaleTest { private static final String TEST_PATH = "/testfile"; +// @Parameterized.Parameters(name = "Size={0}") +// public static Iterable sizes() { +// return Arrays.asList(new Object[][]{{MIN_BUFFER_SIZE}, +// {DEFAULT_READ_BUFFER_SIZE}, +// {APPENDBLOB_MAX_WRITE_BUFFER_SIZE}, +// {MAX_BUFFER_SIZE}}); +// } + @Parameterized.Parameters(name = "Size={0}") public static Iterable sizes() { - return Arrays.asList(new Object[][]{{MIN_BUFFER_SIZE}, - {DEFAULT_READ_BUFFER_SIZE}, - {APPENDBLOB_MAX_WRITE_BUFFER_SIZE}, - {MAX_BUFFER_SIZE}}); + return Arrays.asList(new Object[][]{{MIN_BUFFER_SIZE}}); } private final int size; From 23d9b472d45a21ebc8ab7060f77353fc969d243f Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Mon, 6 Jun 2022 17:04:51 +0530 Subject: [PATCH 05/42] Added final metrics --- .../hadoop/fs/azurebfs/AbfsDriverMetrics.java | 38 ++++++++++----- .../constants/FileSystemConfigurations.java | 2 +- .../services/AzureServiceErrorCode.java | 33 +++++++------ .../azurebfs/services/AbfsRestOperation.java | 48 +++++++++++-------- .../src/test/resources/log4j.properties | 2 + 5 files changed, 76 insertions(+), 47 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java index ee38e151016b0..0de1895c2b224 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java @@ -14,9 +14,9 @@ public class AbfsDriverMetrics { private String retryCount; private AtomicLong numberOfIOPSThrottledRequests; private AtomicLong numberOfBandwidthThrottledRequests; - private AtomicLong numberOfOtherThrottledRequests; - + private AtomicLong maxRetryCount; + private AtomicLong totalNumberOfRequests; private static final Map metricsMap = new LinkedHashMap<>(); public AbfsDriverMetrics() { @@ -24,6 +24,8 @@ public AbfsDriverMetrics() { this.numberOfIOPSThrottledRequests = new AtomicLong(); this.numberOfBandwidthThrottledRequests = new AtomicLong(); this.numberOfOtherThrottledRequests = new AtomicLong(); + this.totalNumberOfRequests = new AtomicLong(); + this.maxRetryCount = new AtomicLong(); } public AbfsDriverMetrics(String retryCount) { this.retryCount = retryCount; @@ -77,22 +79,30 @@ public AtomicLong getNumberOfOtherThrottledRequests() { return numberOfOtherThrottledRequests; } + public AtomicLong getMaxRetryCount() { + return maxRetryCount; + } + + public AtomicLong getTotalNumberOfRequests() { + return totalNumberOfRequests; + } + public Map getMetricsMap() { return metricsMap; } @Override public String toString() { - StringBuilder requestsSucceeded = new StringBuilder(); - StringBuilder minMaxAvg = new StringBuilder(); - StringBuilder throttledRequests = new StringBuilder(); + StringBuilder metricString = new StringBuilder(); + long totalRequestsThrottled = numberOfBandwidthThrottledRequests.get() + numberOfIOPSThrottledRequests.get() + numberOfOtherThrottledRequests.get(); + double percentageOfRequestsThrottled = ((double)totalRequestsThrottled/totalNumberOfRequests.get())*100; for (Map.Entry entry : metricsMap.entrySet()) { - requestsSucceeded.append("#RCTSI#_").append(entry.getKey()) + metricString.append(" #RCTSI#_").append(entry.getKey()) .append("R_").append("=") .append(entry.getValue().getNumberOfRequestsSucceeded()).append(" "); long totalRequests = entry.getValue().getTotalRequests().get(); if(totalRequests > 0) { - minMaxAvg.append("MinMaxAvg#_").append(entry.getKey()) + metricString.append(" MinMaxAvg#_").append(entry.getKey()) .append("R_").append("=") .append(String.format("%.5f", (double) entry.getValue().getMinBackoff().get() / 1000L)) .append(" seconds ") @@ -101,15 +111,19 @@ public String toString() { .append(String.format("%.5f", (double) ((entry.getValue().getTotalBackoff().get() / totalRequests) / 1000L))) .append(" seconds "); }else { - minMaxAvg.append("MinMaxAvg#_").append(entry.getKey()) + metricString.append("MinMaxAvg#_").append(entry.getKey()) .append("R_").append("= 0 seconds "); } } - throttledRequests.append("BandwidthThrottled = ").append(numberOfBandwidthThrottledRequests) - .append("IOPSThrottled = ").append(numberOfIOPSThrottledRequests) - .append("OtherThrottled = ").append(numberOfOtherThrottledRequests); + metricString.append(" BandwidthThrottled = ").append(numberOfBandwidthThrottledRequests) + .append(" IOPSThrottled = ").append(numberOfIOPSThrottledRequests) + .append(" OtherThrottled = ").append(numberOfOtherThrottledRequests) + .append(" Total number of requests = ").append(totalNumberOfRequests); + + metricString.append(" Max retry count is ").append(maxRetryCount); + metricString.append(" Percentage of throttled requests = ").append(percentageOfRequestsThrottled); - return requestsSucceeded + " " + minMaxAvg + " " + throttledRequests; + return metricString + " "; } } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java index 63d62a33b1819..49d0a1fd18917 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java @@ -38,7 +38,7 @@ public final class FileSystemConfigurations { private static final int SIXTY_SECONDS = 60 * 1000; // Retry parameter defaults. - public static final int DEFAULT_MIN_BACKOFF_INTERVAL = 3 * 1000; // 3s + public static final int DEFAULT_MIN_BACKOFF_INTERVAL = 25 * 10; // 3s public static final int DEFAULT_MAX_BACKOFF_INTERVAL = 30 * 1000; // 30s public static final int DEFAULT_BACKOFF_INTERVAL = 3 * 1000; // 3s public static final int DEFAULT_MAX_RETRY_ATTEMPTS = 30; diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/services/AzureServiceErrorCode.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/services/AzureServiceErrorCode.java index 5c4b01b12dfac..ce53b1fd1bfe1 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/services/AzureServiceErrorCode.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/services/AzureServiceErrorCode.java @@ -21,7 +21,8 @@ import java.net.HttpURLConnection; import java.util.ArrayList; import java.util.List; - +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; @@ -42,9 +43,9 @@ public enum AzureServiceErrorCode { INVALID_SOURCE_OR_DESTINATION_RESOURCE_TYPE("InvalidSourceOrDestinationResourceType", HttpURLConnection.HTTP_CONFLICT, null), RENAME_DESTINATION_PARENT_PATH_NOT_FOUND("RenameDestinationParentPathNotFound", HttpURLConnection.HTTP_NOT_FOUND, null), INVALID_RENAME_SOURCE_PATH("InvalidRenameSourcePath", HttpURLConnection.HTTP_CONFLICT, null), - INGRESS_OVER_ACCOUNT_LIMIT(null, HttpURLConnection.HTTP_UNAVAILABLE, "Ingress is over the account limit."), - EGRESS_OVER_ACCOUNT_LIMIT(null, HttpURLConnection.HTTP_UNAVAILABLE, "Egress is over the account limit."), - REQUEST_OVER_ACCOUNT_LIMIT(null, HttpURLConnection.HTTP_UNAVAILABLE, "Operations per second is over the account limit."), + INGRESS_OVER_ACCOUNT_LIMIT("ServerBusy", HttpURLConnection.HTTP_UNAVAILABLE, "Ingress is over the account limit."), + EGRESS_OVER_ACCOUNT_LIMIT("ServerBusy", HttpURLConnection.HTTP_UNAVAILABLE, "Egress is over the account limit."), + REQUEST_OVER_ACCOUNT_LIMIT("ServerBusy", HttpURLConnection.HTTP_UNAVAILABLE, "Operations per second is over the account limit."), INVALID_QUERY_PARAMETER_VALUE("InvalidQueryParameterValue", HttpURLConnection.HTTP_BAD_REQUEST, null), AUTHORIZATION_PERMISSION_MISS_MATCH("AuthorizationPermissionMismatch", HttpURLConnection.HTTP_FORBIDDEN, null), ACCOUNT_REQUIRES_HTTPS("AccountRequiresHttps", HttpURLConnection.HTTP_BAD_REQUEST, null), @@ -53,6 +54,9 @@ public enum AzureServiceErrorCode { private final String errorCode; private final int httpStatusCode; private final String errorMessage; + + private static final Logger LOG1 = LoggerFactory.getLogger(AzureServiceErrorCode.class); + AzureServiceErrorCode(String errorCode, int httpStatusCodes, String errorMessage) { this.errorCode = errorCode; this.httpStatusCode = httpStatusCodes; @@ -67,6 +71,10 @@ public String getErrorCode() { return this.errorCode; } + public String getErrorMessage() { + return this.errorMessage; + } + public static List getAzureServiceCode(int httpStatusCode) { List errorCodes = new ArrayList<>(); if (httpStatusCode == UNKNOWN.httpStatusCode) { @@ -94,24 +102,21 @@ public static AzureServiceErrorCode getAzureServiceCode(int httpStatusCode, Stri return azureServiceErrorCode; } } - return UNKNOWN; } - public static AzureServiceErrorCode getAzureServiceCode(int httpStatusCode, String errorCode, final String errorMessage) { + public static AzureServiceErrorCode getAzureServiceCode(int httpStatusCode, String errorCode, String errorMessage) { if (errorCode == null || errorCode.isEmpty() || httpStatusCode == UNKNOWN.httpStatusCode || errorMessage == null || errorMessage.isEmpty()) { return UNKNOWN; } - + String[] errorMessages = errorMessage.split(System.lineSeparator(), 2); for (AzureServiceErrorCode azureServiceErrorCode : AzureServiceErrorCode.values()) { - if (azureServiceErrorCode.httpStatusCode == httpStatusCode - && errorCode.equalsIgnoreCase(azureServiceErrorCode.errorCode) - && errorMessage.equalsIgnoreCase(azureServiceErrorCode.errorMessage) - ) { - return azureServiceErrorCode; + if (azureServiceErrorCode.getStatusCode() == httpStatusCode + && azureServiceErrorCode.getErrorCode().equalsIgnoreCase(errorCode) && + azureServiceErrorCode.getErrorMessage().equalsIgnoreCase(errorMessages[0])) { + return azureServiceErrorCode; + } } - } - return UNKNOWN; } } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java index ea8dcdfed2d6a..af89a9595db7f 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java @@ -63,19 +63,19 @@ public class AbfsRestOperation { private final String sasToken; private static final Logger LOG = LoggerFactory.getLogger(AbfsClient.class); - + private static final Logger LOG1 = LoggerFactory.getLogger(AbfsRestOperation.class); // For uploads, this is the request entity body. For downloads, // this will hold the response entity body. private byte[] buffer; private int bufferOffset; private int bufferLength; private int retryCount = 0; - + private boolean isThrottledRequest = false; + private long maxRetryCount = 0L; private AbfsHttpOperation result; private AbfsCounters abfsCounters; - private AbfsDriverMetrics abfsDriverMetrics; - private Map metricsMap; + private static Map metricsMap; /** * Checks if there is non-null HTTP response. * @return true if there is a non-null HTTP response from the ABFS call. @@ -123,6 +123,8 @@ String getSasToken() { final URL url, final List requestHeaders) { this(operationType, client, method, url, requestHeaders, null); + this.abfsDriverMetrics = abfsCounters.getAbfsDriverMetrics(); + metricsMap = abfsDriverMetrics.getMetricsMap(); } /** @@ -151,7 +153,7 @@ String getSasToken() { this.sasToken = sasToken; this.abfsCounters = client.getAbfsCounters(); this.abfsDriverMetrics = abfsCounters.getAbfsDriverMetrics(); - this.metricsMap = abfsDriverMetrics.getMetricsMap(); + metricsMap = abfsDriverMetrics.getMetricsMap(); } /** @@ -183,7 +185,7 @@ String getSasToken() { this.bufferLength = bufferLength; this.abfsCounters = client.getAbfsCounters(); this.abfsDriverMetrics = abfsCounters.getAbfsDriverMetrics(); - this.metricsMap = abfsDriverMetrics.getMetricsMap(); + metricsMap = abfsDriverMetrics.getMetricsMap(); } /** @@ -237,20 +239,11 @@ private void completeExecute(TracingContext tracingContext) Thread.currentThread().interrupt(); } } + abfsDriverMetrics.getTotalNumberOfRequests().getAndIncrement(); if(retryCount > 0) { - updateCount(retryCount); - } - if(result.getStatusCode() == HttpURLConnection.HTTP_UNAVAILABLE){ - AzureServiceErrorCode serviceErrorCode = - AzureServiceErrorCode.getAzureServiceCode(result.getStatusCode(), result.getStorageErrorCode(), result.getStorageErrorMessage()); - if(serviceErrorCode.equals(AzureServiceErrorCode.INGRESS_OVER_ACCOUNT_LIMIT) || - serviceErrorCode.equals(AzureServiceErrorCode.EGRESS_OVER_ACCOUNT_LIMIT)){ - abfsDriverMetrics.getNumberOfBandwidthThrottledRequests().getAndIncrement(); - }else if(serviceErrorCode.equals(AzureServiceErrorCode.REQUEST_OVER_ACCOUNT_LIMIT)){ - abfsDriverMetrics.getNumberOfIOPSThrottledRequests().getAndIncrement(); - }else{ - abfsDriverMetrics.getNumberOfOtherThrottledRequests().getAndIncrement(); - } + maxRetryCount = Math.max(abfsDriverMetrics.getMaxRetryCount().get(), retryCount); + abfsDriverMetrics.getMaxRetryCount().set(maxRetryCount); + updateCount(retryCount); } if (result.getStatusCode() >= HttpURLConnection.HTTP_BAD_REQUEST) { throw new AbfsRestOperationException(result.getStatusCode(), result.getStorageErrorCode(), @@ -314,7 +307,22 @@ private boolean executeHttpOperation(final int retryCount, } httpOperation.processResponse(buffer, bufferOffset, bufferLength); - incrementCounter(AbfsStatistic.GET_RESPONSES, 1); + if(!isThrottledRequest && httpOperation.getStatusCode() == HttpURLConnection.HTTP_UNAVAILABLE){ + isThrottledRequest = true; + AzureServiceErrorCode serviceErrorCode = + AzureServiceErrorCode.getAzureServiceCode(httpOperation.getStatusCode(), httpOperation.getStorageErrorCode(), httpOperation.getStorageErrorMessage()); + LOG1.trace("Service code is " + serviceErrorCode + " status code is " + httpOperation.getStatusCode() + " error code is " + httpOperation.getStorageErrorCode() + + " error message is " + httpOperation.getStorageErrorMessage()); + if(serviceErrorCode.equals(AzureServiceErrorCode.INGRESS_OVER_ACCOUNT_LIMIT) || + serviceErrorCode.equals(AzureServiceErrorCode.EGRESS_OVER_ACCOUNT_LIMIT)){ + abfsDriverMetrics.getNumberOfBandwidthThrottledRequests().getAndIncrement(); + }else if(serviceErrorCode.equals(AzureServiceErrorCode.REQUEST_OVER_ACCOUNT_LIMIT)){ + abfsDriverMetrics.getNumberOfIOPSThrottledRequests().getAndIncrement(); + }else{ + abfsDriverMetrics.getNumberOfOtherThrottledRequests().getAndIncrement(); + } + } + incrementCounter(AbfsStatistic.GET_RESPONSES, 1); //Only increment bytesReceived counter when the status code is 2XX. if (httpOperation.getStatusCode() >= HttpURLConnection.HTTP_OK && httpOperation.getStatusCode() <= HttpURLConnection.HTTP_PARTIAL) { diff --git a/hadoop-tools/hadoop-azure/src/test/resources/log4j.properties b/hadoop-tools/hadoop-azure/src/test/resources/log4j.properties index 9f72d03653306..c279febef8851 100644 --- a/hadoop-tools/hadoop-azure/src/test/resources/log4j.properties +++ b/hadoop-tools/hadoop-azure/src/test/resources/log4j.properties @@ -26,6 +26,8 @@ log4j.logger.org.apache.hadoop.fs.azure.AzureFileSystemThreadPoolExecutor=DEBUG log4j.logger.org.apache.hadoop.fs.azure.BlockBlobAppendStream=DEBUG log4j.logger.org.apache.hadoop.fs.azurebfs.contracts.services.TracingService=TRACE log4j.logger.org.apache.hadoop.fs.azurebfs.services.AbfsClient=DEBUG +log4j.logger.org.apache.hadoop.fs.azurebfs.services.AbfsRestOperation=TRACE +log4j.logger.org.apache.hadoop.fs.azurebfs.contracts.services.AzureServiceErrorCode=TRACE # after here: turn off log messages from other parts of the system # which only clutter test reports. From d57a746e2e14c4121a4841182e4ceee3c200e0d0 Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Tue, 7 Jun 2022 15:38:57 +0530 Subject: [PATCH 06/42] Checkstyle fixes --- .../hadoop/fs/azurebfs/AbfsDriverMetrics.java | 20 ++++++++++++++++++- .../fs/azurebfs/AzureBlobFileSystem.java | 4 ++-- .../fs/azurebfs/AzureBlobFileSystemStore.java | 2 +- .../azurebfs/services/AbfsRestOperation.java | 5 +---- 4 files changed, 23 insertions(+), 8 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java index 0de1895c2b224..f1986ab2a88bb 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java @@ -1,3 +1,21 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.hadoop.fs.azurebfs; import java.util.concurrent.atomic.AtomicLong; @@ -87,7 +105,7 @@ public AtomicLong getTotalNumberOfRequests() { return totalNumberOfRequests; } - public Map getMetricsMap() { + public static Map getMetricsMap() { return metricsMap; } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java index 345d3d03d642a..32612c1a9c624 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java @@ -691,12 +691,12 @@ public synchronized void close() throws IOException { URI metricUri; try { metricUri = new URI(getScheme(), abfsMetricUrl, null, null, null); - } catch (Exception ex) { + } catch (URISyntaxException ex) { throw new AssertionError(ex); } AzureBlobFileSystem metricfs = (AzureBlobFileSystem) FileSystem.newInstance(metricUri, metricConfig); metricfs.sentMetric(metric); - } catch (Exception ex) { + } catch (AzureBlobFileSystemException ex) { // do nothing } // does all the delete-on-exit calls, and may be slow. diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java index 681f991facbe2..17a2ffbabf5e2 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java @@ -1975,6 +1975,6 @@ public static String extractEtagHeader(AbfsHttpOperation result) { } public void sentMetric(String metric, TracingContext tracingContext) throws AzureBlobFileSystemException{ - AbfsRestOperation op = client.getPathStatusMetric("/..$$@@", tracingContext, metric); // Will sent a GFS calls that will fail to register in MDM x-ms-client-metric + client.getPathStatusMetric("/..$$@@", tracingContext, metric); // Will sent a GFS calls that will fail to register in MDM x-ms-client-metric } } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java index af89a9595db7f..0cb11ae922451 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java @@ -75,7 +75,7 @@ public class AbfsRestOperation { private AbfsHttpOperation result; private AbfsCounters abfsCounters; private AbfsDriverMetrics abfsDriverMetrics; - private static Map metricsMap; + private static final Map metricsMap = AbfsDriverMetrics.getMetricsMap(); /** * Checks if there is non-null HTTP response. * @return true if there is a non-null HTTP response from the ABFS call. @@ -124,7 +124,6 @@ String getSasToken() { final List requestHeaders) { this(operationType, client, method, url, requestHeaders, null); this.abfsDriverMetrics = abfsCounters.getAbfsDriverMetrics(); - metricsMap = abfsDriverMetrics.getMetricsMap(); } /** @@ -153,7 +152,6 @@ String getSasToken() { this.sasToken = sasToken; this.abfsCounters = client.getAbfsCounters(); this.abfsDriverMetrics = abfsCounters.getAbfsDriverMetrics(); - metricsMap = abfsDriverMetrics.getMetricsMap(); } /** @@ -185,7 +183,6 @@ String getSasToken() { this.bufferLength = bufferLength; this.abfsCounters = client.getAbfsCounters(); this.abfsDriverMetrics = abfsCounters.getAbfsDriverMetrics(); - metricsMap = abfsDriverMetrics.getMetricsMap(); } /** From 4476d33edf3efa40d5ca88e4557dc3a091bd0c1c Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Tue, 14 Jun 2022 09:04:37 +0530 Subject: [PATCH 07/42] Added new params to metrics --- .../hadoop/fs/azurebfs/AbfsDriverMetrics.java | 29 ++++++++++---- .../fs/azurebfs/AzureBlobFileSystem.java | 40 ++++++++++--------- .../fs/azurebfs/services/AbfsClient.java | 1 - .../azurebfs/services/AbfsRestOperation.java | 11 ++++- .../fs/azurebfs/utils/TracingContext.java | 1 - 5 files changed, 52 insertions(+), 30 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java index f1986ab2a88bb..cfeda84b2355a 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java @@ -20,7 +20,7 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.Map; -import java.util.LinkedHashMap; +import java.util.concurrent.ConcurrentHashMap; import java.util.ArrayList; import java.util.Arrays; public class AbfsDriverMetrics { @@ -35,7 +35,9 @@ public class AbfsDriverMetrics { private AtomicLong numberOfOtherThrottledRequests; private AtomicLong maxRetryCount; private AtomicLong totalNumberOfRequests; - private static final Map metricsMap = new LinkedHashMap<>(); + private AtomicLong numberOfRequestsSucceededWithoutRetrying; + private AtomicLong numberOfRequestsFailed; + private final Map metricsMap = new ConcurrentHashMap<>(); public AbfsDriverMetrics() { initializeMap(); @@ -44,6 +46,8 @@ public AbfsDriverMetrics() { this.numberOfOtherThrottledRequests = new AtomicLong(); this.totalNumberOfRequests = new AtomicLong(); this.maxRetryCount = new AtomicLong(); + this.numberOfRequestsSucceededWithoutRetrying = new AtomicLong(); + this.numberOfRequestsFailed = new AtomicLong(); } public AbfsDriverMetrics(String retryCount) { this.retryCount = retryCount; @@ -105,10 +109,18 @@ public AtomicLong getTotalNumberOfRequests() { return totalNumberOfRequests; } - public static Map getMetricsMap() { + public Map getMetricsMap() { return metricsMap; } + public AtomicLong getNumberOfRequestsSucceededWithoutRetrying() { + return numberOfRequestsSucceededWithoutRetrying; + } + + public AtomicLong getNumberOfRequestsFailed() { + return numberOfRequestsFailed; + } + @Override public String toString() { StringBuilder metricString = new StringBuilder(); @@ -135,11 +147,12 @@ public String toString() { } metricString.append(" BandwidthThrottled = ").append(numberOfBandwidthThrottledRequests) .append(" IOPSThrottled = ").append(numberOfIOPSThrottledRequests) - .append(" OtherThrottled = ").append(numberOfOtherThrottledRequests) - .append(" Total number of requests = ").append(totalNumberOfRequests); - - metricString.append(" Max retry count is ").append(maxRetryCount); - metricString.append(" Percentage of throttled requests = ").append(percentageOfRequestsThrottled); + .append(" OtherThrottled = ").append(numberOfOtherThrottledRequests).append('\n'); + metricString.append(" Percentage of throttled requests = ").append(percentageOfRequestsThrottled).append('\n'); + metricString.append(" Total number of requests which succeeded without retrying = ").append(numberOfRequestsSucceededWithoutRetrying).append('\n'); + metricString.append(" Total number of requests which failed = ").append(numberOfRequestsFailed).append('\n'); + metricString.append(" Total number of requests = ").append(totalNumberOfRequests).append('\n'); + metricString.append(" Max retry count = ").append(maxRetryCount).append('\n'); return metricString + " "; } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java index 32612c1a9c624..645bd11db354c 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java @@ -681,23 +681,27 @@ public synchronized void close() throws IOException { if (isClosed) { return; } - try { - String metric = abfsCounters.getAbfsDriverMetrics().toString(); - Configuration metricConfig = getConf(); - String metricAccountName = getConf().get(FS_AZURE_METRIC_ACCOUNT_NAME); - String metricAccountKey = getConf().get(FS_AZURE_METRIC_ACCOUNT_KEY); - final String abfsMetricUrl = "metrics" + "@" + metricAccountName; - metricConfig.set(FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME, metricAccountKey); - URI metricUri; - try { - metricUri = new URI(getScheme(), abfsMetricUrl, null, null, null); - } catch (URISyntaxException ex) { - throw new AssertionError(ex); - } - AzureBlobFileSystem metricfs = (AzureBlobFileSystem) FileSystem.newInstance(metricUri, metricConfig); - metricfs.sentMetric(metric); - } catch (AzureBlobFileSystemException ex) { - // do nothing + if(abfsCounters.getAbfsDriverMetrics().getTotalNumberOfRequests().get() > 0) { + try { + String metric = abfsCounters.getAbfsDriverMetrics().toString(); + Configuration metricConfig = getConf(); + String metricAccountName = getConf().get(FS_AZURE_METRIC_ACCOUNT_NAME); + String metricAccountKey = getConf().get(FS_AZURE_METRIC_ACCOUNT_KEY); + final String abfsMetricUrl = "metrics" + "@" + metricAccountName; + metricConfig.set(FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME, metricAccountKey); + URI metricUri; + try { + metricUri = new URI(getScheme(), abfsMetricUrl, null, null, null); + } catch (URISyntaxException ex) { + throw new AssertionError(ex); + } + AzureBlobFileSystem metricfs + = (AzureBlobFileSystem) FileSystem.newInstance(metricUri, + metricConfig); + metricfs.sentMetric(metric); + }catch (AzureBlobFileSystemException ex) { + // do nothing + } } // does all the delete-on-exit calls, and may be slow. super.close(); @@ -717,7 +721,7 @@ public synchronized void close() throws IOException { public void sentMetric(String metric) throws AzureBlobFileSystemException { TracingContext tracingContext = new TracingContext(clientCorrelationId, - fileSystemId, FSOperationType.SET_ATTR, true, tracingHeaderFormat, + fileSystemId, FSOperationType.GET_ATTR, true, tracingHeaderFormat, listener); abfsStore.sentMetric(metric, tracingContext); } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java index aaa6b384486af..d6a09c2fdf582 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java @@ -760,7 +760,6 @@ public AbfsRestOperation getPathStatusMetric(final String path, operation = SASTokenProvider.GET_STATUS_OPERATION; abfsUriQueryBuilder.addQuery(HttpQueryParams.QUERY_PARAM_UPN, String.valueOf(abfsConfiguration.isUpnUsed())); appendSASTokenToQuery(path, operation, abfsUriQueryBuilder); - final URL url = createRequestUrl(path, abfsUriQueryBuilder.toString()); final AbfsRestOperation op = new AbfsRestOperation( AbfsRestOperationType.GetPathStatus, diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java index 0cb11ae922451..ad6c9e540942c 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java @@ -75,7 +75,7 @@ public class AbfsRestOperation { private AbfsHttpOperation result; private AbfsCounters abfsCounters; private AbfsDriverMetrics abfsDriverMetrics; - private static final Map metricsMap = AbfsDriverMetrics.getMetricsMap(); + private Map metricsMap; /** * Checks if there is non-null HTTP response. * @return true if there is a non-null HTTP response from the ABFS call. @@ -124,6 +124,7 @@ String getSasToken() { final List requestHeaders) { this(operationType, client, method, url, requestHeaders, null); this.abfsDriverMetrics = abfsCounters.getAbfsDriverMetrics(); + this.metricsMap = abfsDriverMetrics.getMetricsMap(); } /** @@ -152,6 +153,7 @@ String getSasToken() { this.sasToken = sasToken; this.abfsCounters = client.getAbfsCounters(); this.abfsDriverMetrics = abfsCounters.getAbfsDriverMetrics(); + this.metricsMap = abfsDriverMetrics.getMetricsMap(); } /** @@ -183,6 +185,7 @@ String getSasToken() { this.bufferLength = bufferLength; this.abfsCounters = client.getAbfsCounters(); this.abfsDriverMetrics = abfsCounters.getAbfsDriverMetrics(); + this.metricsMap = abfsDriverMetrics.getMetricsMap(); } /** @@ -237,10 +240,14 @@ private void completeExecute(TracingContext tracingContext) } } abfsDriverMetrics.getTotalNumberOfRequests().getAndIncrement(); - if(retryCount > 0) { + if(retryCount > 0 && retryCount <= 30) { maxRetryCount = Math.max(abfsDriverMetrics.getMaxRetryCount().get(), retryCount); abfsDriverMetrics.getMaxRetryCount().set(maxRetryCount); updateCount(retryCount); + } else if(retryCount > 30){ + abfsDriverMetrics.getNumberOfRequestsFailed().getAndIncrement(); + }else{ + abfsDriverMetrics.getNumberOfRequestsSucceededWithoutRetrying().getAndIncrement(); } if (result.getStatusCode() >= HttpURLConnection.HTTP_BAD_REQUEST) { throw new AbfsRestOperationException(result.getStatusCode(), result.getStorageErrorCode(), diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/TracingContext.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/TracingContext.java index 05fe33b7a19da..85580919e910c 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/TracingContext.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/TracingContext.java @@ -113,7 +113,6 @@ public TracingContext(TracingContext originalTracingContext) { this.listener = originalTracingContext.listener.getClone(); } } - public static String validateClientCorrelationID(String clientCorrelationID) { if ((clientCorrelationID.length() > MAX_CLIENT_CORRELATION_ID_LENGTH) || (!clientCorrelationID.matches(CLIENT_CORRELATION_ID_PATTERN))) { From d7b6be8302d39ba161b6a9ed912367a0cc115cb8 Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Tue, 21 Jun 2022 16:32:21 +0530 Subject: [PATCH 08/42] Incorrect total requests fix --- .../hadoop/fs/azurebfs/AbfsDriverMetrics.java | 2 +- .../azurebfs/services/AbfsRestOperation.java | 30 ++++++++++++------- 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java index cfeda84b2355a..c3355e73da80b 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java @@ -18,9 +18,9 @@ package org.apache.hadoop.fs.azurebfs; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicLong; import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; import java.util.ArrayList; import java.util.Arrays; public class AbfsDriverMetrics { diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java index ad6c9e540942c..049034fdb5ab9 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java @@ -226,6 +226,7 @@ private void completeExecute(TracingContext tracingContext) retryCount = 0; LOG.debug("First execution of REST operation - {}", operationType); long sleepDuration = 0L; + abfsDriverMetrics.getTotalNumberOfRequests().getAndIncrement(); while (!executeHttpOperation(retryCount, tracingContext)) { try { ++retryCount; @@ -239,16 +240,7 @@ private void completeExecute(TracingContext tracingContext) Thread.currentThread().interrupt(); } } - abfsDriverMetrics.getTotalNumberOfRequests().getAndIncrement(); - if(retryCount > 0 && retryCount <= 30) { - maxRetryCount = Math.max(abfsDriverMetrics.getMaxRetryCount().get(), retryCount); - abfsDriverMetrics.getMaxRetryCount().set(maxRetryCount); - updateCount(retryCount); - } else if(retryCount > 30){ - abfsDriverMetrics.getNumberOfRequestsFailed().getAndIncrement(); - }else{ - abfsDriverMetrics.getNumberOfRequestsSucceededWithoutRetrying().getAndIncrement(); - } + updateDriverMetrics(retryCount, result.getStatusCode()); if (result.getStatusCode() >= HttpURLConnection.HTTP_BAD_REQUEST) { throw new AbfsRestOperationException(result.getStatusCode(), result.getStorageErrorCode(), result.getStorageErrorMessage(), null, result); @@ -257,6 +249,22 @@ private void completeExecute(TracingContext tracingContext) LOG.trace("{} REST operation complete", operationType); } + private void updateDriverMetrics(int retryCount, int statusCode){ + if (statusCode == HttpURLConnection.HTTP_UNAVAILABLE) { + if (retryCount == 30) { + abfsDriverMetrics.getNumberOfRequestsFailed().getAndIncrement(); + } + } else { + if (retryCount > 0 && retryCount <= 30) { + maxRetryCount = Math.max(abfsDriverMetrics.getMaxRetryCount().get(), retryCount); + abfsDriverMetrics.getMaxRetryCount().set(maxRetryCount); + updateCount(retryCount); + } else { + abfsDriverMetrics.getNumberOfRequestsSucceededWithoutRetrying().getAndIncrement(); + } + } + } + /** * Executes a single HTTP operation to complete the REST operation. If it * fails, there may be a retry. The retryCount is incremented with each @@ -341,6 +349,7 @@ private boolean executeHttpOperation(final int retryCount, LOG.warn("Unknown host name: {}. Retrying to resolve the host name...", hostname); if (!client.getRetryPolicy().shouldRetry(retryCount, -1)) { + updateDriverMetrics(retryCount, httpOperation.getStatusCode()); throw new InvalidAbfsRestOperationException(ex); } return false; @@ -350,6 +359,7 @@ private boolean executeHttpOperation(final int retryCount, } if (!client.getRetryPolicy().shouldRetry(retryCount, -1)) { + updateDriverMetrics(retryCount, httpOperation.getStatusCode()); throw new InvalidAbfsRestOperationException(ex); } From 324d2c047ec9475b3497e8681b89b7c2e2421649 Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Thu, 23 Jun 2022 10:42:32 +0530 Subject: [PATCH 09/42] Adding thread safety for metrics update --- .../hadoop/fs/azurebfs/AbfsDriverMetrics.java | 20 +++++++++---------- .../azurebfs/services/AbfsRestOperation.java | 2 +- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java index c3355e73da80b..91cdbbceab28a 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java @@ -132,7 +132,7 @@ public String toString() { .append(entry.getValue().getNumberOfRequestsSucceeded()).append(" "); long totalRequests = entry.getValue().getTotalRequests().get(); if(totalRequests > 0) { - metricString.append(" MinMaxAvg#_").append(entry.getKey()) + metricString.append(" #MinMaxAvg#_").append(entry.getKey()) .append("R_").append("=") .append(String.format("%.5f", (double) entry.getValue().getMinBackoff().get() / 1000L)) .append(" seconds ") @@ -141,18 +141,18 @@ public String toString() { .append(String.format("%.5f", (double) ((entry.getValue().getTotalBackoff().get() / totalRequests) / 1000L))) .append(" seconds "); }else { - metricString.append("MinMaxAvg#_").append(entry.getKey()) + metricString.append(" #MinMaxAvg#_").append(entry.getKey()) .append("R_").append("= 0 seconds "); } } - metricString.append(" BandwidthThrottled = ").append(numberOfBandwidthThrottledRequests) - .append(" IOPSThrottled = ").append(numberOfIOPSThrottledRequests) - .append(" OtherThrottled = ").append(numberOfOtherThrottledRequests).append('\n'); - metricString.append(" Percentage of throttled requests = ").append(percentageOfRequestsThrottled).append('\n'); - metricString.append(" Total number of requests which succeeded without retrying = ").append(numberOfRequestsSucceededWithoutRetrying).append('\n'); - metricString.append(" Total number of requests which failed = ").append(numberOfRequestsFailed).append('\n'); - metricString.append(" Total number of requests = ").append(totalNumberOfRequests).append('\n'); - metricString.append(" Max retry count = ").append(maxRetryCount).append('\n'); + metricString.append(" #BandwidthThrottled = ").append(numberOfBandwidthThrottledRequests) + .append(" #IOPSThrottled = ").append(numberOfIOPSThrottledRequests) + .append(" #OtherThrottled = ").append(numberOfOtherThrottledRequests).append('\n'); + metricString.append(" #Percentage of throttled requests = ").append(percentageOfRequestsThrottled).append('\n'); + metricString.append(" #Total number of requests which succeeded without retrying = ").append(numberOfRequestsSucceededWithoutRetrying).append('\n'); + metricString.append(" #Total number of requests which failed = ").append(numberOfRequestsFailed).append('\n'); + metricString.append(" #Total number of requests = ").append(totalNumberOfRequests).append('\n'); + metricString.append(" #Max retry count = ").append(maxRetryCount).append('\n'); return metricString + " "; } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java index 049034fdb5ab9..d9791e288e80c 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java @@ -249,7 +249,7 @@ private void completeExecute(TracingContext tracingContext) LOG.trace("{} REST operation complete", operationType); } - private void updateDriverMetrics(int retryCount, int statusCode){ + private synchronized void updateDriverMetrics(int retryCount, int statusCode){ if (statusCode == HttpURLConnection.HTTP_UNAVAILABLE) { if (retryCount == 30) { abfsDriverMetrics.getNumberOfRequestsFailed().getAndIncrement(); From b9273d3383dc1d65826093170a414d5826e55ed1 Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Tue, 5 Jul 2022 15:44:20 +0530 Subject: [PATCH 10/42] Made the metric enablement configurable --- .../hadoop/fs/azurebfs/AbfsConfiguration.java | 9 ++++ .../fs/azurebfs/AzureBlobFileSystem.java | 48 ++++++++++--------- .../fs/azurebfs/AzureBlobFileSystemStore.java | 2 +- .../azurebfs/constants/ConfigurationKeys.java | 4 +- .../constants/FileSystemConfigurations.java | 1 + .../fs/azurebfs/services/AbfsClient.java | 6 +-- .../fs/azurebfs/utils/TracingContext.java | 19 +++++++- .../azurebfs/utils/TracingHeaderFormat.java | 5 +- .../utils/TracingHeaderValidator.java | 3 ++ 9 files changed, 65 insertions(+), 32 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java index fafc30372b4a5..4ac8a821a2388 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java @@ -112,6 +112,11 @@ public class AbfsConfiguration{ DefaultValue = DEFAULT_READ_SMALL_FILES_COMPLETELY) private boolean readSmallFilesCompletely; + @BooleanConfigurationValidatorAnnotation( + ConfigurationKey = AZURE_ENABLE_METRIC_COLLECTION, + DefaultValue = DEFAULT_AZURE_ENABLE_METRIC_COLLECTION) + private boolean enableMetricCollection; + @BooleanConfigurationValidatorAnnotation( ConfigurationKey = AZURE_READ_OPTIMIZE_FOOTER_READ, DefaultValue = DEFAULT_OPTIMIZE_FOOTER_READ) @@ -689,6 +694,10 @@ public String getAppendBlobDirs() { return this.azureAppendBlobDirs; } + public boolean isMetricCollectionEnabled() { + return enableMetricCollection; + } + public String getAzureInfiniteLeaseDirs() { return this.azureInfiniteLeaseDirs; } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java index 645bd11db354c..e08eda6777e1f 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java @@ -144,6 +144,8 @@ public class AzureBlobFileSystem extends FileSystem private AbfsDelegationTokenManager delegationTokenManager; private AbfsCounters abfsCounters; private String clientCorrelationId; + + private boolean isMetricCollectionEnabled; private TracingHeaderFormat tracingHeaderFormat; private Listener listener; @@ -201,6 +203,7 @@ public void initialize(URI uri, Configuration configuration) .getAbfsConfiguration(); clientCorrelationId = TracingContext.validateClientCorrelationID( abfsConfiguration.getClientCorrelationId()); + isMetricCollectionEnabled = abfsConfiguration.isMetricCollectionEnabled(); tracingHeaderFormat = abfsConfiguration.getTracingHeaderFormat(); this.setWorkingDirectory(this.getHomeDirectory()); @@ -681,27 +684,27 @@ public synchronized void close() throws IOException { if (isClosed) { return; } + if(isMetricCollectionEnabled) { if(abfsCounters.getAbfsDriverMetrics().getTotalNumberOfRequests().get() > 0) { try { - String metric = abfsCounters.getAbfsDriverMetrics().toString(); - Configuration metricConfig = getConf(); - String metricAccountName = getConf().get(FS_AZURE_METRIC_ACCOUNT_NAME); - String metricAccountKey = getConf().get(FS_AZURE_METRIC_ACCOUNT_KEY); - final String abfsMetricUrl = "metrics" + "@" + metricAccountName; - metricConfig.set(FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME, metricAccountKey); - URI metricUri; - try { - metricUri = new URI(getScheme(), abfsMetricUrl, null, null, null); - } catch (URISyntaxException ex) { - throw new AssertionError(ex); - } - AzureBlobFileSystem metricfs - = (AzureBlobFileSystem) FileSystem.newInstance(metricUri, - metricConfig); - metricfs.sentMetric(metric); - }catch (AzureBlobFileSystemException ex) { + String metric = abfsCounters.getAbfsDriverMetrics().toString(); + Configuration metricConfig = getConf(); + String metricAccountName = getConf().get(FS_AZURE_METRIC_ACCOUNT_NAME); + String metricAccountKey = getConf().get(FS_AZURE_METRIC_ACCOUNT_KEY); + final String abfsMetricUrl = "metrics" + "@" + metricAccountName; + metricConfig.set(FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME, metricAccountKey); + URI metricUri; + try { + metricUri = new URI(getScheme(), abfsMetricUrl, null, null, null); + } catch (URISyntaxException ex) { + throw new AssertionError(ex); + } + AzureBlobFileSystem metricFs = (AzureBlobFileSystem) FileSystem.newInstance(metricUri, metricConfig); + metricFs.sentMetric(metric); + } catch (AzureBlobFileSystemException ex) { // do nothing - } + } + } } // does all the delete-on-exit calls, and may be slow. super.close(); @@ -720,10 +723,11 @@ public synchronized void close() throws IOException { } public void sentMetric(String metric) throws AzureBlobFileSystemException { - TracingContext tracingContext = new TracingContext(clientCorrelationId, - fileSystemId, FSOperationType.GET_ATTR, true, tracingHeaderFormat, - listener); - abfsStore.sentMetric(metric, tracingContext); + TracingHeaderFormat tracingHeaderFormatMetric = TracingHeaderFormat.INTERNAL_METRIC_FORMAT; + TracingContext tracingContextMetric = new TracingContext(clientCorrelationId, + fileSystemId, FSOperationType.GET_ATTR, true, tracingHeaderFormatMetric, + listener, metric); + abfsStore.sentMetric(metric, tracingContextMetric); } @Override diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java index 17a2ffbabf5e2..c5fad10f217e9 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java @@ -1975,6 +1975,6 @@ public static String extractEtagHeader(AbfsHttpOperation result) { } public void sentMetric(String metric, TracingContext tracingContext) throws AzureBlobFileSystemException{ - client.getPathStatusMetric("/..$$@@", tracingContext, metric); // Will sent a GFS calls that will fail to register in MDM x-ms-client-metric + client.getPathStatusMetric("/..$$@@", tracingContext); // Will sent a GFS calls that will fail to register in MDM x-ms-client-metric } } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java index 99963fe0cda89..a79d95d72d830 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java @@ -36,14 +36,12 @@ public final class ConfigurationKeys { */ public static final String FS_AZURE_ACCOUNT_IS_HNS_ENABLED = "fs.azure.account.hns.enabled"; public static final String FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME = "fs.azure.account.key"; - public static final String FS_AZURE_METRIC_ACCOUNT_NAME = "fs.azure.metric.account.name"; - public static final String FS_AZURE_METRIC_ACCOUNT_KEY = "fs.azure.metric.account.key"; public static final String FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME_REGX = "fs\\.azure\\.account\\.key\\.(.*)"; public static final String FS_AZURE_SECURE_MODE = "fs.azure.secure.mode"; - + public static final String AZURE_ENABLE_METRIC_COLLECTION = "fs.azure.enable.metric.collection"; // Retry strategy defined by the user public static final String AZURE_MIN_BACKOFF_INTERVAL = "fs.azure.io.retry.min.backoff.interval"; public static final String AZURE_MAX_BACKOFF_INTERVAL = "fs.azure.io.retry.max.backoff.interval"; diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java index 49d0a1fd18917..3d7263790a0fe 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java @@ -60,6 +60,7 @@ public final class FileSystemConfigurations { public static final int DEFAULT_READ_BUFFER_SIZE = 4 * ONE_MB; // 4 MB public static final boolean DEFAULT_READ_SMALL_FILES_COMPLETELY = false; public static final boolean DEFAULT_OPTIMIZE_FOOTER_READ = false; + public static final boolean DEFAULT_AZURE_ENABLE_METRIC_COLLECTION = false; public static final boolean DEFAULT_ALWAYS_READ_BUFFER_SIZE = false; public static final int DEFAULT_READ_AHEAD_BLOCK_SIZE = 4 * ONE_MB; public static final int DEFAULT_READ_AHEAD_RANGE = 64 * ONE_KB; // 64 KB diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java index d6a09c2fdf582..8f3abd00737a4 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java @@ -751,13 +751,11 @@ public AbfsRestOperation setPathProperties(final String path, final String prope } public AbfsRestOperation getPathStatusMetric(final String path, - TracingContext tracingContext, String metric) throws AzureBlobFileSystemException { + TracingContext tracingContext) throws AzureBlobFileSystemException { final List requestHeaders = createDefaultHeaders(); - requestHeaders.add(new AbfsHttpHeader(X_MS_CLIENT_REQUEST_ID, metric)); final AbfsUriQueryBuilder abfsUriQueryBuilder = createDefaultUriQueryBuilder(); - String operation = SASTokenProvider.GET_PROPERTIES_OPERATION; + String operation = SASTokenProvider.GET_STATUS_OPERATION; abfsUriQueryBuilder.addQuery(HttpQueryParams.QUERY_PARAM_ACTION, AbfsHttpConstants.GET_STATUS); - operation = SASTokenProvider.GET_STATUS_OPERATION; abfsUriQueryBuilder.addQuery(HttpQueryParams.QUERY_PARAM_UPN, String.valueOf(abfsConfiguration.isUpnUsed())); appendSASTokenToQuery(path, operation, abfsUriQueryBuilder); final URL url = createRequestUrl(path, abfsUriQueryBuilder.toString()); diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/TracingContext.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/TracingContext.java index 85580919e910c..70d4371daefe4 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/TracingContext.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/TracingContext.java @@ -63,6 +63,8 @@ public class TracingContext { //final concatenated ID list set into x-ms-client-request-id header private String header = EMPTY_STRING; + private String metricResults = EMPTY_STRING; + private static final Logger LOG = LoggerFactory.getLogger(AbfsClient.class); public static final int MAX_CLIENT_CORRELATION_ID_LENGTH = 72; public static final String CLIENT_CORRELATION_ID_PATTERN = "[a-zA-Z0-9-]*"; @@ -101,6 +103,14 @@ public TracingContext(String clientCorrelationID, String fileSystemID, } } + public TracingContext(String clientCorrelationID, String fileSystemID, + FSOperationType opType, boolean needsPrimaryReqId, + TracingHeaderFormat tracingHeaderFormat, Listener listener, + String metricResults) { + this(clientCorrelationID, fileSystemID, opType, needsPrimaryReqId, tracingHeaderFormat, listener); + this.metricResults = metricResults; + } + public TracingContext(TracingContext originalTracingContext) { this.fileSystemID = originalTracingContext.fileSystemID; this.streamID = originalTracingContext.streamID; @@ -123,6 +133,10 @@ public static String validateClientCorrelationID(String clientCorrelationID) { return clientCorrelationID; } + public String getMetricResults() { + return metricResults; + } + public void setPrimaryRequestID() { primaryRequestId = UUID.randomUUID().toString(); if (listener != null) { @@ -164,13 +178,16 @@ public void constructHeader(AbfsHttpOperation httpOperation) { case TWO_ID_FORMAT: header = clientCorrelationID + ":" + clientRequestId; break; + case INTERNAL_METRIC_FORMAT: + header = clientCorrelationID + ":" + clientRequestId + ":" + fileSystemID + ":" + metricResults; + break; default: header = clientRequestId; //case SINGLE_ID_FORMAT } if (listener != null) { //for testing listener.callTracingHeaderValidator(header, format); } - //httpOperation.setRequestProperty(HttpHeaderConfigurations.X_MS_CLIENT_REQUEST_ID, header); + httpOperation.setRequestProperty(HttpHeaderConfigurations.X_MS_CLIENT_REQUEST_ID, header); } /** diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/TracingHeaderFormat.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/TracingHeaderFormat.java index 3f23ae3ed7c14..010308e85b5b7 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/TracingHeaderFormat.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/TracingHeaderFormat.java @@ -23,6 +23,9 @@ public enum TracingHeaderFormat { TWO_ID_FORMAT, // : - ALL_ID_FORMAT; // :: + ALL_ID_FORMAT, // :: // :::: + + INTERNAL_METRIC_FORMAT; // :: + // : } diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/TracingHeaderValidator.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/TracingHeaderValidator.java index e195f1c381a94..4cd21906c077b 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/TracingHeaderValidator.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/TracingHeaderValidator.java @@ -97,6 +97,9 @@ private void validateBasicFormat(String[] idList) { } else if (format == TracingHeaderFormat.TWO_ID_FORMAT) { Assertions.assertThat(idList) .describedAs("header should have 2 elements").hasSize(2); + } else if (format == TracingHeaderFormat.INTERNAL_METRIC_FORMAT) { + Assertions.assertThat(idList) + .describedAs("header should have 4 elements").hasSize(4); } else { Assertions.assertThat(idList).describedAs("header should have 1 element") .hasSize(1); From 5fc14174136dde4dc491f7c12d345fba96ad42fb Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Wed, 6 Jul 2022 14:17:02 +0530 Subject: [PATCH 11/42] Removed hardcoded value for maxIoRetries --- .../hadoop/fs/azurebfs/services/AbfsClient.java | 8 ++++++++ .../fs/azurebfs/services/AbfsRestOperation.java | 11 ++++------- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java index 8f3abd00737a4..27d6e99b4db36 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java @@ -1243,6 +1243,14 @@ protected AbfsCounters getAbfsCounters() { return abfsCounters; } + /** + * Getter for abfsConfiguration from AbfsClient. + * @return AbfsConfiguration instance + */ + protected AbfsConfiguration getAbfsConfiguration() { + return abfsConfiguration; + } + public int getNumLeaseThreads() { return abfsConfiguration.getNumLeaseThreads(); } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java index d9791e288e80c..ec4c8ab7c6b4d 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java @@ -72,6 +72,7 @@ public class AbfsRestOperation { private int retryCount = 0; private boolean isThrottledRequest = false; private long maxRetryCount = 0L; + private int maxIoRetries = 0; private AbfsHttpOperation result; private AbfsCounters abfsCounters; private AbfsDriverMetrics abfsDriverMetrics; @@ -123,8 +124,6 @@ String getSasToken() { final URL url, final List requestHeaders) { this(operationType, client, method, url, requestHeaders, null); - this.abfsDriverMetrics = abfsCounters.getAbfsDriverMetrics(); - this.metricsMap = abfsDriverMetrics.getMetricsMap(); } /** @@ -154,6 +153,7 @@ String getSasToken() { this.abfsCounters = client.getAbfsCounters(); this.abfsDriverMetrics = abfsCounters.getAbfsDriverMetrics(); this.metricsMap = abfsDriverMetrics.getMetricsMap(); + this.maxIoRetries = client.getAbfsConfiguration().getMaxIoRetries(); } /** @@ -183,9 +183,6 @@ String getSasToken() { this.buffer = buffer; this.bufferOffset = bufferOffset; this.bufferLength = bufferLength; - this.abfsCounters = client.getAbfsCounters(); - this.abfsDriverMetrics = abfsCounters.getAbfsDriverMetrics(); - this.metricsMap = abfsDriverMetrics.getMetricsMap(); } /** @@ -251,11 +248,11 @@ private void completeExecute(TracingContext tracingContext) private synchronized void updateDriverMetrics(int retryCount, int statusCode){ if (statusCode == HttpURLConnection.HTTP_UNAVAILABLE) { - if (retryCount == 30) { + if (retryCount >= maxIoRetries) { abfsDriverMetrics.getNumberOfRequestsFailed().getAndIncrement(); } } else { - if (retryCount > 0 && retryCount <= 30) { + if (retryCount > 0 && retryCount <= maxIoRetries) { maxRetryCount = Math.max(abfsDriverMetrics.getMaxRetryCount().get(), retryCount); abfsDriverMetrics.getMaxRetryCount().set(maxRetryCount); updateCount(retryCount); From b185269396001ffa2842d1b18e78da954c719751 Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Thu, 7 Jul 2022 14:23:16 +0530 Subject: [PATCH 12/42] Fix null pointer exceptions --- .../fs/azurebfs/AzureBlobFileSystem.java | 6 +++--- .../azurebfs/services/AbfsRestOperation.java | 19 ++++++++++++++----- .../azurebfs/ITestAbfsReadWriteAndSeek.java | 13 ++++--------- .../fs/azurebfs/services/TestAbfsClient.java | 2 +- 4 files changed, 22 insertions(+), 18 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java index e08eda6777e1f..ffe0e46d16986 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java @@ -689,8 +689,8 @@ public synchronized void close() throws IOException { try { String metric = abfsCounters.getAbfsDriverMetrics().toString(); Configuration metricConfig = getConf(); - String metricAccountName = getConf().get(FS_AZURE_METRIC_ACCOUNT_NAME); - String metricAccountKey = getConf().get(FS_AZURE_METRIC_ACCOUNT_KEY); + String metricAccountName = metricConfig.get(FS_AZURE_METRIC_ACCOUNT_NAME); + String metricAccountKey = metricConfig.get(FS_AZURE_METRIC_ACCOUNT_KEY); final String abfsMetricUrl = "metrics" + "@" + metricAccountName; metricConfig.set(FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME, metricAccountKey); URI metricUri; @@ -702,7 +702,7 @@ public synchronized void close() throws IOException { AzureBlobFileSystem metricFs = (AzureBlobFileSystem) FileSystem.newInstance(metricUri, metricConfig); metricFs.sentMetric(metric); } catch (AzureBlobFileSystemException ex) { - // do nothing + //do nothing } } } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java index ec4c8ab7c6b4d..d68b4093ed8e9 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java @@ -151,8 +151,11 @@ String getSasToken() { || AbfsHttpConstants.HTTP_METHOD_PATCH.equals(method)); this.sasToken = sasToken; this.abfsCounters = client.getAbfsCounters(); - this.abfsDriverMetrics = abfsCounters.getAbfsDriverMetrics(); - this.metricsMap = abfsDriverMetrics.getMetricsMap(); + if(abfsCounters != null) { + this.abfsDriverMetrics = abfsCounters.getAbfsDriverMetrics(); + }if(abfsDriverMetrics != null) { + this.metricsMap = abfsDriverMetrics.getMetricsMap(); + } this.maxIoRetries = client.getAbfsConfiguration().getMaxIoRetries(); } @@ -223,7 +226,9 @@ private void completeExecute(TracingContext tracingContext) retryCount = 0; LOG.debug("First execution of REST operation - {}", operationType); long sleepDuration = 0L; - abfsDriverMetrics.getTotalNumberOfRequests().getAndIncrement(); + if(abfsDriverMetrics != null) { + abfsDriverMetrics.getTotalNumberOfRequests().getAndIncrement(); + } while (!executeHttpOperation(retryCount, tracingContext)) { try { ++retryCount; @@ -231,13 +236,17 @@ private void completeExecute(TracingContext tracingContext) LOG.debug("Retrying REST operation {}. RetryCount = {}", operationType, retryCount); sleepDuration = client.getRetryPolicy().getRetryInterval(retryCount); - updateTimeMetrics(retryCount, sleepDuration); + if(abfsDriverMetrics != null) { + updateTimeMetrics(retryCount, sleepDuration); + } Thread.sleep(sleepDuration); } catch (InterruptedException ex) { Thread.currentThread().interrupt(); } } - updateDriverMetrics(retryCount, result.getStatusCode()); + if(abfsDriverMetrics != null) { + updateDriverMetrics(retryCount, result.getStatusCode()); + } if (result.getStatusCode() >= HttpURLConnection.HTTP_BAD_REQUEST) { throw new AbfsRestOperationException(result.getStatusCode(), result.getStorageErrorCode(), result.getStorageErrorMessage(), null, result); diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsReadWriteAndSeek.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsReadWriteAndSeek.java index ff60306dac9b5..73c8dc8d62068 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsReadWriteAndSeek.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsReadWriteAndSeek.java @@ -51,16 +51,11 @@ public class ITestAbfsReadWriteAndSeek extends AbstractAbfsScaleTest { private static final String TEST_PATH = "/testfile"; // @Parameterized.Parameters(name = "Size={0}") -// public static Iterable sizes() { -// return Arrays.asList(new Object[][]{{MIN_BUFFER_SIZE}, -// {DEFAULT_READ_BUFFER_SIZE}, -// {APPENDBLOB_MAX_WRITE_BUFFER_SIZE}, -// {MAX_BUFFER_SIZE}}); -// } - - @Parameterized.Parameters(name = "Size={0}") public static Iterable sizes() { - return Arrays.asList(new Object[][]{{MIN_BUFFER_SIZE}}); + return Arrays.asList(new Object[][]{{MIN_BUFFER_SIZE}, + {DEFAULT_READ_BUFFER_SIZE}, + {APPENDBLOB_MAX_WRITE_BUFFER_SIZE}, + {MAX_BUFFER_SIZE}}); } private final int size; diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsClient.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsClient.java index a725bf3175a5c..68907d20b02fb 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsClient.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsClient.java @@ -306,7 +306,7 @@ public static AbfsClient getMockAbfsClient(AbfsClient baseAbfsClientInstance, when(client.getAccessToken()).thenCallRealMethod(); when(client.getSharedKeyCredentials()).thenCallRealMethod(); when(client.createDefaultHeaders()).thenCallRealMethod(); - + when(client.getAbfsConfiguration()).thenReturn(abfsConfig); // override baseurl client = TestAbfsClient.setAbfsClientField(client, "abfsConfiguration", abfsConfig); From ef9691a29d3889b7d89c6a884fb0e13a38490efe Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Fri, 8 Jul 2022 17:02:55 +0530 Subject: [PATCH 13/42] Fix for acronyms --- .../hadoop/fs/azurebfs/AbfsDriverMetrics.java | 44 ++++++++++++------- .../azurebfs/services/AbfsRestOperation.java | 2 +- .../azurebfs/ITestAbfsReadWriteAndSeek.java | 2 +- 3 files changed, 31 insertions(+), 17 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java index 91cdbbceab28a..6d95d61c7b11f 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java @@ -59,7 +59,7 @@ public AbfsDriverMetrics(String retryCount) { } private void initializeMap() { - ArrayList retryCountList = new ArrayList(Arrays.asList("1","2","3","4","5_15","15_25","25_30")); + ArrayList retryCountList = new ArrayList(Arrays.asList("1","2","3","4","5_15","15_25","25andabove")); for (String s : retryCountList) { metricsMap.put(s, new AbfsDriverMetrics(s)); } @@ -121,6 +121,20 @@ public AtomicLong getNumberOfRequestsFailed() { return numberOfRequestsFailed; } + /* + Acronyms :- + 1.RCTSI :- Request count that succeeded in x retries + 2.MMA :- Min Max Average (This refers to the backoff or sleep time between 2 requests) + 3.s :- seconds + 4.BWT :- Number of Bandwidth throttled requests + 5.IT :- Number of IOPS throttled requests + 6.OT :- Number of Other throttled requests + 7.%RT :- Percentage of requests that are throttled + 8.TRNR :- Total number of requests which succeeded without retrying + 9.TRF :- Total number of requests which failed + 10.TR :- Total number of requests which were made + 11.MRC :- Max retry count across all requests + */ @Override public String toString() { StringBuilder metricString = new StringBuilder(); @@ -132,27 +146,27 @@ public String toString() { .append(entry.getValue().getNumberOfRequestsSucceeded()).append(" "); long totalRequests = entry.getValue().getTotalRequests().get(); if(totalRequests > 0) { - metricString.append(" #MinMaxAvg#_").append(entry.getKey()) + metricString.append(" #MMA#_").append(entry.getKey()) .append("R_").append("=") .append(String.format("%.5f", (double) entry.getValue().getMinBackoff().get() / 1000L)) - .append(" seconds ") + .append(" s ") .append(String.format("%.5f", (double) entry.getValue().getMaxBackoff().get() / 1000L)) - .append(" seconds ") + .append(" s ") .append(String.format("%.5f", (double) ((entry.getValue().getTotalBackoff().get() / totalRequests) / 1000L))) - .append(" seconds "); + .append(" s "); }else { - metricString.append(" #MinMaxAvg#_").append(entry.getKey()) - .append("R_").append("= 0 seconds "); + metricString.append(" #MMA#_").append(entry.getKey()) + .append("R_").append("= 0 s "); } } - metricString.append(" #BandwidthThrottled = ").append(numberOfBandwidthThrottledRequests) - .append(" #IOPSThrottled = ").append(numberOfIOPSThrottledRequests) - .append(" #OtherThrottled = ").append(numberOfOtherThrottledRequests).append('\n'); - metricString.append(" #Percentage of throttled requests = ").append(percentageOfRequestsThrottled).append('\n'); - metricString.append(" #Total number of requests which succeeded without retrying = ").append(numberOfRequestsSucceededWithoutRetrying).append('\n'); - metricString.append(" #Total number of requests which failed = ").append(numberOfRequestsFailed).append('\n'); - metricString.append(" #Total number of requests = ").append(totalNumberOfRequests).append('\n'); - metricString.append(" #Max retry count = ").append(maxRetryCount).append('\n'); + metricString.append(" #BWT=").append(numberOfBandwidthThrottledRequests) + .append(" #IT=").append(numberOfIOPSThrottledRequests) + .append(" #OT=").append(numberOfOtherThrottledRequests).append('\n'); + metricString.append(" #%RT=").append(percentageOfRequestsThrottled).append('\n'); + metricString.append(" #TRNR=").append(numberOfRequestsSucceededWithoutRetrying).append('\n'); + metricString.append(" #TRF=").append(numberOfRequestsFailed).append('\n'); + metricString.append(" #TR=").append(totalNumberOfRequests).append('\n'); + metricString.append(" #MRC = ").append(maxRetryCount).append('\n'); return metricString + " "; } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java index d68b4093ed8e9..bd6abe8dc2532 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java @@ -423,7 +423,7 @@ private String getKey(int retryCount) { }else if(retryCount >= 15 && retryCount < 25){ retryCounter = "15_25"; }else{ - retryCounter = "25_30"; + retryCounter = "25andabove"; } return retryCounter; } diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsReadWriteAndSeek.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsReadWriteAndSeek.java index 73c8dc8d62068..5bd6eaff42e84 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsReadWriteAndSeek.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsReadWriteAndSeek.java @@ -50,7 +50,7 @@ public class ITestAbfsReadWriteAndSeek extends AbstractAbfsScaleTest { private static final String TEST_PATH = "/testfile"; -// @Parameterized.Parameters(name = "Size={0}") + @Parameterized.Parameters(name = "Size={0}") public static Iterable sizes() { return Arrays.asList(new Object[][]{{MIN_BUFFER_SIZE}, {DEFAULT_READ_BUFFER_SIZE}, From 3bddb11b6dbe58f5065973e25d8dc3ac677da135 Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Mon, 11 Jul 2022 12:51:31 +0530 Subject: [PATCH 14/42] Fix for spacing issues --- .../hadoop/fs/azurebfs/AbfsDriverMetrics.java | 22 +++++++++---------- .../azurebfs/services/AbfsRestOperation.java | 2 +- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java index 6d95d61c7b11f..3558f7cae3d05 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java @@ -149,24 +149,24 @@ public String toString() { metricString.append(" #MMA#_").append(entry.getKey()) .append("R_").append("=") .append(String.format("%.5f", (double) entry.getValue().getMinBackoff().get() / 1000L)) - .append(" s ") + .append("s ") .append(String.format("%.5f", (double) entry.getValue().getMaxBackoff().get() / 1000L)) - .append(" s ") + .append("s ") .append(String.format("%.5f", (double) ((entry.getValue().getTotalBackoff().get() / totalRequests) / 1000L))) - .append(" s "); + .append("s "); }else { metricString.append(" #MMA#_").append(entry.getKey()) - .append("R_").append("= 0 s "); + .append("R_").append("=0s "); } } - metricString.append(" #BWT=").append(numberOfBandwidthThrottledRequests) + metricString.append("#BWT=").append(numberOfBandwidthThrottledRequests) .append(" #IT=").append(numberOfIOPSThrottledRequests) - .append(" #OT=").append(numberOfOtherThrottledRequests).append('\n'); - metricString.append(" #%RT=").append(percentageOfRequestsThrottled).append('\n'); - metricString.append(" #TRNR=").append(numberOfRequestsSucceededWithoutRetrying).append('\n'); - metricString.append(" #TRF=").append(numberOfRequestsFailed).append('\n'); - metricString.append(" #TR=").append(totalNumberOfRequests).append('\n'); - metricString.append(" #MRC = ").append(maxRetryCount).append('\n'); + .append(" #OT=").append(numberOfOtherThrottledRequests); + metricString.append(" #%RT=").append(String.format("%.3f",percentageOfRequestsThrottled)); + metricString.append(" #TRNR=").append(numberOfRequestsSucceededWithoutRetrying); + metricString.append(" #TRF=").append(numberOfRequestsFailed); + metricString.append(" #TR=").append(totalNumberOfRequests); + metricString.append(" #MRC=").append(maxRetryCount); return metricString + " "; } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java index bd6abe8dc2532..d8ca9db178edb 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java @@ -74,7 +74,7 @@ public class AbfsRestOperation { private long maxRetryCount = 0L; private int maxIoRetries = 0; private AbfsHttpOperation result; - private AbfsCounters abfsCounters; + private final AbfsCounters abfsCounters; private AbfsDriverMetrics abfsDriverMetrics; private Map metricsMap; /** From 4afba696af035940577f99a136af75af7ce25787 Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Tue, 12 Jul 2022 09:52:34 +0530 Subject: [PATCH 15/42] Added documentation for configs --- .../apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java | 12 ++++++------ .../hadoop/fs/azurebfs/AzureBlobFileSystem.java | 10 +++++----- .../hadoop/fs/azurebfs/AzureBlobFileSystemStore.java | 2 +- hadoop-tools/hadoop-azure/src/site/markdown/abfs.md | 12 ++++++++++++ 4 files changed, 24 insertions(+), 12 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java index 3558f7cae3d05..029b0eff56ea1 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java @@ -141,21 +141,21 @@ public String toString() { long totalRequestsThrottled = numberOfBandwidthThrottledRequests.get() + numberOfIOPSThrottledRequests.get() + numberOfOtherThrottledRequests.get(); double percentageOfRequestsThrottled = ((double)totalRequestsThrottled/totalNumberOfRequests.get())*100; for (Map.Entry entry : metricsMap.entrySet()) { - metricString.append(" #RCTSI#_").append(entry.getKey()) + metricString.append("#RCTSI#_").append(entry.getKey()) .append("R_").append("=") .append(entry.getValue().getNumberOfRequestsSucceeded()).append(" "); long totalRequests = entry.getValue().getTotalRequests().get(); if(totalRequests > 0) { - metricString.append(" #MMA#_").append(entry.getKey()) + metricString.append("#MMA#_").append(entry.getKey()) .append("R_").append("=") - .append(String.format("%.5f", (double) entry.getValue().getMinBackoff().get() / 1000L)) + .append(String.format("%.3f", (double) entry.getValue().getMinBackoff().get() / 1000L)) .append("s ") - .append(String.format("%.5f", (double) entry.getValue().getMaxBackoff().get() / 1000L)) + .append(String.format("%.3f", (double) entry.getValue().getMaxBackoff().get() / 1000L)) .append("s ") - .append(String.format("%.5f", (double) ((entry.getValue().getTotalBackoff().get() / totalRequests) / 1000L))) + .append(String.format("%.3f", (double) ((entry.getValue().getTotalBackoff().get() / totalRequests) / 1000L))) .append("s "); }else { - metricString.append(" #MMA#_").append(entry.getKey()) + metricString.append("#MMA#_").append(entry.getKey()) .append("R_").append("=0s "); } } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java index ffe0e46d16986..0918e25886a6c 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java @@ -144,7 +144,6 @@ public class AzureBlobFileSystem extends FileSystem private AbfsDelegationTokenManager delegationTokenManager; private AbfsCounters abfsCounters; private String clientCorrelationId; - private boolean isMetricCollectionEnabled; private TracingHeaderFormat tracingHeaderFormat; private Listener listener; @@ -684,10 +683,11 @@ public synchronized void close() throws IOException { if (isClosed) { return; } + String metric = abfsCounters.getAbfsDriverMetrics().toString(); + LOG.debug("The metrics collected over this instance are " + metric); if(isMetricCollectionEnabled) { if(abfsCounters.getAbfsDriverMetrics().getTotalNumberOfRequests().get() > 0) { try { - String metric = abfsCounters.getAbfsDriverMetrics().toString(); Configuration metricConfig = getConf(); String metricAccountName = metricConfig.get(FS_AZURE_METRIC_ACCOUNT_NAME); String metricAccountKey = metricConfig.get(FS_AZURE_METRIC_ACCOUNT_KEY); @@ -700,7 +700,7 @@ public synchronized void close() throws IOException { throw new AssertionError(ex); } AzureBlobFileSystem metricFs = (AzureBlobFileSystem) FileSystem.newInstance(metricUri, metricConfig); - metricFs.sentMetric(metric); + metricFs.sendMetric(metric); } catch (AzureBlobFileSystemException ex) { //do nothing } @@ -722,12 +722,12 @@ public synchronized void close() throws IOException { } } - public void sentMetric(String metric) throws AzureBlobFileSystemException { + public void sendMetric(String metric) throws AzureBlobFileSystemException { TracingHeaderFormat tracingHeaderFormatMetric = TracingHeaderFormat.INTERNAL_METRIC_FORMAT; TracingContext tracingContextMetric = new TracingContext(clientCorrelationId, fileSystemId, FSOperationType.GET_ATTR, true, tracingHeaderFormatMetric, listener, metric); - abfsStore.sentMetric(metric, tracingContextMetric); + abfsStore.sendMetric(metric, tracingContextMetric); } @Override diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java index c5fad10f217e9..e45fa3141604d 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java @@ -1974,7 +1974,7 @@ public static String extractEtagHeader(AbfsHttpOperation result) { return etag; } - public void sentMetric(String metric, TracingContext tracingContext) throws AzureBlobFileSystemException{ + public void sendMetric(String metric, TracingContext tracingContext) throws AzureBlobFileSystemException{ client.getPathStatusMetric("/..$$@@", tracingContext); // Will sent a GFS calls that will fail to register in MDM x-ms-client-metric } } diff --git a/hadoop-tools/hadoop-azure/src/site/markdown/abfs.md b/hadoop-tools/hadoop-azure/src/site/markdown/abfs.md index 35d360556047e..46bef8546ae98 100644 --- a/hadoop-tools/hadoop-azure/src/site/markdown/abfs.md +++ b/hadoop-tools/hadoop-azure/src/site/markdown/abfs.md @@ -991,6 +991,18 @@ Note that these performance numbers are also sent back to the ADLS Gen 2 API end in the `x-ms-abfs-client-latency` HTTP headers in subsequent requests. Azure uses these settings to track their end-to-end latency. +### Driver Metric Options + +`fs.azure.metric.account.name`: This configuration parameter is used to specify +the name of the account which will be used to push a failed(404) GetPathStatus request to the +backend. We can configure a separate account to push metrics to the store or use the same for as the existing account on which other requests are made. + +`fs.azure.metric.account.key`: This is the access key for the storage account +used for pushing metrics to the store. + +`fs.azure.enable.metric.collection`: This configuration provides an option to specify whether we want to push the metrics or not. +By default, this config will be set to false. + ## Troubleshooting The problems associated with the connector usually come down to, in order From 0edb3687cf8c943a44ccf572f951f2d0d0a15e96 Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Tue, 12 Jul 2022 09:59:33 +0530 Subject: [PATCH 16/42] Update variable name --- .../org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java index 0918e25886a6c..c19754b4746a4 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java @@ -144,7 +144,7 @@ public class AzureBlobFileSystem extends FileSystem private AbfsDelegationTokenManager delegationTokenManager; private AbfsCounters abfsCounters; private String clientCorrelationId; - private boolean isMetricCollectionEnabled; + private boolean sendMetricsToStore; private TracingHeaderFormat tracingHeaderFormat; private Listener listener; @@ -202,7 +202,7 @@ public void initialize(URI uri, Configuration configuration) .getAbfsConfiguration(); clientCorrelationId = TracingContext.validateClientCorrelationID( abfsConfiguration.getClientCorrelationId()); - isMetricCollectionEnabled = abfsConfiguration.isMetricCollectionEnabled(); + sendMetricsToStore = abfsConfiguration.isMetricCollectionEnabled(); tracingHeaderFormat = abfsConfiguration.getTracingHeaderFormat(); this.setWorkingDirectory(this.getHomeDirectory()); @@ -685,7 +685,7 @@ public synchronized void close() throws IOException { } String metric = abfsCounters.getAbfsDriverMetrics().toString(); LOG.debug("The metrics collected over this instance are " + metric); - if(isMetricCollectionEnabled) { + if(sendMetricsToStore) { if(abfsCounters.getAbfsDriverMetrics().getTotalNumberOfRequests().get() > 0) { try { Configuration metricConfig = getConf(); From 791b84e60254286b72bd8539fc8f76df486ef814 Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Tue, 12 Jul 2022 12:42:53 +0530 Subject: [PATCH 17/42] Added config for URI --- .../fs/azurebfs/AzureBlobFileSystem.java | 11 ++++++++--- .../fs/azurebfs/AzureBlobFileSystemStore.java | 2 +- .../azurebfs/constants/ConfigurationKeys.java | 1 + .../fs/azurebfs/services/AbfsClient.java | 19 ------------------- .../hadoop-azure/src/site/markdown/abfs.md | 3 +++ 5 files changed, 13 insertions(+), 23 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java index c19754b4746a4..9b3e458656796 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java @@ -49,7 +49,6 @@ import org.apache.hadoop.util.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.concurrent.atomic.AtomicLong; import org.apache.commons.lang3.ArrayUtils; import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.classification.InterfaceAudience; @@ -113,14 +112,17 @@ import static org.apache.hadoop.fs.CommonConfigurationKeys.IOSTATISTICS_LOGGING_LEVEL; import static org.apache.hadoop.fs.CommonConfigurationKeys.IOSTATISTICS_LOGGING_LEVEL_DEFAULT; import static org.apache.hadoop.fs.azurebfs.AbfsStatistic.*; +import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.DATA_BLOCKS_BUFFER; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_BLOCK_UPLOAD_ACTIVE_BLOCKS; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_BLOCK_UPLOAD_BUFFER_DIR; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_METRIC_ACCOUNT_KEY; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_METRIC_ACCOUNT_NAME; +import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_METRIC_URI; import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.BLOCK_UPLOAD_ACTIVE_BLOCKS_DEFAULT; import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DATA_BLOCKS_BUFFER_DEFAULT; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import static org.apache.hadoop.fs.impl.PathCapabilitiesSupport.validatePathCapabilityArgs; import static org.apache.hadoop.fs.statistics.IOStatisticsLogging.logIOStatisticsAtLevel; import static org.apache.hadoop.util.functional.RemoteIterators.filteringRemoteIterator; @@ -689,10 +691,13 @@ public synchronized void close() throws IOException { if(abfsCounters.getAbfsDriverMetrics().getTotalNumberOfRequests().get() > 0) { try { Configuration metricConfig = getConf(); - String metricAccountName = metricConfig.get(FS_AZURE_METRIC_ACCOUNT_NAME); String metricAccountKey = metricConfig.get(FS_AZURE_METRIC_ACCOUNT_KEY); - final String abfsMetricUrl = "metrics" + "@" + metricAccountName; + final String abfsMetricUrl = metricConfig.get(FS_AZURE_METRIC_URI); + if(abfsMetricUrl == null){ + return; + } metricConfig.set(FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME, metricAccountKey); + metricConfig.set(AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION, "false"); URI metricUri; try { metricUri = new URI(getScheme(), abfsMetricUrl, null, null, null); diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java index e45fa3141604d..658291e83e135 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystemStore.java @@ -1975,6 +1975,6 @@ public static String extractEtagHeader(AbfsHttpOperation result) { } public void sendMetric(String metric, TracingContext tracingContext) throws AzureBlobFileSystemException{ - client.getPathStatusMetric("/..$$@@", tracingContext); // Will sent a GFS calls that will fail to register in MDM x-ms-client-metric + client.getPathStatus("/..$$@@", true, tracingContext); // Will sent a GFS calls that will fail to register in MDM x-ms-client-metric } } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java index a79d95d72d830..eddb0ef9a5391 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java @@ -38,6 +38,7 @@ public final class ConfigurationKeys { public static final String FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME = "fs.azure.account.key"; public static final String FS_AZURE_METRIC_ACCOUNT_NAME = "fs.azure.metric.account.name"; public static final String FS_AZURE_METRIC_ACCOUNT_KEY = "fs.azure.metric.account.key"; + public static final String FS_AZURE_METRIC_URI = "fs.azure.metric.uri"; public static final String FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME_REGX = "fs\\.azure\\.account\\.key\\.(.*)"; public static final String FS_AZURE_SECURE_MODE = "fs.azure.secure.mode"; diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java index 27d6e99b4db36..3b351a53205c5 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java @@ -750,25 +750,6 @@ public AbfsRestOperation setPathProperties(final String path, final String prope return op; } - public AbfsRestOperation getPathStatusMetric(final String path, - TracingContext tracingContext) throws AzureBlobFileSystemException { - final List requestHeaders = createDefaultHeaders(); - final AbfsUriQueryBuilder abfsUriQueryBuilder = createDefaultUriQueryBuilder(); - String operation = SASTokenProvider.GET_STATUS_OPERATION; - abfsUriQueryBuilder.addQuery(HttpQueryParams.QUERY_PARAM_ACTION, AbfsHttpConstants.GET_STATUS); - abfsUriQueryBuilder.addQuery(HttpQueryParams.QUERY_PARAM_UPN, String.valueOf(abfsConfiguration.isUpnUsed())); - appendSASTokenToQuery(path, operation, abfsUriQueryBuilder); - final URL url = createRequestUrl(path, abfsUriQueryBuilder.toString()); - final AbfsRestOperation op = new AbfsRestOperation( - AbfsRestOperationType.GetPathStatus, - this, - HTTP_METHOD_HEAD, - url, - requestHeaders); - op.execute(tracingContext); - return op; - } - public AbfsRestOperation getPathStatus(final String path, final boolean includeProperties, TracingContext tracingContext) throws AzureBlobFileSystemException { final List requestHeaders = createDefaultHeaders(); diff --git a/hadoop-tools/hadoop-azure/src/site/markdown/abfs.md b/hadoop-tools/hadoop-azure/src/site/markdown/abfs.md index 46bef8546ae98..12e2bc6a5d7c6 100644 --- a/hadoop-tools/hadoop-azure/src/site/markdown/abfs.md +++ b/hadoop-tools/hadoop-azure/src/site/markdown/abfs.md @@ -1003,6 +1003,9 @@ used for pushing metrics to the store. `fs.azure.enable.metric.collection`: This configuration provides an option to specify whether we want to push the metrics or not. By default, this config will be set to false. +`fs.azure.metric.uri`: This configuration provides the uri in the format of containername@accountname.dfs.core.windows.net. +This should be a part of the config in order to prevent extra calls to create the filesystem. We use an existing filsystem to push the metrics. + ## Troubleshooting The problems associated with the connector usually come down to, in order From f6bad55ee40526b658934f8191d578c6691a4de0 Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Wed, 13 Jul 2022 10:11:53 +0530 Subject: [PATCH 18/42] Test for max retries --- .../ITestAzureBlobFileSystemDelete.java | 35 +++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelete.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelete.java index db181fb5dd660..fc4059d3d7908 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelete.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelete.java @@ -45,10 +45,12 @@ import org.apache.hadoop.fs.Path; import static java.net.HttpURLConnection.HTTP_BAD_REQUEST; +import static java.net.HttpURLConnection.HTTP_INTERNAL_ERROR; import static java.net.HttpURLConnection.HTTP_NOT_FOUND; import static java.net.HttpURLConnection.HTTP_OK; - +import org.mockito.Mockito; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.doCallRealMethod; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; @@ -61,7 +63,6 @@ import static org.apache.hadoop.fs.contract.ContractTestUtils.assertPathDoesNotExist; import static org.apache.hadoop.test.LambdaTestUtils.intercept; - /** * Test delete operation. */ @@ -219,6 +220,36 @@ public void testDeleteIdempotency() throws Exception { } + @Test + public void testTryStatus() throws java.io.IOException { + AbfsConfiguration abfsConfig + = TestAbfsConfigurationFieldsValidation.updateRetryConfigs( + getConfiguration(), + 5, 5000); + + final AzureBlobFileSystem fs = this.getFileSystem(); + AbfsClient abfsClient = fs.getAbfsStore().getClient(); + AbfsClient testClient = TestAbfsClient.createTestClientFromCurrentContext( + abfsClient, + abfsConfig); + + final AbfsRestOperation mockRestOperation = mock(AbfsRestOperation.class); + when(mockRestOperation.isARetriedRequest()).thenReturn(true); + + // Case 1: Mock instance of Http Operation response. This will return + // HTTP:Not Found + AbfsHttpOperation httpOperation = mock(AbfsHttpOperation.class); + when(httpOperation.getStatusCode()).thenReturn(HTTP_INTERNAL_ERROR); + + // Mock delete response to 404 + when(mockRestOperation.getResult()).thenReturn(httpOperation); + when(mockRestOperation.hasResult()).thenReturn(true); + TracingContext tracingContext = new TracingContext("abcd", + fs.getFileSystemId(), FSOperationType.TEST_OP, + org.apache.hadoop.fs.azurebfs.utils.TracingHeaderFormat.ALL_ID_FORMAT, null); + fs.close(); + } + @Test public void testDeleteIdempotencyTriggerHttp404() throws Exception { From 9230bd8792968a732a8baabc4a650a27c0dce475 Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Wed, 13 Jul 2022 16:28:55 +0530 Subject: [PATCH 19/42] Add test case for retry --- .../hadoop/fs/azurebfs/AbfsDriverMetrics.java | 48 +++++++++++-------- .../fs/azurebfs/AzureBlobFileSystem.java | 2 - .../fs/azurebfs/services/AbfsClient.java | 2 +- .../azurebfs/services/AbfsRestOperation.java | 10 ++-- .../ITestAzureBlobFileSystemDelete.java | 35 +++++++++++++- .../fs/azurebfs/TestAbfsRestOperation.java | 41 ++++++++++++++++ 6 files changed, 109 insertions(+), 29 deletions(-) create mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestAbfsRestOperation.java diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java index 029b0eff56ea1..1826f1bd0f210 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java @@ -33,6 +33,7 @@ public class AbfsDriverMetrics { private AtomicLong numberOfIOPSThrottledRequests; private AtomicLong numberOfBandwidthThrottledRequests; private AtomicLong numberOfOtherThrottledRequests; + private AtomicLong numberOfNetworkFailedRequests; private AtomicLong maxRetryCount; private AtomicLong totalNumberOfRequests; private AtomicLong numberOfRequestsSucceededWithoutRetrying; @@ -48,6 +49,7 @@ public AbfsDriverMetrics() { this.maxRetryCount = new AtomicLong(); this.numberOfRequestsSucceededWithoutRetrying = new AtomicLong(); this.numberOfRequestsFailed = new AtomicLong(); + this.numberOfNetworkFailedRequests = new AtomicLong(); } public AbfsDriverMetrics(String retryCount) { this.retryCount = retryCount; @@ -59,7 +61,7 @@ public AbfsDriverMetrics(String retryCount) { } private void initializeMap() { - ArrayList retryCountList = new ArrayList(Arrays.asList("1","2","3","4","5_15","15_25","25andabove")); + ArrayList retryCountList = new ArrayList(Arrays.asList("1","2","3","4","5_15","15_25","25AndAbove")); for (String s : retryCountList) { metricsMap.put(s, new AbfsDriverMetrics(s)); } @@ -121,20 +123,25 @@ public AtomicLong getNumberOfRequestsFailed() { return numberOfRequestsFailed; } + public AtomicLong getNumberOfNetworkFailedRequests() { + return numberOfNetworkFailedRequests; + } + /* - Acronyms :- - 1.RCTSI :- Request count that succeeded in x retries - 2.MMA :- Min Max Average (This refers to the backoff or sleep time between 2 requests) - 3.s :- seconds - 4.BWT :- Number of Bandwidth throttled requests - 5.IT :- Number of IOPS throttled requests - 6.OT :- Number of Other throttled requests - 7.%RT :- Percentage of requests that are throttled - 8.TRNR :- Total number of requests which succeeded without retrying - 9.TRF :- Total number of requests which failed - 10.TR :- Total number of requests which were made - 11.MRC :- Max retry count across all requests - */ + Acronyms :- + 1.RCTSI :- Request count that succeeded in x retries + 2.MMA :- Min Max Average (This refers to the backoff or sleep time between 2 requests) + 3.s :- seconds + 4.BWT :- Number of Bandwidth throttled requests + 5.IT :- Number of IOPS throttled requests + 6.OT :- Number of Other throttled requests + 7.NFR :- Number of requests which failed due to network errors + 7.%RT :- Percentage of requests that are throttled + 8.TRNR :- Total number of requests which succeeded without retrying + 9.TRF :- Total number of requests which failed + 10.TR :- Total number of requests which were made + 11.MRC :- Max retry count across all requests + */ @Override public String toString() { StringBuilder metricString = new StringBuilder(); @@ -161,12 +168,13 @@ public String toString() { } metricString.append("#BWT=").append(numberOfBandwidthThrottledRequests) .append(" #IT=").append(numberOfIOPSThrottledRequests) - .append(" #OT=").append(numberOfOtherThrottledRequests); - metricString.append(" #%RT=").append(String.format("%.3f",percentageOfRequestsThrottled)); - metricString.append(" #TRNR=").append(numberOfRequestsSucceededWithoutRetrying); - metricString.append(" #TRF=").append(numberOfRequestsFailed); - metricString.append(" #TR=").append(totalNumberOfRequests); - metricString.append(" #MRC=").append(maxRetryCount); + .append(" #OT=").append(numberOfOtherThrottledRequests) + .append(" #%RT=").append(String.format("%.3f",percentageOfRequestsThrottled)) + .append( "#NFR").append(numberOfNetworkFailedRequests) + .append(" #TRNR=").append(numberOfRequestsSucceededWithoutRetrying) + .append(" #TRF=").append(numberOfRequestsFailed) + .append(" #TR=").append(totalNumberOfRequests) + .append(" #MRC=").append(maxRetryCount); return metricString + " "; } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java index 9b3e458656796..52ca49213addb 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java @@ -118,11 +118,9 @@ import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_BLOCK_UPLOAD_ACTIVE_BLOCKS; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_BLOCK_UPLOAD_BUFFER_DIR; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_METRIC_ACCOUNT_KEY; -import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_METRIC_ACCOUNT_NAME; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_METRIC_URI; import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.BLOCK_UPLOAD_ACTIVE_BLOCKS_DEFAULT; import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DATA_BLOCKS_BUFFER_DEFAULT; -import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import static org.apache.hadoop.fs.impl.PathCapabilitiesSupport.validatePathCapabilityArgs; import static org.apache.hadoop.fs.statistics.IOStatisticsLogging.logIOStatisticsAtLevel; import static org.apache.hadoop.util.functional.RemoteIterators.filteringRemoteIterator; diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java index 3b351a53205c5..c2e698cb63790 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java @@ -1220,7 +1220,7 @@ public SASTokenProvider getSasTokenProvider() { * Getter for abfsCounters from AbfsClient. * @return AbfsCounters instance. */ - protected AbfsCounters getAbfsCounters() { + public AbfsCounters getAbfsCounters() { return abfsCounters; } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java index d8ca9db178edb..580b09e0ceb15 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java @@ -256,7 +256,8 @@ private void completeExecute(TracingContext tracingContext) } private synchronized void updateDriverMetrics(int retryCount, int statusCode){ - if (statusCode == HttpURLConnection.HTTP_UNAVAILABLE) { + if(statusCode < HttpURLConnection.HTTP_OK + || statusCode >= HttpURLConnection.HTTP_INTERNAL_ERROR) { if (retryCount >= maxIoRetries) { abfsDriverMetrics.getNumberOfRequestsFailed().getAndIncrement(); } @@ -325,7 +326,7 @@ private boolean executeHttpOperation(final int retryCount, } httpOperation.processResponse(buffer, bufferOffset, bufferLength); - if(!isThrottledRequest && httpOperation.getStatusCode() == HttpURLConnection.HTTP_UNAVAILABLE){ + if(!isThrottledRequest && httpOperation.getStatusCode() >= HttpURLConnection.HTTP_INTERNAL_ERROR){ isThrottledRequest = true; AzureServiceErrorCode serviceErrorCode = AzureServiceErrorCode.getAzureServiceCode(httpOperation.getStatusCode(), httpOperation.getStorageErrorCode(), httpOperation.getStorageErrorMessage()); @@ -354,6 +355,7 @@ private boolean executeHttpOperation(final int retryCount, hostname = httpOperation.getHost(); LOG.warn("Unknown host name: {}. Retrying to resolve the host name...", hostname); + abfsDriverMetrics.getNumberOfRequestsFailed().getAndIncrement(); if (!client.getRetryPolicy().shouldRetry(retryCount, -1)) { updateDriverMetrics(retryCount, httpOperation.getStatusCode()); throw new InvalidAbfsRestOperationException(ex); @@ -363,7 +365,7 @@ private boolean executeHttpOperation(final int retryCount, if (LOG.isDebugEnabled()) { LOG.debug("HttpRequestFailure: {}, {}", httpOperation, ex); } - + abfsDriverMetrics.getNumberOfNetworkFailedRequests().getAndIncrement(); if (!client.getRetryPolicy().shouldRetry(retryCount, -1)) { updateDriverMetrics(retryCount, httpOperation.getStatusCode()); throw new InvalidAbfsRestOperationException(ex); @@ -423,7 +425,7 @@ private String getKey(int retryCount) { }else if(retryCount >= 15 && retryCount < 25){ retryCounter = "15_25"; }else{ - retryCounter = "25andabove"; + retryCounter = "25AndAbove"; } return retryCounter; } diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelete.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelete.java index db181fb5dd660..fc4059d3d7908 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelete.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelete.java @@ -45,10 +45,12 @@ import org.apache.hadoop.fs.Path; import static java.net.HttpURLConnection.HTTP_BAD_REQUEST; +import static java.net.HttpURLConnection.HTTP_INTERNAL_ERROR; import static java.net.HttpURLConnection.HTTP_NOT_FOUND; import static java.net.HttpURLConnection.HTTP_OK; - +import org.mockito.Mockito; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.doCallRealMethod; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; @@ -61,7 +63,6 @@ import static org.apache.hadoop.fs.contract.ContractTestUtils.assertPathDoesNotExist; import static org.apache.hadoop.test.LambdaTestUtils.intercept; - /** * Test delete operation. */ @@ -219,6 +220,36 @@ public void testDeleteIdempotency() throws Exception { } + @Test + public void testTryStatus() throws java.io.IOException { + AbfsConfiguration abfsConfig + = TestAbfsConfigurationFieldsValidation.updateRetryConfigs( + getConfiguration(), + 5, 5000); + + final AzureBlobFileSystem fs = this.getFileSystem(); + AbfsClient abfsClient = fs.getAbfsStore().getClient(); + AbfsClient testClient = TestAbfsClient.createTestClientFromCurrentContext( + abfsClient, + abfsConfig); + + final AbfsRestOperation mockRestOperation = mock(AbfsRestOperation.class); + when(mockRestOperation.isARetriedRequest()).thenReturn(true); + + // Case 1: Mock instance of Http Operation response. This will return + // HTTP:Not Found + AbfsHttpOperation httpOperation = mock(AbfsHttpOperation.class); + when(httpOperation.getStatusCode()).thenReturn(HTTP_INTERNAL_ERROR); + + // Mock delete response to 404 + when(mockRestOperation.getResult()).thenReturn(httpOperation); + when(mockRestOperation.hasResult()).thenReturn(true); + TracingContext tracingContext = new TracingContext("abcd", + fs.getFileSystemId(), FSOperationType.TEST_OP, + org.apache.hadoop.fs.azurebfs.utils.TracingHeaderFormat.ALL_ID_FORMAT, null); + fs.close(); + } + @Test public void testDeleteIdempotencyTriggerHttp404() throws Exception { diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestAbfsRestOperation.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestAbfsRestOperation.java new file mode 100644 index 0000000000000..39017b5770500 --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestAbfsRestOperation.java @@ -0,0 +1,41 @@ +package org.apache.hadoop.fs.azurebfs; + +import org.junit.Test; +import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.HTTP_METHOD_DELETE; +import static org.apache.hadoop.fs.azurebfs.services.AbfsRestOperationType.DeletePath; +import org.apache.hadoop.fs.azurebfs.services.AbfsClient; +import org.apache.hadoop.fs.azurebfs.services.TestAbfsClient; +import java.lang.reflect.Method; +import org.apache.hadoop.fs.azurebfs.services.AbfsRestOperation; +import java.util.ArrayList; +import java.util.Arrays; +import org.junit.Assert; +import java.net.HttpURLConnection; + +public class TestAbfsRestOperation extends + AbstractAbfsIntegrationTest { + + public TestAbfsRestOperation() throws Exception { + } + + @Test + public void testDriverRetryMetrics() throws Exception { + final AzureBlobFileSystem fs = getFileSystem(); + AbfsClient client = fs.getAbfsClient(); + + // Mock instance of AbfsRestOperation + AbfsRestOperation op = TestAbfsClient.getRestOp( + DeletePath, client, HTTP_METHOD_DELETE, + TestAbfsClient.getTestUrl(client, "/NonExistingPath"), TestAbfsClient.getTestRequestHeaders(client)); + + ArrayList retryCounts = new ArrayList<>(Arrays.asList(35, 28, 31, 45, 10, 2, 9)); + int statusCode = HttpURLConnection.HTTP_UNAVAILABLE; + Method getMetrics = AbfsRestOperation.class.getDeclaredMethod("updateDriverMetrics", int.class, int.class); + getMetrics.setAccessible(true); + for(int retryCount: retryCounts) { + getMetrics.invoke(op, retryCount, statusCode); + } + Assert.assertEquals(client.getAbfsCounters().getAbfsDriverMetrics().getNumberOfRequestsFailed().toString(), "3"); + fs.close(); + } +} From e1c4c618bde21627e31f29d0bdd2e652a2c9459d Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Thu, 14 Jul 2022 09:31:04 +0530 Subject: [PATCH 20/42] Refactored the test class --- .../hadoop/fs/azurebfs/services/AbfsClient.java | 2 +- .../{ => services}/TestAbfsRestOperation.java | 15 +++++++-------- 2 files changed, 8 insertions(+), 9 deletions(-) rename hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/{ => services}/TestAbfsRestOperation.java (67%) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java index c2e698cb63790..3b351a53205c5 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java @@ -1220,7 +1220,7 @@ public SASTokenProvider getSasTokenProvider() { * Getter for abfsCounters from AbfsClient. * @return AbfsCounters instance. */ - public AbfsCounters getAbfsCounters() { + protected AbfsCounters getAbfsCounters() { return abfsCounters; } diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestAbfsRestOperation.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsRestOperation.java similarity index 67% rename from hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestAbfsRestOperation.java rename to hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsRestOperation.java index 39017b5770500..dea6c3f4edd92 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/TestAbfsRestOperation.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsRestOperation.java @@ -1,12 +1,11 @@ -package org.apache.hadoop.fs.azurebfs; +package org.apache.hadoop.fs.azurebfs.services; import org.junit.Test; import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.HTTP_METHOD_DELETE; import static org.apache.hadoop.fs.azurebfs.services.AbfsRestOperationType.DeletePath; -import org.apache.hadoop.fs.azurebfs.services.AbfsClient; -import org.apache.hadoop.fs.azurebfs.services.TestAbfsClient; import java.lang.reflect.Method; -import org.apache.hadoop.fs.azurebfs.services.AbfsRestOperation; +import org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem; +import org.apache.hadoop.fs.azurebfs.AbstractAbfsIntegrationTest; import java.util.ArrayList; import java.util.Arrays; import org.junit.Assert; @@ -21,12 +20,12 @@ public TestAbfsRestOperation() throws Exception { @Test public void testDriverRetryMetrics() throws Exception { final AzureBlobFileSystem fs = getFileSystem(); - AbfsClient client = fs.getAbfsClient(); + AbfsClient testClient = super.getAbfsClient(super.getAbfsStore(getFileSystem())); // Mock instance of AbfsRestOperation AbfsRestOperation op = TestAbfsClient.getRestOp( - DeletePath, client, HTTP_METHOD_DELETE, - TestAbfsClient.getTestUrl(client, "/NonExistingPath"), TestAbfsClient.getTestRequestHeaders(client)); + DeletePath, testClient, HTTP_METHOD_DELETE, + TestAbfsClient.getTestUrl(testClient, "/NonExistingPath"), TestAbfsClient.getTestRequestHeaders(testClient)); ArrayList retryCounts = new ArrayList<>(Arrays.asList(35, 28, 31, 45, 10, 2, 9)); int statusCode = HttpURLConnection.HTTP_UNAVAILABLE; @@ -35,7 +34,7 @@ public void testDriverRetryMetrics() throws Exception { for(int retryCount: retryCounts) { getMetrics.invoke(op, retryCount, statusCode); } - Assert.assertEquals(client.getAbfsCounters().getAbfsDriverMetrics().getNumberOfRequestsFailed().toString(), "3"); + Assert.assertEquals(testClient.getAbfsCounters().getAbfsDriverMetrics().getNumberOfRequestsFailed().toString(), "3"); fs.close(); } } From 7686ccee13024ba47638a26847d9e8caa2777ff8 Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Thu, 14 Jul 2022 11:42:23 +0530 Subject: [PATCH 21/42] Added comments --- .../hadoop/fs/azurebfs/services/TestAbfsRestOperation.java | 1 + 1 file changed, 1 insertion(+) diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsRestOperation.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsRestOperation.java index dea6c3f4edd92..54c72f966e29c 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsRestOperation.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsRestOperation.java @@ -34,6 +34,7 @@ public void testDriverRetryMetrics() throws Exception { for(int retryCount: retryCounts) { getMetrics.invoke(op, retryCount, statusCode); } + //For retry count greater than max configured value, the request should fail Assert.assertEquals(testClient.getAbfsCounters().getAbfsDriverMetrics().getNumberOfRequestsFailed().toString(), "3"); fs.close(); } From f66bfe396ccd10ade811d44db15919899a132f1b Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Thu, 14 Jul 2022 12:16:24 +0530 Subject: [PATCH 22/42] Back to previous configs --- .../hadoop/fs/azurebfs/constants/FileSystemConfigurations.java | 2 +- .../apache/hadoop/fs/azurebfs/services/AbfsHttpOperation.java | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java index 3d7263790a0fe..227e1d6810b8b 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java @@ -38,7 +38,7 @@ public final class FileSystemConfigurations { private static final int SIXTY_SECONDS = 60 * 1000; // Retry parameter defaults. - public static final int DEFAULT_MIN_BACKOFF_INTERVAL = 25 * 10; // 3s + public static final int DEFAULT_MIN_BACKOFF_INTERVAL = 3 * 1000; // 3s public static final int DEFAULT_MAX_BACKOFF_INTERVAL = 30 * 1000; // 30s public static final int DEFAULT_BACKOFF_INTERVAL = 3 * 1000; // 3s public static final int DEFAULT_MAX_RETRY_ATTEMPTS = 30; diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsHttpOperation.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsHttpOperation.java index 0cfbf03106227..6d9110dca03aa 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsHttpOperation.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsHttpOperation.java @@ -345,7 +345,6 @@ public void processResponse(final byte[] buffer, final int offset, final int len } this.statusCode = this.connection.getResponseCode(); - //this.statusCode = 503; if (this.isTraceEnabled) { this.recvResponseTimeMs = elapsedTimeMs(startTime); } From d0c0f68007d851b345af61864ebf3482e7197b69 Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Thu, 14 Jul 2022 12:22:51 +0530 Subject: [PATCH 23/42] remove test case --- .../ITestAzureBlobFileSystemDelete.java | 30 ------------------- 1 file changed, 30 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelete.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelete.java index fc4059d3d7908..5562f08a7bcd0 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelete.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelete.java @@ -220,36 +220,6 @@ public void testDeleteIdempotency() throws Exception { } - @Test - public void testTryStatus() throws java.io.IOException { - AbfsConfiguration abfsConfig - = TestAbfsConfigurationFieldsValidation.updateRetryConfigs( - getConfiguration(), - 5, 5000); - - final AzureBlobFileSystem fs = this.getFileSystem(); - AbfsClient abfsClient = fs.getAbfsStore().getClient(); - AbfsClient testClient = TestAbfsClient.createTestClientFromCurrentContext( - abfsClient, - abfsConfig); - - final AbfsRestOperation mockRestOperation = mock(AbfsRestOperation.class); - when(mockRestOperation.isARetriedRequest()).thenReturn(true); - - // Case 1: Mock instance of Http Operation response. This will return - // HTTP:Not Found - AbfsHttpOperation httpOperation = mock(AbfsHttpOperation.class); - when(httpOperation.getStatusCode()).thenReturn(HTTP_INTERNAL_ERROR); - - // Mock delete response to 404 - when(mockRestOperation.getResult()).thenReturn(httpOperation); - when(mockRestOperation.hasResult()).thenReturn(true); - TracingContext tracingContext = new TracingContext("abcd", - fs.getFileSystemId(), FSOperationType.TEST_OP, - org.apache.hadoop.fs.azurebfs.utils.TracingHeaderFormat.ALL_ID_FORMAT, null); - fs.close(); - } - @Test public void testDeleteIdempotencyTriggerHttp404() throws Exception { From 637060f986966bd0e2f41280cd99311d27294e7b Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Thu, 14 Jul 2022 12:24:11 +0530 Subject: [PATCH 24/42] remove test case --- .../hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelete.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelete.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelete.java index 5562f08a7bcd0..e3adf1f98ef77 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelete.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelete.java @@ -45,12 +45,9 @@ import org.apache.hadoop.fs.Path; import static java.net.HttpURLConnection.HTTP_BAD_REQUEST; -import static java.net.HttpURLConnection.HTTP_INTERNAL_ERROR; import static java.net.HttpURLConnection.HTTP_NOT_FOUND; import static java.net.HttpURLConnection.HTTP_OK; -import org.mockito.Mockito; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.doCallRealMethod; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; From a4774007b88ed879a257ed95e048398ec17883f9 Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Thu, 14 Jul 2022 12:26:10 +0530 Subject: [PATCH 25/42] remove test case --- .../hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelete.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelete.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelete.java index e3adf1f98ef77..db181fb5dd660 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelete.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelete.java @@ -47,6 +47,7 @@ import static java.net.HttpURLConnection.HTTP_BAD_REQUEST; import static java.net.HttpURLConnection.HTTP_NOT_FOUND; import static java.net.HttpURLConnection.HTTP_OK; + import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doCallRealMethod; import static org.mockito.Mockito.doReturn; @@ -60,6 +61,7 @@ import static org.apache.hadoop.fs.contract.ContractTestUtils.assertPathDoesNotExist; import static org.apache.hadoop.test.LambdaTestUtils.intercept; + /** * Test delete operation. */ From 0f44a9bf4c38d473accb3a4e188f24063bdea831 Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Fri, 15 Jul 2022 09:57:05 +0530 Subject: [PATCH 26/42] Fix for checkstyle errors --- .../hadoop/fs/azurebfs/AbfsDriverMetrics.java | 71 ++++++++++++++----- .../fs/azurebfs/AzureBlobFileSystem.java | 20 ++++-- .../services/AzureServiceErrorCode.java | 7 +- .../azurebfs/services/AbfsRestOperation.java | 57 +++++++++------ .../hadoop-azure/src/site/markdown/abfs.md | 19 +++-- .../services/TestAbfsRestOperation.java | 20 +++++- 6 files changed, 137 insertions(+), 57 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java index 1826f1bd0f210..19c9f40bdcf33 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java @@ -23,22 +23,39 @@ import java.util.Map; import java.util.ArrayList; import java.util.Arrays; + public class AbfsDriverMetrics { + private AtomicLong numberOfRequestsSucceeded; + private AtomicLong minBackoff; + private AtomicLong maxBackoff; + private AtomicLong totalRequests; + private AtomicLong totalBackoff; + private String retryCount; + private AtomicLong numberOfIOPSThrottledRequests; + private AtomicLong numberOfBandwidthThrottledRequests; + private AtomicLong numberOfOtherThrottledRequests; + private AtomicLong numberOfNetworkFailedRequests; + private AtomicLong maxRetryCount; + private AtomicLong totalNumberOfRequests; + private AtomicLong numberOfRequestsSucceededWithoutRetrying; + private AtomicLong numberOfRequestsFailed; - private final Map metricsMap = new ConcurrentHashMap<>(); + + private final Map metricsMap + = new ConcurrentHashMap<>(); public AbfsDriverMetrics() { initializeMap(); @@ -51,6 +68,7 @@ public AbfsDriverMetrics() { this.numberOfRequestsFailed = new AtomicLong(); this.numberOfNetworkFailedRequests = new AtomicLong(); } + public AbfsDriverMetrics(String retryCount) { this.retryCount = retryCount; this.numberOfRequestsSucceeded = new AtomicLong(); @@ -61,7 +79,8 @@ public AbfsDriverMetrics(String retryCount) { } private void initializeMap() { - ArrayList retryCountList = new ArrayList(Arrays.asList("1","2","3","4","5_15","15_25","25AndAbove")); + ArrayList retryCountList = new ArrayList( + Arrays.asList("1", "2", "3", "4", "5_15", "15_25", "25AndAbove")); for (String s : retryCountList) { metricsMap.put(s, new AbfsDriverMetrics(s)); } @@ -145,36 +164,52 @@ public AtomicLong getNumberOfNetworkFailedRequests() { @Override public String toString() { StringBuilder metricString = new StringBuilder(); - long totalRequestsThrottled = numberOfBandwidthThrottledRequests.get() + numberOfIOPSThrottledRequests.get() + numberOfOtherThrottledRequests.get(); - double percentageOfRequestsThrottled = ((double)totalRequestsThrottled/totalNumberOfRequests.get())*100; + long totalRequestsThrottled = numberOfBandwidthThrottledRequests.get() + + numberOfIOPSThrottledRequests.get() + + numberOfOtherThrottledRequests.get(); + double percentageOfRequestsThrottled = + ((double) totalRequestsThrottled / totalNumberOfRequests.get()) * 100; for (Map.Entry entry : metricsMap.entrySet()) { metricString.append("#RCTSI#_").append(entry.getKey()) .append("R_").append("=") .append(entry.getValue().getNumberOfRequestsSucceeded()).append(" "); long totalRequests = entry.getValue().getTotalRequests().get(); - if(totalRequests > 0) { + if (totalRequests > 0) { metricString.append("#MMA#_").append(entry.getKey()) .append("R_").append("=") - .append(String.format("%.3f", (double) entry.getValue().getMinBackoff().get() / 1000L)) + .append(String.format("%.3f", + (double) entry.getValue().getMinBackoff().get() / 1000L)) .append("s ") - .append(String.format("%.3f", (double) entry.getValue().getMaxBackoff().get() / 1000L)) + .append(String.format("%.3f", + (double) entry.getValue().getMaxBackoff().get() / 1000L)) .append("s ") - .append(String.format("%.3f", (double) ((entry.getValue().getTotalBackoff().get() / totalRequests) / 1000L))) + .append(String.format("%.3f", (double) ( + (entry.getValue().getTotalBackoff().get() / totalRequests) + / 1000L))) .append("s "); - }else { + } else { metricString.append("#MMA#_").append(entry.getKey()) .append("R_").append("=0s "); } } - metricString.append("#BWT=").append(numberOfBandwidthThrottledRequests) - .append(" #IT=").append(numberOfIOPSThrottledRequests) - .append(" #OT=").append(numberOfOtherThrottledRequests) - .append(" #%RT=").append(String.format("%.3f",percentageOfRequestsThrottled)) - .append( "#NFR").append(numberOfNetworkFailedRequests) - .append(" #TRNR=").append(numberOfRequestsSucceededWithoutRetrying) - .append(" #TRF=").append(numberOfRequestsFailed) - .append(" #TR=").append(totalNumberOfRequests) - .append(" #MRC=").append(maxRetryCount); + metricString.append("#BWT=") + .append(numberOfBandwidthThrottledRequests) + .append(" #IT=") + .append(numberOfIOPSThrottledRequests) + .append(" #OT=") + .append(numberOfOtherThrottledRequests) + .append(" #%RT=") + .append(String.format("%.3f", percentageOfRequestsThrottled)) + .append("#NFR") + .append(numberOfNetworkFailedRequests) + .append(" #TRNR=") + .append(numberOfRequestsSucceededWithoutRetrying) + .append(" #TRF=") + .append(numberOfRequestsFailed) + .append(" #TR=") + .append(totalNumberOfRequests) + .append(" #MRC=") + .append(maxRetryCount); return metricString + " "; } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java index 87da314329448..2ff453f8d8780 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java @@ -685,24 +685,30 @@ public synchronized void close() throws IOException { } String metric = abfsCounters.getAbfsDriverMetrics().toString(); LOG.debug("The metrics collected over this instance are " + metric); - if(sendMetricsToStore) { - if(abfsCounters.getAbfsDriverMetrics().getTotalNumberOfRequests().get() > 0) { + if (sendMetricsToStore) { + if (abfsCounters.getAbfsDriverMetrics().getTotalNumberOfRequests().get() + > 0) { try { Configuration metricConfig = getConf(); - String metricAccountKey = metricConfig.get(FS_AZURE_METRIC_ACCOUNT_KEY); + String metricAccountKey = metricConfig.get( + FS_AZURE_METRIC_ACCOUNT_KEY); final String abfsMetricUrl = metricConfig.get(FS_AZURE_METRIC_URI); - if(abfsMetricUrl == null){ + if (abfsMetricUrl == null) { return; } - metricConfig.set(FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME, metricAccountKey); - metricConfig.set(AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION, "false"); + metricConfig.set(FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME, + metricAccountKey); + metricConfig.set(AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION, + "false"); URI metricUri; try { metricUri = new URI(getScheme(), abfsMetricUrl, null, null, null); } catch (URISyntaxException ex) { throw new AssertionError(ex); } - AzureBlobFileSystem metricFs = (AzureBlobFileSystem) FileSystem.newInstance(metricUri, metricConfig); + AzureBlobFileSystem metricFs + = (AzureBlobFileSystem) FileSystem.newInstance(metricUri, + metricConfig); metricFs.sendMetric(metric); } catch (AzureBlobFileSystemException ex) { //do nothing diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/services/AzureServiceErrorCode.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/services/AzureServiceErrorCode.java index ce53b1fd1bfe1..262d37ec63d4e 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/services/AzureServiceErrorCode.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/services/AzureServiceErrorCode.java @@ -113,10 +113,11 @@ public static AzureServiceErrorCode getAzureServiceCode(int httpStatusCode, Stri for (AzureServiceErrorCode azureServiceErrorCode : AzureServiceErrorCode.values()) { if (azureServiceErrorCode.getStatusCode() == httpStatusCode && azureServiceErrorCode.getErrorCode().equalsIgnoreCase(errorCode) && - azureServiceErrorCode.getErrorMessage().equalsIgnoreCase(errorMessages[0])) { - return azureServiceErrorCode; - } + azureServiceErrorCode.getErrorMessage() + .equalsIgnoreCase(errorMessages[0])) { + return azureServiceErrorCode; } + } return UNKNOWN; } } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java index 580b09e0ceb15..0c4eb1f0b19bd 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java @@ -151,9 +151,10 @@ String getSasToken() { || AbfsHttpConstants.HTTP_METHOD_PATCH.equals(method)); this.sasToken = sasToken; this.abfsCounters = client.getAbfsCounters(); - if(abfsCounters != null) { + if (abfsCounters != null) { this.abfsDriverMetrics = abfsCounters.getAbfsDriverMetrics(); - }if(abfsDriverMetrics != null) { + } + if (abfsDriverMetrics != null) { this.metricsMap = abfsDriverMetrics.getMetricsMap(); } this.maxIoRetries = client.getAbfsConfiguration().getMaxIoRetries(); @@ -226,7 +227,7 @@ private void completeExecute(TracingContext tracingContext) retryCount = 0; LOG.debug("First execution of REST operation - {}", operationType); long sleepDuration = 0L; - if(abfsDriverMetrics != null) { + if (abfsDriverMetrics != null) { abfsDriverMetrics.getTotalNumberOfRequests().getAndIncrement(); } while (!executeHttpOperation(retryCount, tracingContext)) { @@ -244,7 +245,7 @@ private void completeExecute(TracingContext tracingContext) Thread.currentThread().interrupt(); } } - if(abfsDriverMetrics != null) { + if (abfsDriverMetrics != null) { updateDriverMetrics(retryCount, result.getStatusCode()); } if (result.getStatusCode() >= HttpURLConnection.HTTP_BAD_REQUEST) { @@ -256,18 +257,20 @@ private void completeExecute(TracingContext tracingContext) } private synchronized void updateDriverMetrics(int retryCount, int statusCode){ - if(statusCode < HttpURLConnection.HTTP_OK + if (statusCode < HttpURLConnection.HTTP_OK || statusCode >= HttpURLConnection.HTTP_INTERNAL_ERROR) { if (retryCount >= maxIoRetries) { abfsDriverMetrics.getNumberOfRequestsFailed().getAndIncrement(); } } else { if (retryCount > 0 && retryCount <= maxIoRetries) { - maxRetryCount = Math.max(abfsDriverMetrics.getMaxRetryCount().get(), retryCount); + maxRetryCount = Math.max(abfsDriverMetrics.getMaxRetryCount().get(), + retryCount); abfsDriverMetrics.getMaxRetryCount().set(maxRetryCount); updateCount(retryCount); } else { - abfsDriverMetrics.getNumberOfRequestsSucceededWithoutRetrying().getAndIncrement(); + abfsDriverMetrics.getNumberOfRequestsSucceededWithoutRetrying() + .getAndIncrement(); } } } @@ -326,19 +329,31 @@ private boolean executeHttpOperation(final int retryCount, } httpOperation.processResponse(buffer, bufferOffset, bufferLength); - if(!isThrottledRequest && httpOperation.getStatusCode() >= HttpURLConnection.HTTP_INTERNAL_ERROR){ + if (!isThrottledRequest && httpOperation.getStatusCode() + >= HttpURLConnection.HTTP_INTERNAL_ERROR) { isThrottledRequest = true; AzureServiceErrorCode serviceErrorCode = - AzureServiceErrorCode.getAzureServiceCode(httpOperation.getStatusCode(), httpOperation.getStorageErrorCode(), httpOperation.getStorageErrorMessage()); - LOG1.trace("Service code is " + serviceErrorCode + " status code is " + httpOperation.getStatusCode() + " error code is " + httpOperation.getStorageErrorCode() + AzureServiceErrorCode.getAzureServiceCode( + httpOperation.getStatusCode(), + httpOperation.getStorageErrorCode(), + httpOperation.getStorageErrorMessage()); + LOG1.trace("Service code is " + serviceErrorCode + " status code is " + + httpOperation.getStatusCode() + " error code is " + + httpOperation.getStorageErrorCode() + " error message is " + httpOperation.getStorageErrorMessage()); - if(serviceErrorCode.equals(AzureServiceErrorCode.INGRESS_OVER_ACCOUNT_LIMIT) || - serviceErrorCode.equals(AzureServiceErrorCode.EGRESS_OVER_ACCOUNT_LIMIT)){ - abfsDriverMetrics.getNumberOfBandwidthThrottledRequests().getAndIncrement(); - }else if(serviceErrorCode.equals(AzureServiceErrorCode.REQUEST_OVER_ACCOUNT_LIMIT)){ - abfsDriverMetrics.getNumberOfIOPSThrottledRequests().getAndIncrement(); - }else{ - abfsDriverMetrics.getNumberOfOtherThrottledRequests().getAndIncrement(); + if (serviceErrorCode.equals( + AzureServiceErrorCode.INGRESS_OVER_ACCOUNT_LIMIT) || + serviceErrorCode.equals( + AzureServiceErrorCode.EGRESS_OVER_ACCOUNT_LIMIT)) { + abfsDriverMetrics.getNumberOfBandwidthThrottledRequests() + .getAndIncrement(); + } else if (serviceErrorCode.equals( + AzureServiceErrorCode.REQUEST_OVER_ACCOUNT_LIMIT)) { + abfsDriverMetrics.getNumberOfIOPSThrottledRequests() + .getAndIncrement(); + } else { + abfsDriverMetrics.getNumberOfOtherThrottledRequests() + .getAndIncrement(); } } incrementCounter(AbfsStatistic.GET_RESPONSES, 1); @@ -418,13 +433,13 @@ private void updateTimeMetrics(int retryCount, long sleepDuration){ private String getKey(int retryCount) { String retryCounter; - if(retryCount >= 1 && retryCount <= 4){ + if (retryCount >= 1 && retryCount <= 4) { retryCounter = Integer.toString(retryCount); - }else if(retryCount >= 5 && retryCount < 15){ + } else if (retryCount >= 5 && retryCount < 15) { retryCounter = "5_15"; - }else if(retryCount >= 15 && retryCount < 25){ + } else if (retryCount >= 15 && retryCount < 25) { retryCounter = "15_25"; - }else{ + } else { retryCounter = "25AndAbove"; } return retryCounter; diff --git a/hadoop-tools/hadoop-azure/src/site/markdown/abfs.md b/hadoop-tools/hadoop-azure/src/site/markdown/abfs.md index 12e2bc6a5d7c6..7250e6893a0ef 100644 --- a/hadoop-tools/hadoop-azure/src/site/markdown/abfs.md +++ b/hadoop-tools/hadoop-azure/src/site/markdown/abfs.md @@ -993,18 +993,23 @@ settings to track their end-to-end latency. ### Driver Metric Options -`fs.azure.metric.account.name`: This configuration parameter is used to specify -the name of the account which will be used to push a failed(404) GetPathStatus request to the -backend. We can configure a separate account to push metrics to the store or use the same for as the existing account on which other requests are made. +`fs.azure.metric.account.name`: This configuration parameter is used to specify +the name of the account which will be used to push a failed(404) GetPathStatus +request to the +backend. We can configure a separate account to push metrics to the store or use +the same for as the existing account on which other requests are made. -`fs.azure.metric.account.key`: This is the access key for the storage account +`fs.azure.metric.account.key`: This is the access key for the storage account used for pushing metrics to the store. -`fs.azure.enable.metric.collection`: This configuration provides an option to specify whether we want to push the metrics or not. +`fs.azure.enable.metric.collection`: This configuration provides an option to +specify whether we want to push the metrics or not. By default, this config will be set to false. -`fs.azure.metric.uri`: This configuration provides the uri in the format of containername@accountname.dfs.core.windows.net. -This should be a part of the config in order to prevent extra calls to create the filesystem. We use an existing filsystem to push the metrics. +`fs.azure.metric.uri`: This configuration provides the uri in the format of +containername@accountname.dfs.core.windows.net. +This should be a part of the config in order to prevent extra calls to create +the filesystem. We use an existing filsystem to push the metrics. ## Troubleshooting diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsRestOperation.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsRestOperation.java index 54c72f966e29c..1de3d97d0aaaf 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsRestOperation.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsRestOperation.java @@ -1,3 +1,21 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.hadoop.fs.azurebfs.services; import org.junit.Test; @@ -31,7 +49,7 @@ public void testDriverRetryMetrics() throws Exception { int statusCode = HttpURLConnection.HTTP_UNAVAILABLE; Method getMetrics = AbfsRestOperation.class.getDeclaredMethod("updateDriverMetrics", int.class, int.class); getMetrics.setAccessible(true); - for(int retryCount: retryCounts) { + for (int retryCount : retryCounts) { getMetrics.invoke(op, retryCount, statusCode); } //For retry count greater than max configured value, the request should fail From 312853cb4395e27b6112f8002943395e97765dd5 Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Mon, 18 Jul 2022 10:34:26 +0530 Subject: [PATCH 27/42] Checkstyle error fixes --- .../contracts/services/AzureServiceErrorCode.java | 4 ++-- .../fs/azurebfs/services/AbfsRestOperation.java | 11 ++++------- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/services/AzureServiceErrorCode.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/services/AzureServiceErrorCode.java index 262d37ec63d4e..7a4ffaca4ead7 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/services/AzureServiceErrorCode.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/contracts/services/AzureServiceErrorCode.java @@ -112,8 +112,8 @@ public static AzureServiceErrorCode getAzureServiceCode(int httpStatusCode, Stri String[] errorMessages = errorMessage.split(System.lineSeparator(), 2); for (AzureServiceErrorCode azureServiceErrorCode : AzureServiceErrorCode.values()) { if (azureServiceErrorCode.getStatusCode() == httpStatusCode - && azureServiceErrorCode.getErrorCode().equalsIgnoreCase(errorCode) && - azureServiceErrorCode.getErrorMessage() + && azureServiceErrorCode.getErrorCode().equalsIgnoreCase(errorCode) + && azureServiceErrorCode.getErrorMessage() .equalsIgnoreCase(errorMessages[0])) { return azureServiceErrorCode; } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java index 0c4eb1f0b19bd..be611e14fc011 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java @@ -237,7 +237,7 @@ private void completeExecute(TracingContext tracingContext) LOG.debug("Retrying REST operation {}. RetryCount = {}", operationType, retryCount); sleepDuration = client.getRetryPolicy().getRetryInterval(retryCount); - if(abfsDriverMetrics != null) { + if (abfsDriverMetrics != null) { updateTimeMetrics(retryCount, sleepDuration); } Thread.sleep(sleepDuration); @@ -341,14 +341,11 @@ private boolean executeHttpOperation(final int retryCount, + httpOperation.getStatusCode() + " error code is " + httpOperation.getStorageErrorCode() + " error message is " + httpOperation.getStorageErrorMessage()); - if (serviceErrorCode.equals( - AzureServiceErrorCode.INGRESS_OVER_ACCOUNT_LIMIT) || - serviceErrorCode.equals( - AzureServiceErrorCode.EGRESS_OVER_ACCOUNT_LIMIT)) { + if (serviceErrorCode.equals(AzureServiceErrorCode.INGRESS_OVER_ACCOUNT_LIMIT) + || serviceErrorCode.equals(AzureServiceErrorCode.EGRESS_OVER_ACCOUNT_LIMIT)) { abfsDriverMetrics.getNumberOfBandwidthThrottledRequests() .getAndIncrement(); - } else if (serviceErrorCode.equals( - AzureServiceErrorCode.REQUEST_OVER_ACCOUNT_LIMIT)) { + } else if (serviceErrorCode.equals(AzureServiceErrorCode.REQUEST_OVER_ACCOUNT_LIMIT)) { abfsDriverMetrics.getNumberOfIOPSThrottledRequests() .getAndIncrement(); } else { From bca9c4e900d70d446dbacb932ea9f713ac3eaf53 Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Mon, 18 Jul 2022 12:34:21 +0530 Subject: [PATCH 28/42] Spotbugs fixes --- .../org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java index 19c9f40bdcf33..3caecb5295f88 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java @@ -183,9 +183,9 @@ public String toString() { .append(String.format("%.3f", (double) entry.getValue().getMaxBackoff().get() / 1000L)) .append("s ") - .append(String.format("%.3f", (double) ( - (entry.getValue().getTotalBackoff().get() / totalRequests) - / 1000L))) + .append(String.format("%.3f", + ((double)entry.getValue().getTotalBackoff().get() / totalRequests) + / 1000L)) .append("s "); } else { metricString.append("#MMA#_").append(entry.getKey()) From 15337ff25257d0cbf23a644ef4afc9c25e1988b9 Mon Sep 17 00:00:00 2001 From: Pranav Saxena Date: Thu, 28 Jul 2022 16:10:57 +0530 Subject: [PATCH 29/42] Added support for read footer metrics --- .../hadoop/fs/azurebfs/AbfsCountersImpl.java | 12 +- .../hadoop/fs/azurebfs/AbfsDriverMetrics.java | 2 +- .../fs/azurebfs/AzureBlobFileSystem.java | 103 +++++++++++++++--- .../fs/azurebfs/AzureBlobFileSystemStore.java | 2 + .../fs/azurebfs/services/AbfsCounters.java | 4 +- .../fs/azurebfs/services/AbfsInputStream.java | 34 +++++- .../services/AbfsInputStreamContext.java | 11 ++ .../services/AbfsReadFooterMetrics.java | 69 ++++++++++++ .../fs/azurebfs/utils/TracingContext.java | 6 +- 9 files changed, 218 insertions(+), 25 deletions(-) create mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsReadFooterMetrics.java diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsCountersImpl.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsCountersImpl.java index 1367855cb9d97..bad4905a1de99 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsCountersImpl.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsCountersImpl.java @@ -21,7 +21,8 @@ import java.net.URI; import java.util.Map; import java.util.UUID; - +import java.util.List; +import java.util.ArrayList; import org.apache.hadoop.classification.VisibleForTesting; import org.apache.hadoop.fs.azurebfs.services.AbfsCounters; @@ -33,7 +34,7 @@ import org.apache.hadoop.metrics2.lib.MetricsRegistry; import org.apache.hadoop.metrics2.lib.MutableCounterLong; import org.apache.hadoop.metrics2.lib.MutableMetric; - +import org.apache.hadoop.fs.azurebfs.services.AbfsReadFooterMetrics; import static org.apache.hadoop.fs.azurebfs.AbfsStatistic.*; import static org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.iostatisticsStore; import java.util.concurrent.atomic.AtomicReference; @@ -66,6 +67,8 @@ public class AbfsCountersImpl implements AbfsCounters { private AtomicReference abfsDriverMetrics = null; + private List readFooterMetricsList; + private static final AbfsStatistic[] STATISTIC_LIST = { CALL_CREATE, CALL_OPEN, @@ -125,6 +128,7 @@ public AbfsCountersImpl(URI uri) { } ioStatisticsStore = ioStatisticsStoreBuilder.build(); abfsDriverMetrics = new AtomicReference<>(new AbfsDriverMetrics()); + readFooterMetricsList = new ArrayList<>(); } /** @@ -196,6 +200,10 @@ public AbfsDriverMetrics getAbfsDriverMetrics() { return abfsDriverMetrics.get(); } + public List getAbfsReadFooterMetrics() { + return readFooterMetricsList; + } + /** * {@inheritDoc} * diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java index 3caecb5295f88..a827b083d8f57 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java @@ -200,7 +200,7 @@ public String toString() { .append(numberOfOtherThrottledRequests) .append(" #%RT=") .append(String.format("%.3f", percentageOfRequestsThrottled)) - .append("#NFR") + .append(" #NFR=") .append(numberOfNetworkFailedRequests) .append(" #TRNR=") .append(numberOfRequestsSucceededWithoutRetrying) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java index 2ff453f8d8780..a6a2d96c8f696 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java @@ -34,6 +34,7 @@ import java.util.Collections; import java.util.EnumSet; import java.util.Map; +import java.util.HashMap; import java.util.Optional; import java.util.UUID; import java.util.concurrent.Callable; @@ -42,7 +43,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; - +import java.util.concurrent.atomic.AtomicLong; import javax.annotation.Nullable; import org.apache.hadoop.classification.VisibleForTesting; @@ -108,7 +109,7 @@ import org.apache.hadoop.util.DurationInfo; import org.apache.hadoop.util.LambdaUtils; import org.apache.hadoop.util.Progressable; - +import org.apache.hadoop.fs.azurebfs.services.AbfsReadFooterMetrics; import static org.apache.hadoop.fs.CommonConfigurationKeys.IOSTATISTICS_LOGGING_LEVEL; import static org.apache.hadoop.fs.CommonConfigurationKeys.IOSTATISTICS_LOGGING_LEVEL_DEFAULT; import static org.apache.hadoop.fs.azurebfs.AbfsStatistic.*; @@ -157,7 +158,6 @@ public class AzureBlobFileSystem extends FileSystem /** Rate limiting for operations which use it to throttle their IO. */ private RateLimiting rateLimiting; - @Override public void initialize(URI uri, Configuration configuration) throws IOException { @@ -683,32 +683,49 @@ public synchronized void close() throws IOException { if (isClosed) { return; } - String metric = abfsCounters.getAbfsDriverMetrics().toString(); + List readFooterMetricsList = abfsCounters.getAbfsReadFooterMetrics(); + List isParquetList = new ArrayList<>(); + List isNonParquetList = new ArrayList<>(); + for (AbfsReadFooterMetrics abfsReadFooterMetrics : readFooterMetricsList) { + if (abfsReadFooterMetrics.getIsParquetFile()) { + isParquetList.add(abfsReadFooterMetrics); + } else { + isNonParquetList.add(abfsReadFooterMetrics); + } + } + AbfsReadFooterMetrics avgParquetReadFooterMetrics = new AbfsReadFooterMetrics(); + AbfsReadFooterMetrics avgNonparquetReadFooterMetrics = new AbfsReadFooterMetrics(); + String readFooterMetric = ""; + if (!isParquetList.isEmpty()){ + avgParquetReadFooterMetrics.setParquetFile(true); + getParquetReadFooterMetricsAverage(isParquetList, avgParquetReadFooterMetrics); + readFooterMetric += getReadFooterMetrics(avgParquetReadFooterMetrics); + } + if(!isNonParquetList.isEmpty()) { + avgNonparquetReadFooterMetrics.setParquetFile(false); + getNonParquetReadFooterMetricsAverage(isNonParquetList, avgNonparquetReadFooterMetrics); + readFooterMetric += getReadFooterMetrics(avgNonparquetReadFooterMetrics); + } + String metric = abfsCounters.getAbfsDriverMetrics().toString() + readFooterMetric; LOG.debug("The metrics collected over this instance are " + metric); if (sendMetricsToStore) { - if (abfsCounters.getAbfsDriverMetrics().getTotalNumberOfRequests().get() - > 0) { + if (abfsCounters.getAbfsDriverMetrics().getTotalNumberOfRequests().get() > 0) { try { Configuration metricConfig = getConf(); - String metricAccountKey = metricConfig.get( - FS_AZURE_METRIC_ACCOUNT_KEY); + String metricAccountKey = metricConfig.get(FS_AZURE_METRIC_ACCOUNT_KEY); final String abfsMetricUrl = metricConfig.get(FS_AZURE_METRIC_URI); if (abfsMetricUrl == null) { return; } - metricConfig.set(FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME, - metricAccountKey); - metricConfig.set(AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION, - "false"); + metricConfig.set(FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME, metricAccountKey); + metricConfig.set(AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION, "false"); URI metricUri; try { metricUri = new URI(getScheme(), abfsMetricUrl, null, null, null); } catch (URISyntaxException ex) { throw new AssertionError(ex); } - AzureBlobFileSystem metricFs - = (AzureBlobFileSystem) FileSystem.newInstance(metricUri, - metricConfig); + AzureBlobFileSystem metricFs = (AzureBlobFileSystem) FileSystem.newInstance(metricUri, metricConfig); metricFs.sendMetric(metric); } catch (AzureBlobFileSystemException ex) { //do nothing @@ -731,6 +748,62 @@ public synchronized void close() throws IOException { } } + private void getParquetReadFooterMetricsAverage(List isParquetList, + AbfsReadFooterMetrics avgParquetReadFooterMetrics){ + avgParquetReadFooterMetrics.setSizeReadByFirstRead( + String.format("%.3f", isParquetList.stream() + .map(AbfsReadFooterMetrics::getSizeReadByFirstRead).mapToDouble( + Double::parseDouble).average().orElse(0.0))); + avgParquetReadFooterMetrics.setOffsetDiffBetweenFirstAndSecondRead( + String.format("%.3f", isParquetList.stream() + .map(AbfsReadFooterMetrics::getOffsetDiffBetweenFirstAndSecondRead) + .mapToDouble(Double::parseDouble).average().orElse(0.0))); + avgParquetReadFooterMetrics.setAvgFileLength(isParquetList.stream() + .map(AbfsReadFooterMetrics::getFileLength) + .mapToDouble(AtomicLong::get).average().orElse(0.0)); + } + + private void getNonParquetReadFooterMetricsAverage(List isNonParquetList, + AbfsReadFooterMetrics avgNonParquetReadFooterMetrics){ + int size = isNonParquetList.get(0).getSizeReadByFirstRead().split("_").length; + double[] store = new double[2*size]; + for (AbfsReadFooterMetrics abfsReadFooterMetrics : isNonParquetList) { + String[] firstReadSize = abfsReadFooterMetrics.getSizeReadByFirstRead().split("_"); + String[] offDiffFirstSecondRead = abfsReadFooterMetrics.getOffsetDiffBetweenFirstAndSecondRead().split("_"); + for(int i=0; i getAbfsReadFooterMetrics(); } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java index 553ccdcbc0a43..f85deed6854cf 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java @@ -86,6 +86,7 @@ public class AbfsInputStream extends FSInputStream implements CanUnbuffer, private final int readAheadRange; private boolean firstRead = true; + private long offsetOfFirstRead = 0; // SAS tokens can be re-used until they expire private CachedSASToken cachedSasToken; private byte[] buffer = null; // will be initialized on first use @@ -105,13 +106,15 @@ public class AbfsInputStream extends FSInputStream implements CanUnbuffer, private int bCursorBkp; private long fCursorBkp; private long fCursorAfterLastReadBkp; - + private final AbfsReadFooterMetrics abfsReadFooterMetrics; /** Stream statistics. */ private final AbfsInputStreamStatistics streamStatistics; private long bytesFromReadAhead; // bytes read from readAhead; for testing private long bytesFromRemoteRead; // bytes read remotely; for testing private Listener listener; + private boolean collectMetricsForNextRead = false; + private boolean collectStreamMetrics = false; private final AbfsInputStreamContext context; private IOStatistics ioStatistics; /** @@ -145,6 +148,7 @@ public AbfsInputStream( this.cachedSasToken = new CachedSASToken( abfsInputStreamContext.getSasTokenRenewPeriodForStreamsInSeconds()); this.streamStatistics = abfsInputStreamContext.getStreamStatistics(); + this.abfsReadFooterMetrics = abfsInputStreamContext.getAbfsReadFooterMetrics(); this.inputStreamId = createInputStreamId(); this.tracingContext = new TracingContext(tracingContext); this.tracingContext.setOperation(FSOperationType.READ); @@ -239,6 +243,17 @@ public synchronized int read(final byte[] b, final int off, final int len) throw // go back and read from buffer is fCursor - limit. // There maybe case that we read less than requested data. long filePosAtStartOfBuffer = fCursor - limit; + if (firstRead && nextReadPos >= contentLength - 16 * ONE_KB) { + this.collectStreamMetrics = true; + this.collectMetricsForNextRead = true; + this.offsetOfFirstRead = nextReadPos; + this.abfsReadFooterMetrics.setSizeReadByFirstRead(len+"_"+(Math.abs(contentLength - nextReadPos))); + this.abfsReadFooterMetrics.getFileLength().set(contentLength); + } + if (!firstRead && collectMetricsForNextRead){ + this.abfsReadFooterMetrics.setOffsetDiffBetweenFirstAndSecondRead(len+"_"+(Math.abs(nextReadPos - offsetOfFirstRead))); + this.collectMetricsForNextRead = false; + } if (nextReadPos >= filePosAtStartOfBuffer && nextReadPos <= fCursor) { // Determining position in buffer from where data is to be read. bCursor = (int) (nextReadPos - filePosAtStartOfBuffer); @@ -325,7 +340,6 @@ private int readOneBlock(final byte[] b, final int off, final int len) throws IO if (firstRead) { firstRead = false; } - if (bytesRead == -1) { return -1; } @@ -696,9 +710,25 @@ public synchronized void close() throws IOException { LOG.debug("Closing {}", this); closed = true; buffer = null; // de-reference the buffer so it can be GC'ed sooner + if(this.collectStreamMetrics) { + checkIsParquet(abfsReadFooterMetrics); + this.client.getAbfsCounters() + .getAbfsReadFooterMetrics() + .add(abfsReadFooterMetrics); + } ReadBufferManager.getBufferManager().purgeBuffersForStream(this); } + private void checkIsParquet(AbfsReadFooterMetrics abfsReadFooterMetrics) { + String[] firstReadSize = abfsReadFooterMetrics.getSizeReadByFirstRead().split("_"); + String[] offDiffFirstSecondRead = abfsReadFooterMetrics.getOffsetDiffBetweenFirstAndSecondRead().split("_"); + if((firstReadSize[0].equals(firstReadSize[1])) && (offDiffFirstSecondRead[0].equals(offDiffFirstSecondRead[1]))){ + abfsReadFooterMetrics.setParquetFile(true); + abfsReadFooterMetrics.setSizeReadByFirstRead(firstReadSize[0]); + abfsReadFooterMetrics.setOffsetDiffBetweenFirstAndSecondRead(offDiffFirstSecondRead[0]); + } + } + /** * Not supported by this stream. Throws {@link UnsupportedOperationException} * @param readlimit ignored diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStreamContext.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStreamContext.java index ae69cde6efac1..5dc3bec2eb87a 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStreamContext.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStreamContext.java @@ -49,6 +49,8 @@ public class AbfsInputStreamContext extends AbfsStreamContext { private boolean bufferedPreadDisabled; + private AbfsReadFooterMetrics abfsReadFooterMetrics; + public AbfsInputStreamContext(final long sasTokenRenewPeriodForStreamsInSeconds) { super(sasTokenRenewPeriodForStreamsInSeconds); } @@ -84,6 +86,12 @@ public AbfsInputStreamContext withStreamStatistics( return this; } + public AbfsInputStreamContext withReadFooterMetrics( + final AbfsReadFooterMetrics abfsReadFooterMetrics) { + this.abfsReadFooterMetrics = abfsReadFooterMetrics; + return this; + } + public AbfsInputStreamContext withReadSmallFilesCompletely( final boolean readSmallFilesCompletely) { this.readSmallFilesCompletely = readSmallFilesCompletely; @@ -148,6 +156,9 @@ public int getReadAheadRange() { public AbfsInputStreamStatistics getStreamStatistics() { return streamStatistics; } + public AbfsReadFooterMetrics getAbfsReadFooterMetrics() { + return abfsReadFooterMetrics; + } public boolean readSmallFilesCompletely() { return this.readSmallFilesCompletely; diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsReadFooterMetrics.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsReadFooterMetrics.java new file mode 100644 index 0000000000000..145fc151d1d85 --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsReadFooterMetrics.java @@ -0,0 +1,69 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs.azurebfs.services; + +import java.util.concurrent.atomic.AtomicLong; + +public class AbfsReadFooterMetrics { + private boolean isParquetFile; + private String sizeReadByFirstRead; + private String offsetDiffBetweenFirstAndSecondRead; + private AtomicLong fileLength; + private double avgFileLength; + + public AbfsReadFooterMetrics() { + this.fileLength = new AtomicLong(); + } + + public boolean getIsParquetFile() { + return isParquetFile; + } + + public void setParquetFile(final boolean parquetFile) { + isParquetFile = parquetFile; + } + + public String getSizeReadByFirstRead() { + return sizeReadByFirstRead; + } + + public void setSizeReadByFirstRead(final String sizeReadByFirstRead) { + this.sizeReadByFirstRead = sizeReadByFirstRead; + } + + public String getOffsetDiffBetweenFirstAndSecondRead() { + return offsetDiffBetweenFirstAndSecondRead; + } + + public void setOffsetDiffBetweenFirstAndSecondRead(final String offsetDiffBetweenFirstAndSecondRead) { + this.offsetDiffBetweenFirstAndSecondRead + = offsetDiffBetweenFirstAndSecondRead; + } + + public AtomicLong getFileLength() { + return fileLength; + } + + public double getAvgFileLength() { + return avgFileLength; + } + + public void setAvgFileLength(final double avgFileLength) { + this.avgFileLength = avgFileLength; + } +} diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/TracingContext.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/TracingContext.java index 70d4371daefe4..67c9789457707 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/TracingContext.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/TracingContext.java @@ -19,10 +19,10 @@ package org.apache.hadoop.fs.azurebfs.utils; import java.util.UUID; - +import java.util.Set; import org.slf4j.Logger; import org.slf4j.LoggerFactory; - +import java.util.LinkedHashSet; import org.apache.hadoop.fs.azurebfs.constants.FSOperationType; import org.apache.hadoop.fs.azurebfs.constants.HttpHeaderConfigurations; import org.apache.hadoop.fs.azurebfs.services.AbfsClient; @@ -62,9 +62,7 @@ public class TracingContext { private Listener listener = null; // null except when testing //final concatenated ID list set into x-ms-client-request-id header private String header = EMPTY_STRING; - private String metricResults = EMPTY_STRING; - private static final Logger LOG = LoggerFactory.getLogger(AbfsClient.class); public static final int MAX_CLIENT_CORRELATION_ID_LENGTH = 72; public static final String CLIENT_CORRELATION_ID_PATTERN = "[a-zA-Z0-9-]*"; From 5706ca04db07d33ef58b63789279f514cc10cdc0 Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Tue, 2 Aug 2022 10:07:10 +0530 Subject: [PATCH 30/42] fix for file size --- .../org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java index f85deed6854cf..e00b07d04f352 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java @@ -243,7 +243,7 @@ public synchronized int read(final byte[] b, final int off, final int len) throw // go back and read from buffer is fCursor - limit. // There maybe case that we read less than requested data. long filePosAtStartOfBuffer = fCursor - limit; - if (firstRead && nextReadPos >= contentLength - 16 * ONE_KB) { + if (firstRead && nextReadPos >= contentLength - 20 * ONE_KB) { this.collectStreamMetrics = true; this.collectMetricsForNextRead = true; this.offsetOfFirstRead = nextReadPos; From 3abca57857ec2dbe746d70b60c9778ec9cb2128c Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Wed, 3 Aug 2022 10:22:08 +0530 Subject: [PATCH 31/42] Checkstyle fixes --- .../hadoop-azure/src/config/checkstyle-suppressions.xml | 6 ++++++ .../org/apache/hadoop/fs/azurebfs/AbfsDriverMetrics.java | 4 ++-- .../hadoop-azure/src/test/resources/log4j.properties | 2 -- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/config/checkstyle-suppressions.xml b/hadoop-tools/hadoop-azure/src/config/checkstyle-suppressions.xml index fd2a7c210e706..c6a67f0b26107 100644 --- a/hadoop-tools/hadoop-azure/src/config/checkstyle-suppressions.xml +++ b/hadoop-tools/hadoop-azure/src/config/checkstyle-suppressions.xml @@ -44,6 +44,12 @@ + + + getMetricsMap() { + public Map getMetricsMap() { return metricsMap; } @@ -169,7 +169,7 @@ public String toString() { + numberOfOtherThrottledRequests.get(); double percentageOfRequestsThrottled = ((double) totalRequestsThrottled / totalNumberOfRequests.get()) * 100; - for (Map.Entry entry : metricsMap.entrySet()) { + for (Map.Entry entry : metricsMap.entrySet()) { metricString.append("#RCTSI#_").append(entry.getKey()) .append("R_").append("=") .append(entry.getValue().getNumberOfRequestsSucceeded()).append(" "); diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java index 4ac8a821a2388..63a9ff136e88c 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java @@ -113,9 +113,14 @@ public class AbfsConfiguration{ private boolean readSmallFilesCompletely; @BooleanConfigurationValidatorAnnotation( - ConfigurationKey = AZURE_ENABLE_METRIC_COLLECTION, - DefaultValue = DEFAULT_AZURE_ENABLE_METRIC_COLLECTION) - private boolean enableMetricCollection; + ConfigurationKey = AZURE_ENABLE_BACKOFF_METRIC_COLLECTION, + DefaultValue = DEFAULT_AZURE_ENABLE_BACKOFF_METRIC_COLLECTION) + private boolean enableBackoffMetricCollection; + + @BooleanConfigurationValidatorAnnotation( + ConfigurationKey = AZURE_ENABLE_FOOTER_METRIC_COLLECTION, + DefaultValue = DEFAULT_AZURE_ENABLE_FOOTER_METRIC_COLLECTION) + private boolean enableFooterMetricCollection; @BooleanConfigurationValidatorAnnotation( ConfigurationKey = AZURE_READ_OPTIMIZE_FOOTER_READ, @@ -694,8 +699,12 @@ public String getAppendBlobDirs() { return this.azureAppendBlobDirs; } - public boolean isMetricCollectionEnabled() { - return enableMetricCollection; + public boolean isBackoffMetricCollectionEnabled() { + return enableBackoffMetricCollection; + } + + public boolean isFooterMetricCollectionEnabled() { + return enableFooterMetricCollection; } public String getAzureInfiniteLeaseDirs() { diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsCountersImpl.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsCountersImpl.java index bad4905a1de99..768a1b6df1059 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsCountersImpl.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsCountersImpl.java @@ -65,7 +65,7 @@ public class AbfsCountersImpl implements AbfsCounters { private final IOStatisticsStore ioStatisticsStore; - private AtomicReference abfsDriverMetrics = null; + private AtomicReference abfsBackoffMetrics = null; private List readFooterMetricsList; @@ -127,7 +127,7 @@ public AbfsCountersImpl(URI uri) { ioStatisticsStoreBuilder.withDurationTracking(durationStats.getStatName()); } ioStatisticsStore = ioStatisticsStoreBuilder.build(); - abfsDriverMetrics = new AtomicReference<>(new AbfsDriverMetrics()); + abfsBackoffMetrics = new AtomicReference<>(new AbfsBackoffMetrics()); readFooterMetricsList = new ArrayList<>(); } @@ -196,8 +196,8 @@ private MetricsRegistry getRegistry() { return registry; } - public AbfsDriverMetrics getAbfsDriverMetrics() { - return abfsDriverMetrics.get(); + public AbfsBackoffMetrics getAbfsBackoffMetrics() { + return abfsBackoffMetrics.get(); } public List getAbfsReadFooterMetrics() { diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java index a6a2d96c8f696..7061cacaa9f88 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java @@ -122,6 +122,9 @@ import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_METRIC_URI; import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.BLOCK_UPLOAD_ACTIVE_BLOCKS_DEFAULT; import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DATA_BLOCKS_BUFFER_DEFAULT; +import static org.apache.hadoop.fs.azurebfs.utils.TracingHeaderFormat.INTERNAL_METRIC_FORMAT; +import static org.apache.hadoop.fs.azurebfs.utils.TracingHeaderFormat.INTERNAL_BACKOFF_METRIC_FORMAT; +import static org.apache.hadoop.fs.azurebfs.utils.TracingHeaderFormat.INTERNAL_FOOTER_METRIC_FORMAT; import static org.apache.hadoop.fs.impl.PathCapabilitiesSupport.validatePathCapabilityArgs; import static org.apache.hadoop.fs.statistics.IOStatisticsLogging.logIOStatisticsAtLevel; import static org.apache.hadoop.util.functional.RemoteIterators.filteringRemoteIterator; @@ -145,7 +148,8 @@ public class AzureBlobFileSystem extends FileSystem private AbfsDelegationTokenManager delegationTokenManager; private AbfsCounters abfsCounters; private String clientCorrelationId; - private boolean sendMetricsToStore; + private boolean sendBackoffMetricsToStore; + private boolean sendFooterMetricsToStore; private TracingHeaderFormat tracingHeaderFormat; private Listener listener; @@ -202,7 +206,8 @@ public void initialize(URI uri, Configuration configuration) .getAbfsConfiguration(); clientCorrelationId = TracingContext.validateClientCorrelationID( abfsConfiguration.getClientCorrelationId()); - sendMetricsToStore = abfsConfiguration.isMetricCollectionEnabled(); + sendBackoffMetricsToStore = abfsConfiguration.isBackoffMetricCollectionEnabled(); + sendFooterMetricsToStore = abfsConfiguration.isFooterMetricCollectionEnabled(); tracingHeaderFormat = abfsConfiguration.getTracingHeaderFormat(); this.setWorkingDirectory(this.getHomeDirectory()); @@ -683,33 +688,32 @@ public synchronized void close() throws IOException { if (isClosed) { return; } - List readFooterMetricsList = abfsCounters.getAbfsReadFooterMetrics(); - List isParquetList = new ArrayList<>(); - List isNonParquetList = new ArrayList<>(); - for (AbfsReadFooterMetrics abfsReadFooterMetrics : readFooterMetricsList) { - if (abfsReadFooterMetrics.getIsParquetFile()) { - isParquetList.add(abfsReadFooterMetrics); - } else { - isNonParquetList.add(abfsReadFooterMetrics); - } - } - AbfsReadFooterMetrics avgParquetReadFooterMetrics = new AbfsReadFooterMetrics(); - AbfsReadFooterMetrics avgNonparquetReadFooterMetrics = new AbfsReadFooterMetrics(); String readFooterMetric = ""; - if (!isParquetList.isEmpty()){ - avgParquetReadFooterMetrics.setParquetFile(true); - getParquetReadFooterMetricsAverage(isParquetList, avgParquetReadFooterMetrics); - readFooterMetric += getReadFooterMetrics(avgParquetReadFooterMetrics); - } - if(!isNonParquetList.isEmpty()) { - avgNonparquetReadFooterMetrics.setParquetFile(false); - getNonParquetReadFooterMetricsAverage(isNonParquetList, avgNonparquetReadFooterMetrics); - readFooterMetric += getReadFooterMetrics(avgNonparquetReadFooterMetrics); + List readFooterMetricsList = abfsCounters.getAbfsReadFooterMetrics(); + if (!readFooterMetricsList.isEmpty()) { + readFooterMetric = AbfsReadFooterMetrics.getFooterMetrics(readFooterMetricsList, readFooterMetric); } - String metric = abfsCounters.getAbfsDriverMetrics().toString() + readFooterMetric; + String metric = abfsCounters.getAbfsBackoffMetrics().toString() + "@" + readFooterMetric; LOG.debug("The metrics collected over this instance are " + metric); - if (sendMetricsToStore) { - if (abfsCounters.getAbfsDriverMetrics().getTotalNumberOfRequests().get() > 0) { + TracingHeaderFormat tracingHeaderFormatMetric = null; + TracingContext tracingContextMetric = null; + if (sendBackoffMetricsToStore && sendFooterMetricsToStore) { + tracingHeaderFormatMetric = TracingHeaderFormat.INTERNAL_METRIC_FORMAT; + tracingContextMetric = getTracingContext(tracingHeaderFormatMetric, + metric); + } else if (sendBackoffMetricsToStore) { + tracingHeaderFormatMetric + = TracingHeaderFormat.INTERNAL_BACKOFF_METRIC_FORMAT; + tracingContextMetric = getTracingContext(tracingHeaderFormatMetric, + abfsCounters.getAbfsBackoffMetrics().toString()); + } else if (sendFooterMetricsToStore) { + tracingHeaderFormatMetric + = TracingHeaderFormat.INTERNAL_FOOTER_METRIC_FORMAT; + tracingContextMetric = getTracingContext(tracingHeaderFormatMetric, + readFooterMetric); + } + if (tracingHeaderFormatMetric != null) { + if (abfsCounters.getAbfsBackoffMetrics().getTotalNumberOfRequests().get() > 0) { try { Configuration metricConfig = getConf(); String metricAccountKey = metricConfig.get(FS_AZURE_METRIC_ACCOUNT_KEY); @@ -726,7 +730,7 @@ public synchronized void close() throws IOException { throw new AssertionError(ex); } AzureBlobFileSystem metricFs = (AzureBlobFileSystem) FileSystem.newInstance(metricUri, metricConfig); - metricFs.sendMetric(metric); + metricFs.sendMetric(tracingContextMetric); } catch (AzureBlobFileSystemException ex) { //do nothing } @@ -748,67 +752,13 @@ public synchronized void close() throws IOException { } } - private void getParquetReadFooterMetricsAverage(List isParquetList, - AbfsReadFooterMetrics avgParquetReadFooterMetrics){ - avgParquetReadFooterMetrics.setSizeReadByFirstRead( - String.format("%.3f", isParquetList.stream() - .map(AbfsReadFooterMetrics::getSizeReadByFirstRead).mapToDouble( - Double::parseDouble).average().orElse(0.0))); - avgParquetReadFooterMetrics.setOffsetDiffBetweenFirstAndSecondRead( - String.format("%.3f", isParquetList.stream() - .map(AbfsReadFooterMetrics::getOffsetDiffBetweenFirstAndSecondRead) - .mapToDouble(Double::parseDouble).average().orElse(0.0))); - avgParquetReadFooterMetrics.setAvgFileLength(isParquetList.stream() - .map(AbfsReadFooterMetrics::getFileLength) - .mapToDouble(AtomicLong::get).average().orElse(0.0)); - } - - private void getNonParquetReadFooterMetricsAverage(List isNonParquetList, - AbfsReadFooterMetrics avgNonParquetReadFooterMetrics){ - int size = isNonParquetList.get(0).getSizeReadByFirstRead().split("_").length; - double[] store = new double[2*size]; - for (AbfsReadFooterMetrics abfsReadFooterMetrics : isNonParquetList) { - String[] firstReadSize = abfsReadFooterMetrics.getSizeReadByFirstRead().split("_"); - String[] offDiffFirstSecondRead = abfsReadFooterMetrics.getOffsetDiffBetweenFirstAndSecondRead().split("_"); - for(int i=0; i getAbfsReadFooterMetrics(); } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsReadFooterMetrics.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsReadFooterMetrics.java index 145fc151d1d85..02eaf4ba79f13 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsReadFooterMetrics.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsReadFooterMetrics.java @@ -18,6 +18,7 @@ package org.apache.hadoop.fs.azurebfs.services; import java.util.concurrent.atomic.AtomicLong; +import java.util.List; public class AbfsReadFooterMetrics { private boolean isParquetFile; @@ -66,4 +67,97 @@ public double getAvgFileLength() { public void setAvgFileLength(final double avgFileLength) { this.avgFileLength = avgFileLength; } + + public static void getParquetReadFooterMetricsAverage(List isParquetList, + AbfsReadFooterMetrics avgParquetReadFooterMetrics){ + avgParquetReadFooterMetrics.setSizeReadByFirstRead( + String.format("%.3f", isParquetList.stream() + .map(AbfsReadFooterMetrics::getSizeReadByFirstRead).mapToDouble( + Double::parseDouble).average().orElse(0.0))); + avgParquetReadFooterMetrics.setOffsetDiffBetweenFirstAndSecondRead( + String.format("%.3f", isParquetList.stream() + .map(AbfsReadFooterMetrics::getOffsetDiffBetweenFirstAndSecondRead) + .mapToDouble(Double::parseDouble).average().orElse(0.0))); + avgParquetReadFooterMetrics.setAvgFileLength(isParquetList.stream() + .map(AbfsReadFooterMetrics::getFileLength) + .mapToDouble(AtomicLong::get).average().orElse(0.0)); + } + + public static void getNonParquetReadFooterMetricsAverage(List isNonParquetList, + AbfsReadFooterMetrics avgNonParquetReadFooterMetrics){ + int size = isNonParquetList.get(0).getSizeReadByFirstRead().split("_").length; + double[] store = new double[2*size]; + for (AbfsReadFooterMetrics abfsReadFooterMetrics : isNonParquetList) { + String[] firstReadSize = abfsReadFooterMetrics.getSizeReadByFirstRead().split("_"); + String[] offDiffFirstSecondRead = abfsReadFooterMetrics.getOffsetDiffBetweenFirstAndSecondRead().split("_"); + for(int i=0; i readFooterMetricsList, String readFooterMetric){ + List isParquetList = new java.util.ArrayList<>(); + List isNonParquetList = new java.util.ArrayList<>(); + for (AbfsReadFooterMetrics abfsReadFooterMetrics : readFooterMetricsList) { + if (abfsReadFooterMetrics.getIsParquetFile()) { + isParquetList.add(abfsReadFooterMetrics); + } else { + isNonParquetList.add(abfsReadFooterMetrics); + } + } + AbfsReadFooterMetrics avgParquetReadFooterMetrics = new AbfsReadFooterMetrics(); + AbfsReadFooterMetrics avgNonparquetReadFooterMetrics = new AbfsReadFooterMetrics(); + + if (!isParquetList.isEmpty()){ + avgParquetReadFooterMetrics.setParquetFile(true); + AbfsReadFooterMetrics.getParquetReadFooterMetricsAverage(isParquetList, avgParquetReadFooterMetrics); + readFooterMetric += AbfsReadFooterMetrics.getReadFooterMetrics(avgParquetReadFooterMetrics); + } + if(!isNonParquetList.isEmpty()) { + avgNonparquetReadFooterMetrics.setParquetFile(false); + AbfsReadFooterMetrics.getNonParquetReadFooterMetricsAverage(isNonParquetList, avgNonparquetReadFooterMetrics); + readFooterMetric += AbfsReadFooterMetrics.getReadFooterMetrics(avgNonparquetReadFooterMetrics); + } + return readFooterMetric; + } } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java index be611e14fc011..14aa650c757b0 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java @@ -38,7 +38,7 @@ import org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding; import org.apache.hadoop.fs.azurebfs.contracts.services.AzureServiceErrorCode; import java.util.Map; -import org.apache.hadoop.fs.azurebfs.AbfsDriverMetrics; +import org.apache.hadoop.fs.azurebfs.AbfsBackoffMetrics; /** * The AbfsRestOperation for Rest AbfsClient. @@ -75,8 +75,8 @@ public class AbfsRestOperation { private int maxIoRetries = 0; private AbfsHttpOperation result; private final AbfsCounters abfsCounters; - private AbfsDriverMetrics abfsDriverMetrics; - private Map metricsMap; + private AbfsBackoffMetrics abfsBackoffMetrics; + private Map metricsMap; /** * Checks if there is non-null HTTP response. * @return true if there is a non-null HTTP response from the ABFS call. @@ -152,10 +152,10 @@ String getSasToken() { this.sasToken = sasToken; this.abfsCounters = client.getAbfsCounters(); if (abfsCounters != null) { - this.abfsDriverMetrics = abfsCounters.getAbfsDriverMetrics(); + this.abfsBackoffMetrics = abfsCounters.getAbfsBackoffMetrics(); } - if (abfsDriverMetrics != null) { - this.metricsMap = abfsDriverMetrics.getMetricsMap(); + if (abfsBackoffMetrics != null) { + this.metricsMap = abfsBackoffMetrics.getMetricsMap(); } this.maxIoRetries = client.getAbfsConfiguration().getMaxIoRetries(); } @@ -227,8 +227,8 @@ private void completeExecute(TracingContext tracingContext) retryCount = 0; LOG.debug("First execution of REST operation - {}", operationType); long sleepDuration = 0L; - if (abfsDriverMetrics != null) { - abfsDriverMetrics.getTotalNumberOfRequests().getAndIncrement(); + if (abfsBackoffMetrics != null) { + abfsBackoffMetrics.getTotalNumberOfRequests().getAndIncrement(); } while (!executeHttpOperation(retryCount, tracingContext)) { try { @@ -237,16 +237,16 @@ private void completeExecute(TracingContext tracingContext) LOG.debug("Retrying REST operation {}. RetryCount = {}", operationType, retryCount); sleepDuration = client.getRetryPolicy().getRetryInterval(retryCount); - if (abfsDriverMetrics != null) { - updateTimeMetrics(retryCount, sleepDuration); + if (abfsBackoffMetrics != null) { + updateBackoffTimeMetrics(retryCount, sleepDuration); } Thread.sleep(sleepDuration); } catch (InterruptedException ex) { Thread.currentThread().interrupt(); } } - if (abfsDriverMetrics != null) { - updateDriverMetrics(retryCount, result.getStatusCode()); + if (abfsBackoffMetrics != null) { + updateBackoffMetrics(retryCount, result.getStatusCode()); } if (result.getStatusCode() >= HttpURLConnection.HTTP_BAD_REQUEST) { throw new AbfsRestOperationException(result.getStatusCode(), result.getStorageErrorCode(), @@ -256,21 +256,19 @@ private void completeExecute(TracingContext tracingContext) LOG.trace("{} REST operation complete", operationType); } - private synchronized void updateDriverMetrics(int retryCount, int statusCode){ + private synchronized void updateBackoffMetrics(int retryCount, int statusCode){ if (statusCode < HttpURLConnection.HTTP_OK || statusCode >= HttpURLConnection.HTTP_INTERNAL_ERROR) { if (retryCount >= maxIoRetries) { - abfsDriverMetrics.getNumberOfRequestsFailed().getAndIncrement(); + abfsBackoffMetrics.getNumberOfRequestsFailed().getAndIncrement(); } } else { if (retryCount > 0 && retryCount <= maxIoRetries) { - maxRetryCount = Math.max(abfsDriverMetrics.getMaxRetryCount().get(), - retryCount); - abfsDriverMetrics.getMaxRetryCount().set(maxRetryCount); + maxRetryCount = Math.max(abfsBackoffMetrics.getMaxRetryCount().get(), retryCount); + abfsBackoffMetrics.getMaxRetryCount().set(maxRetryCount); updateCount(retryCount); } else { - abfsDriverMetrics.getNumberOfRequestsSucceededWithoutRetrying() - .getAndIncrement(); + abfsBackoffMetrics.getNumberOfRequestsSucceededWithoutRetrying().getAndIncrement(); } } } @@ -343,14 +341,11 @@ private boolean executeHttpOperation(final int retryCount, + " error message is " + httpOperation.getStorageErrorMessage()); if (serviceErrorCode.equals(AzureServiceErrorCode.INGRESS_OVER_ACCOUNT_LIMIT) || serviceErrorCode.equals(AzureServiceErrorCode.EGRESS_OVER_ACCOUNT_LIMIT)) { - abfsDriverMetrics.getNumberOfBandwidthThrottledRequests() - .getAndIncrement(); + abfsBackoffMetrics.getNumberOfBandwidthThrottledRequests().getAndIncrement(); } else if (serviceErrorCode.equals(AzureServiceErrorCode.REQUEST_OVER_ACCOUNT_LIMIT)) { - abfsDriverMetrics.getNumberOfIOPSThrottledRequests() - .getAndIncrement(); + abfsBackoffMetrics.getNumberOfIOPSThrottledRequests().getAndIncrement(); } else { - abfsDriverMetrics.getNumberOfOtherThrottledRequests() - .getAndIncrement(); + abfsBackoffMetrics.getNumberOfOtherThrottledRequests().getAndIncrement(); } } incrementCounter(AbfsStatistic.GET_RESPONSES, 1); @@ -367,9 +362,9 @@ private boolean executeHttpOperation(final int retryCount, hostname = httpOperation.getHost(); LOG.warn("Unknown host name: {}. Retrying to resolve the host name...", hostname); - abfsDriverMetrics.getNumberOfRequestsFailed().getAndIncrement(); + abfsBackoffMetrics.getNumberOfRequestsFailed().getAndIncrement(); if (!client.getRetryPolicy().shouldRetry(retryCount, -1)) { - updateDriverMetrics(retryCount, httpOperation.getStatusCode()); + updateBackoffMetrics(retryCount, httpOperation.getStatusCode()); throw new InvalidAbfsRestOperationException(ex); } return false; @@ -377,12 +372,11 @@ private boolean executeHttpOperation(final int retryCount, if (LOG.isDebugEnabled()) { LOG.debug("HttpRequestFailure: {}, {}", httpOperation, ex); } - abfsDriverMetrics.getNumberOfNetworkFailedRequests().getAndIncrement(); + abfsBackoffMetrics.getNumberOfNetworkFailedRequests().getAndIncrement(); if (!client.getRetryPolicy().shouldRetry(retryCount, -1)) { - updateDriverMetrics(retryCount, httpOperation.getStatusCode()); + updateBackoffMetrics(retryCount, httpOperation.getStatusCode()); throw new InvalidAbfsRestOperationException(ex); } - return false; } finally { AbfsClientThrottlingIntercept.updateMetrics(operationType, httpOperation); @@ -416,7 +410,7 @@ private void updateCount(int retryCount){ metricsMap.get(retryCounter).getNumberOfRequestsSucceeded().getAndIncrement(); } - private void updateTimeMetrics(int retryCount, long sleepDuration){ + private void updateBackoffTimeMetrics(int retryCount, long sleepDuration){ String retryCounter = getKey(retryCount); long minBackoffTime = Math.min(metricsMap.get(retryCounter).getMinBackoff().get(), sleepDuration); long maxBackoffForTime = Math.max(metricsMap.get(retryCounter).getMaxBackoff().get(), sleepDuration); diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/TracingContext.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/TracingContext.java index 67c9789457707..3d1908390346c 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/TracingContext.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/TracingContext.java @@ -177,7 +177,14 @@ public void constructHeader(AbfsHttpOperation httpOperation) { header = clientCorrelationID + ":" + clientRequestId; break; case INTERNAL_METRIC_FORMAT: - header = clientCorrelationID + ":" + clientRequestId + ":" + fileSystemID + ":" + metricResults; + String[] metric = metricResults.split("@"); + header = clientCorrelationID + ":" + clientRequestId + ":" + fileSystemID + ":" + "BO:" + metric[0] + "FO:" + metric[1]; + break; + case INTERNAL_FOOTER_METRIC_FORMAT: + header = clientCorrelationID + ":" + clientRequestId + ":" + fileSystemID + ":" + "FO:" + metricResults; + break; + case INTERNAL_BACKOFF_METRIC_FORMAT: + header = clientCorrelationID + ":" + clientRequestId + ":" + fileSystemID + ":" + "BO:" + metricResults; break; default: header = clientRequestId; //case SINGLE_ID_FORMAT diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/TracingHeaderFormat.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/TracingHeaderFormat.java index 010308e85b5b7..ab36191db3898 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/TracingHeaderFormat.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/TracingHeaderFormat.java @@ -26,6 +26,12 @@ public enum TracingHeaderFormat { ALL_ID_FORMAT, // :: // :::: + INTERNAL_BACKOFF_METRIC_FORMAT, // :: + // : + + INTERNAL_FOOTER_METRIC_FORMAT, // :: + // : + INTERNAL_METRIC_FORMAT; // :: - // : + // :: } diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsRestOperation.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsRestOperation.java index 1de3d97d0aaaf..954fcf8cf3aee 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsRestOperation.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsRestOperation.java @@ -36,7 +36,7 @@ public TestAbfsRestOperation() throws Exception { } @Test - public void testDriverRetryMetrics() throws Exception { + public void testBackoffRetryMetrics() throws Exception { final AzureBlobFileSystem fs = getFileSystem(); AbfsClient testClient = super.getAbfsClient(super.getAbfsStore(getFileSystem())); @@ -53,7 +53,7 @@ public void testDriverRetryMetrics() throws Exception { getMetrics.invoke(op, retryCount, statusCode); } //For retry count greater than max configured value, the request should fail - Assert.assertEquals(testClient.getAbfsCounters().getAbfsDriverMetrics().getNumberOfRequestsFailed().toString(), "3"); + Assert.assertEquals(testClient.getAbfsCounters().getAbfsBackoffMetrics().getNumberOfRequestsFailed().toString(), "3"); fs.close(); } } From 05acafc326d01100ca559a5f1bec31e5c1484307 Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Wed, 3 Aug 2022 19:25:13 +0530 Subject: [PATCH 33/42] Update documentation --- hadoop-tools/hadoop-azure/src/site/markdown/abfs.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/site/markdown/abfs.md b/hadoop-tools/hadoop-azure/src/site/markdown/abfs.md index 7250e6893a0ef..9ec90f6160a75 100644 --- a/hadoop-tools/hadoop-azure/src/site/markdown/abfs.md +++ b/hadoop-tools/hadoop-azure/src/site/markdown/abfs.md @@ -1002,8 +1002,14 @@ the same for as the existing account on which other requests are made. `fs.azure.metric.account.key`: This is the access key for the storage account used for pushing metrics to the store. -`fs.azure.enable.metric.collection`: This configuration provides an option to -specify whether we want to push the metrics or not. +`fs.azure.enable.backoff.metric.collection`: This configuration provides an +option to +specify whether we want to push the backoff metrics or not. +By default, this config will be set to false. + +`fs.azure.enable.footer.metric.collection`: This configuration provides an +option to +specify whether we want to push the footer metrics or not. By default, this config will be set to false. `fs.azure.metric.uri`: This configuration provides the uri in the format of From 388d837c0d4bb2c32c0a4e69512d8c63f6ba3e29 Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Thu, 4 Aug 2022 09:22:11 +0530 Subject: [PATCH 34/42] Checkstyle fixes --- .../src/config/checkstyle-suppressions.xml | 4 ++++ .../fs/azurebfs/AzureBlobFileSystem.java | 5 ---- .../fs/azurebfs/services/AbfsInputStream.java | 8 +++---- .../services/AbfsReadFooterMetrics.java | 24 +++++++++++-------- .../fs/azurebfs/utils/TracingContext.java | 2 -- 5 files changed, 22 insertions(+), 21 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/config/checkstyle-suppressions.xml b/hadoop-tools/hadoop-azure/src/config/checkstyle-suppressions.xml index c6a67f0b26107..749d35d857808 100644 --- a/hadoop-tools/hadoop-azure/src/config/checkstyle-suppressions.xml +++ b/hadoop-tools/hadoop-azure/src/config/checkstyle-suppressions.xml @@ -46,8 +46,12 @@ files="org[\\/]apache[\\/]hadoop[\\/]fs[\\/]azurebfs[\\/]AzureBlobFileSystemStore.java"/> + + readFooterMetr AbfsReadFooterMetrics.getParquetReadFooterMetricsAverage(isParquetList, avgParquetReadFooterMetrics); readFooterMetric += AbfsReadFooterMetrics.getReadFooterMetrics(avgParquetReadFooterMetrics); } - if(!isNonParquetList.isEmpty()) { + if (!isNonParquetList.isEmpty()) { avgNonparquetReadFooterMetrics.setParquetFile(false); AbfsReadFooterMetrics.getNonParquetReadFooterMetricsAverage(isNonParquetList, avgNonparquetReadFooterMetrics); readFooterMetric += AbfsReadFooterMetrics.getReadFooterMetrics(avgNonparquetReadFooterMetrics); diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/TracingContext.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/TracingContext.java index 3d1908390346c..9614b409f53ba 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/TracingContext.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/TracingContext.java @@ -19,10 +19,8 @@ package org.apache.hadoop.fs.azurebfs.utils; import java.util.UUID; -import java.util.Set; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.LinkedHashSet; import org.apache.hadoop.fs.azurebfs.constants.FSOperationType; import org.apache.hadoop.fs.azurebfs.constants.HttpHeaderConfigurations; import org.apache.hadoop.fs.azurebfs.services.AbfsClient; From da418363440baf58b6ab3b222cbb232bc3fd9853 Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Mon, 8 Aug 2022 15:27:04 +0530 Subject: [PATCH 35/42] Added new tracing class for metrics --- .../services/AbfsReadFooterMetrics.java | 7 ++- .../fs/azurebfs/utils/TracingContext.java | 10 ---- .../azurebfs/utils/TracingMetricContext.java | 59 +++++++++++++++++++ 3 files changed, 63 insertions(+), 13 deletions(-) create mode 100644 hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/TracingMetricContext.java diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsReadFooterMetrics.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsReadFooterMetrics.java index 09440e5edb72e..51b6ce014e385 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsReadFooterMetrics.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsReadFooterMetrics.java @@ -19,12 +19,13 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.List; +import java.util.ArrayList; public class AbfsReadFooterMetrics { private boolean isParquetFile; private String sizeReadByFirstRead; private String offsetDiffBetweenFirstAndSecondRead; - private AtomicLong fileLength; + private final AtomicLong fileLength; private double avgFileLength; public AbfsReadFooterMetrics() { @@ -140,8 +141,8 @@ public static String getReadFooterMetrics(AbfsReadFooterMetrics avgReadFooterMet } public static String getFooterMetrics(List readFooterMetricsList, String readFooterMetric){ - List isParquetList = new java.util.ArrayList<>(); - List isNonParquetList = new java.util.ArrayList<>(); + List isParquetList = new ArrayList<>(); + List isNonParquetList = new ArrayList<>(); for (AbfsReadFooterMetrics abfsReadFooterMetrics : readFooterMetricsList) { if (abfsReadFooterMetrics.getIsParquetFile()) { isParquetList.add(abfsReadFooterMetrics); diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/TracingContext.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/TracingContext.java index 9614b409f53ba..21f6429433886 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/TracingContext.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/TracingContext.java @@ -174,16 +174,6 @@ public void constructHeader(AbfsHttpOperation httpOperation) { case TWO_ID_FORMAT: header = clientCorrelationID + ":" + clientRequestId; break; - case INTERNAL_METRIC_FORMAT: - String[] metric = metricResults.split("@"); - header = clientCorrelationID + ":" + clientRequestId + ":" + fileSystemID + ":" + "BO:" + metric[0] + "FO:" + metric[1]; - break; - case INTERNAL_FOOTER_METRIC_FORMAT: - header = clientCorrelationID + ":" + clientRequestId + ":" + fileSystemID + ":" + "FO:" + metricResults; - break; - case INTERNAL_BACKOFF_METRIC_FORMAT: - header = clientCorrelationID + ":" + clientRequestId + ":" + fileSystemID + ":" + "BO:" + metricResults; - break; default: header = clientRequestId; //case SINGLE_ID_FORMAT } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/TracingMetricContext.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/TracingMetricContext.java new file mode 100644 index 0000000000000..2c0e7798bfde0 --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/TracingMetricContext.java @@ -0,0 +1,59 @@ +package org.apache.hadoop.fs.azurebfs.utils; + +import org.apache.hadoop.fs.azurebfs.services.AbfsCounters; +import org.apache.hadoop.fs.azurebfs.constants.FSOperationType; +import org.apache.hadoop.fs.azurebfs.services.AbfsHttpOperation; +import java.util.UUID; +import java.util.List; +import org.apache.hadoop.fs.azurebfs.services.AbfsReadFooterMetrics; +import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.EMPTY_STRING; + +public class TracingMetricContext extends TracingContext{ + private AbfsCounters abfsCounters; + private String header = EMPTY_STRING; + + private final String clientCorrelationID; // passed over config by client + private final String fileSystemID; // GUID for fileSystem instance + private String clientRequestId = EMPTY_STRING; + private TracingHeaderFormat tracingHeaderFormat; + + public TracingMetricContext(String clientCorrelationID, String fileSystemID, + FSOperationType opType, boolean needsPrimaryReqId, + TracingHeaderFormat tracingHeaderFormat, Listener listener, + AbfsCounters abfsCounters) { + super(clientCorrelationID, fileSystemID, opType, needsPrimaryReqId, tracingHeaderFormat, listener); + this.clientCorrelationID = clientCorrelationID; + this.fileSystemID = fileSystemID; + this.tracingHeaderFormat = tracingHeaderFormat; + this.abfsCounters = abfsCounters; + } + + private String getFooterMetrics(){ + List readFooterMetricsList = abfsCounters.getAbfsReadFooterMetrics(); + String readFooterMetric = ""; + if (!readFooterMetricsList.isEmpty()) { + readFooterMetric = AbfsReadFooterMetrics.getFooterMetrics(readFooterMetricsList, readFooterMetric); + } + return readFooterMetric; + } + + @Override + public void constructHeader(AbfsHttpOperation httpOperation){ + clientRequestId = UUID.randomUUID().toString(); + switch (tracingHeaderFormat) { + case INTERNAL_METRIC_FORMAT: + header = clientCorrelationID + ":" + clientRequestId + ":" + fileSystemID + + ":" + "BO:" + abfsCounters.getAbfsBackoffMetrics().toString() + + "FO:" + getFooterMetrics(); + break; + case INTERNAL_FOOTER_METRIC_FORMAT: + header = clientCorrelationID + ":" + clientRequestId + ":" + fileSystemID + + ":" + "FO:" + getFooterMetrics(); + break; + case INTERNAL_BACKOFF_METRIC_FORMAT: + header = clientCorrelationID + ":" + clientRequestId + ":" + fileSystemID + + ":" + "BO:" + abfsCounters.getAbfsBackoffMetrics().toString(); + break; + } + } +} From d55aa472289f8956e4e2b3088b2798e5d68a1beb Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Wed, 10 Aug 2022 12:20:23 +0530 Subject: [PATCH 36/42] Additional tracing header formats --- .../hadoop/fs/azurebfs/AbfsConfiguration.java | 7 ++++ .../fs/azurebfs/AzureBlobFileSystem.java | 42 ++++--------------- .../azurebfs/constants/ConfigurationKeys.java | 1 + .../fs/azurebfs/services/AbfsInputStream.java | 17 ++++++-- .../services/AbfsReadFooterMetrics.java | 19 ++++++++- .../azurebfs/utils/TracingHeaderFormat.java | 9 +++- .../azurebfs/utils/TracingMetricContext.java | 16 +++++-- .../hadoop-azure/src/site/markdown/abfs.md | 22 +++++----- 8 files changed, 81 insertions(+), 52 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java index 63a9ff136e88c..cb25ca3a75a36 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java @@ -777,6 +777,13 @@ public TracingHeaderFormat getTracingHeaderFormat() { return getEnum(FS_AZURE_TRACINGHEADER_FORMAT, TracingHeaderFormat.ALL_ID_FORMAT); } + /** + * Enum config to allow user to pick format of x-ms-client-request-id header + * @return tracingContextFormat config if valid, else default ALL_ID_FORMAT + */ + public TracingHeaderFormat getTracingMetricHeaderFormat() { + return getEnum(FS_AZURE_TRACINGMETRICHEADER_FORMAT, TracingHeaderFormat.EMPTY); + } public AuthType getAuthType(String accountName) { return getEnum(FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME, AuthType.SharedKey); } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java index a3419e2f52f50..ee8864e33d762 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java @@ -107,7 +107,7 @@ import org.apache.hadoop.util.DurationInfo; import org.apache.hadoop.util.LambdaUtils; import org.apache.hadoop.util.Progressable; -import org.apache.hadoop.fs.azurebfs.services.AbfsReadFooterMetrics; +import org.apache.hadoop.fs.azurebfs.utils.TracingMetricContext; import static org.apache.hadoop.fs.CommonConfigurationKeys.IOSTATISTICS_LOGGING_LEVEL; import static org.apache.hadoop.fs.CommonConfigurationKeys.IOSTATISTICS_LOGGING_LEVEL_DEFAULT; import static org.apache.hadoop.fs.azurebfs.AbfsStatistic.*; @@ -146,6 +146,8 @@ public class AzureBlobFileSystem extends FileSystem private boolean sendBackoffMetricsToStore; private boolean sendFooterMetricsToStore; private TracingHeaderFormat tracingHeaderFormat; + + private TracingHeaderFormat tracingMetricHeaderFormat; private Listener listener; /** Name of blockFactory to be used by AbfsOutputStream. */ @@ -204,6 +206,7 @@ public void initialize(URI uri, Configuration configuration) sendBackoffMetricsToStore = abfsConfiguration.isBackoffMetricCollectionEnabled(); sendFooterMetricsToStore = abfsConfiguration.isFooterMetricCollectionEnabled(); tracingHeaderFormat = abfsConfiguration.getTracingHeaderFormat(); + tracingMetricHeaderFormat = abfsConfiguration.getTracingMetricHeaderFormat(); this.setWorkingDirectory(this.getHomeDirectory()); if (abfsConfiguration.getCreateRemoteFileSystemDuringInitialization()) { @@ -683,31 +686,10 @@ public synchronized void close() throws IOException { if (isClosed) { return; } - String readFooterMetric = ""; - List readFooterMetricsList = abfsCounters.getAbfsReadFooterMetrics(); - if (!readFooterMetricsList.isEmpty()) { - readFooterMetric = AbfsReadFooterMetrics.getFooterMetrics(readFooterMetricsList, readFooterMetric); - } - String metric = abfsCounters.getAbfsBackoffMetrics().toString() + "@" + readFooterMetric; - LOG.debug("The metrics collected over this instance are " + metric); - TracingHeaderFormat tracingHeaderFormatMetric = null; - TracingContext tracingContextMetric = null; - if (sendBackoffMetricsToStore && sendFooterMetricsToStore) { - tracingHeaderFormatMetric = TracingHeaderFormat.INTERNAL_METRIC_FORMAT; - tracingContextMetric = getTracingContext(tracingHeaderFormatMetric, - metric); - } else if (sendBackoffMetricsToStore) { - tracingHeaderFormatMetric - = TracingHeaderFormat.INTERNAL_BACKOFF_METRIC_FORMAT; - tracingContextMetric = getTracingContext(tracingHeaderFormatMetric, - abfsCounters.getAbfsBackoffMetrics().toString()); - } else if (sendFooterMetricsToStore) { - tracingHeaderFormatMetric - = TracingHeaderFormat.INTERNAL_FOOTER_METRIC_FORMAT; - tracingContextMetric = getTracingContext(tracingHeaderFormatMetric, - readFooterMetric); - } - if (tracingHeaderFormatMetric != null) { + TracingMetricContext tracingMetricContext = new TracingMetricContext(clientCorrelationId, + fileSystemId, FSOperationType.GET_ATTR, true, tracingMetricHeaderFormat, + listener, abfsCounters); + if (!tracingMetricHeaderFormat.toString().equals("")) { if (abfsCounters.getAbfsBackoffMetrics().getTotalNumberOfRequests().get() > 0) { try { Configuration metricConfig = getConf(); @@ -725,7 +707,7 @@ public synchronized void close() throws IOException { throw new AssertionError(ex); } AzureBlobFileSystem metricFs = (AzureBlobFileSystem) FileSystem.newInstance(metricUri, metricConfig); - metricFs.sendMetric(tracingContextMetric); + metricFs.sendMetric(tracingMetricContext); } catch (AzureBlobFileSystemException ex) { //do nothing } @@ -747,12 +729,6 @@ public synchronized void close() throws IOException { } } - public TracingContext getTracingContext(TracingHeaderFormat tracingHeaderFormatMetric, String metric) { - return new TracingContext(clientCorrelationId, - fileSystemId, FSOperationType.GET_ATTR, true, tracingHeaderFormatMetric, - listener, metric); - } - public void sendMetric(TracingContext tracingContextMetric) throws AzureBlobFileSystemException { abfsStore.sendMetric(tracingContextMetric); } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java index 28fd0fe5a0927..f8030a107740f 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java @@ -159,6 +159,7 @@ public final class ConfigurationKeys { * character constraints are not satisfied. **/ public static final String FS_AZURE_CLIENT_CORRELATIONID = "fs.azure.client.correlationid"; public static final String FS_AZURE_TRACINGHEADER_FORMAT = "fs.azure.tracingheader.format"; + public static final String FS_AZURE_TRACINGMETRICHEADER_FORMAT = "fs.azure.tracingmetricheader.format"; public static final String FS_AZURE_CLUSTER_NAME = "fs.azure.cluster.name"; public static final String FS_AZURE_CLUSTER_TYPE = "fs.azure.cluster.type"; public static final String FS_AZURE_SSL_CHANNEL_MODE_KEY = "fs.azure.ssl.channel.mode"; diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java index d280edf10e0d1..27b4281741dcb 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java @@ -113,7 +113,9 @@ public class AbfsInputStream extends FSInputStream implements CanUnbuffer, private long bytesFromRemoteRead; // bytes read remotely; for testing private Listener listener; private boolean collectMetricsForNextRead = false; - + private long dataLenRequested; + private long readReqCount; + private boolean collectLenMetrics = false; private boolean collectStreamMetrics = false; private final AbfsInputStreamContext context; private IOStatistics ioStatistics; @@ -247,12 +249,17 @@ public synchronized int read(final byte[] b, final int off, final int len) throw this.collectStreamMetrics = true; this.collectMetricsForNextRead = true; this.offsetOfFirstRead = nextReadPos; - this.abfsReadFooterMetrics.setSizeReadByFirstRead(len+"_"+(Math.abs(contentLength - nextReadPos))); + this.abfsReadFooterMetrics.setSizeReadByFirstRead(len + "_" + (Math.abs(contentLength - nextReadPos))); this.abfsReadFooterMetrics.getFileLength().set(contentLength); } + if (collectLenMetrics) { + dataLenRequested += len; + readReqCount += 1; + } if (!firstRead && collectMetricsForNextRead){ - this.abfsReadFooterMetrics.setOffsetDiffBetweenFirstAndSecondRead(len+"_"+(Math.abs(nextReadPos - offsetOfFirstRead))); + this.abfsReadFooterMetrics.setOffsetDiffBetweenFirstAndSecondRead(len + "_" + (Math.abs(nextReadPos - offsetOfFirstRead))); this.collectMetricsForNextRead = false; + this.collectLenMetrics = true; } if (nextReadPos >= filePosAtStartOfBuffer && nextReadPos <= fCursor) { // Determining position in buffer from where data is to be read. @@ -712,6 +719,10 @@ public synchronized void close() throws IOException { buffer = null; // de-reference the buffer so it can be GC'ed sooner if (this.collectStreamMetrics) { checkIsParquet(abfsReadFooterMetrics); + if (readReqCount > 0) { + abfsReadFooterMetrics.setAvgReadLenRequested( + (double) dataLenRequested / readReqCount); + } this.client.getAbfsCounters().getAbfsReadFooterMetrics() .add(abfsReadFooterMetrics); } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsReadFooterMetrics.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsReadFooterMetrics.java index 51b6ce014e385..ccf752fb9f5d6 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsReadFooterMetrics.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsReadFooterMetrics.java @@ -27,6 +27,7 @@ public class AbfsReadFooterMetrics { private String offsetDiffBetweenFirstAndSecondRead; private final AtomicLong fileLength; private double avgFileLength; + private double avgReadLenRequested;; public AbfsReadFooterMetrics() { this.fileLength = new AtomicLong(); @@ -69,6 +70,14 @@ public void setAvgFileLength(final double avgFileLength) { this.avgFileLength = avgFileLength; } + public double getAvgReadLenRequested() { + return avgReadLenRequested; + } + + public void setAvgReadLenRequested(final double avgReadLenRequested) { + this.avgReadLenRequested = avgReadLenRequested; + } + public static void getParquetReadFooterMetricsAverage(List isParquetList, AbfsReadFooterMetrics avgParquetReadFooterMetrics){ avgParquetReadFooterMetrics.setSizeReadByFirstRead( @@ -82,6 +91,9 @@ public static void getParquetReadFooterMetricsAverage(List isNonParquetList, @@ -111,6 +123,9 @@ public static void getNonParquetReadFooterMetricsAverage(List:: // : - INTERNAL_METRIC_FORMAT; // :: + INTERNAL_METRIC_FORMAT, // :: // :: + + EMPTY; + + @Override + public String toString() { + return this == EMPTY ? "" : this.name(); + } } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/TracingMetricContext.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/TracingMetricContext.java index 2c0e7798bfde0..dc1a6e04fe044 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/TracingMetricContext.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/TracingMetricContext.java @@ -7,14 +7,16 @@ import java.util.List; import org.apache.hadoop.fs.azurebfs.services.AbfsReadFooterMetrics; import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.EMPTY_STRING; +import org.apache.hadoop.fs.azurebfs.constants.HttpHeaderConfigurations; public class TracingMetricContext extends TracingContext{ - private AbfsCounters abfsCounters; + private final AbfsCounters abfsCounters; private String header = EMPTY_STRING; private final String clientCorrelationID; // passed over config by client private final String fileSystemID; // GUID for fileSystem instance private String clientRequestId = EMPTY_STRING; + private Listener listener = null; private TracingHeaderFormat tracingHeaderFormat; public TracingMetricContext(String clientCorrelationID, String fileSystemID, @@ -26,6 +28,7 @@ public TracingMetricContext(String clientCorrelationID, String fileSystemID, this.fileSystemID = fileSystemID; this.tracingHeaderFormat = tracingHeaderFormat; this.abfsCounters = abfsCounters; + this.listener = listener; } private String getFooterMetrics(){ @@ -43,8 +46,8 @@ public void constructHeader(AbfsHttpOperation httpOperation){ switch (tracingHeaderFormat) { case INTERNAL_METRIC_FORMAT: header = clientCorrelationID + ":" + clientRequestId + ":" + fileSystemID - + ":" + "BO:" + abfsCounters.getAbfsBackoffMetrics().toString() + - "FO:" + getFooterMetrics(); + + ":" + "BO:" + abfsCounters.getAbfsBackoffMetrics().toString() + + "FO:" + getFooterMetrics(); break; case INTERNAL_FOOTER_METRIC_FORMAT: header = clientCorrelationID + ":" + clientRequestId + ":" + fileSystemID @@ -54,6 +57,13 @@ public void constructHeader(AbfsHttpOperation httpOperation){ header = clientCorrelationID + ":" + clientRequestId + ":" + fileSystemID + ":" + "BO:" + abfsCounters.getAbfsBackoffMetrics().toString(); break; + default: + header = ""; + break; + } + if (listener != null) { //for testing + listener.callTracingHeaderValidator(header, tracingHeaderFormat); } + httpOperation.setRequestProperty(HttpHeaderConfigurations.X_MS_CLIENT_REQUEST_ID, header); } } diff --git a/hadoop-tools/hadoop-azure/src/site/markdown/abfs.md b/hadoop-tools/hadoop-azure/src/site/markdown/abfs.md index 9ec90f6160a75..4e25a94b3593f 100644 --- a/hadoop-tools/hadoop-azure/src/site/markdown/abfs.md +++ b/hadoop-tools/hadoop-azure/src/site/markdown/abfs.md @@ -742,13 +742,23 @@ input is invalid. #### 1. Correlation IDs Display Options -Config `fs.azure.tracingcontext.format` provides an option to select the format +Config `fs.azure.tracingheader.format` provides an option to select the format of IDs included in the `request-id-header`. This config accepts a String value corresponding to the following enum options. `SINGLE_ID_FORMAT` : clientRequestId `ALL_ID_FORMAT` : all IDs (default) `TWO_ID_FORMAT` : clientCorrelationId:clientRequestId +Config `fs.azure.tracingmetricheader.format` provides an option to select the +format +of IDs included in the `request-id-header` for metrics. This config accepts a +String value +corresponding to the following enum options. +`INTERNAL_METRIC_FORMAT` : all IDs + backoff + footer metrics +`INTERNAL_BACKOFF_METRIC_FORMAT` : all IDs (default) + backoff metrics +`INTERNAL_FOOTER_METRIC_FORMAT` : all IDs (default) + footer metrics +`EMPTY` : default + ### Flush Options #### 1. Azure Blob File System Flush Options @@ -1002,16 +1012,6 @@ the same for as the existing account on which other requests are made. `fs.azure.metric.account.key`: This is the access key for the storage account used for pushing metrics to the store. -`fs.azure.enable.backoff.metric.collection`: This configuration provides an -option to -specify whether we want to push the backoff metrics or not. -By default, this config will be set to false. - -`fs.azure.enable.footer.metric.collection`: This configuration provides an -option to -specify whether we want to push the footer metrics or not. -By default, this config will be set to false. - `fs.azure.metric.uri`: This configuration provides the uri in the format of containername@accountname.dfs.core.windows.net. This should be a part of the config in order to prevent extra calls to create From e26ee5f22d9d7ab4193c2da9d79da66cd8fdaf52 Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Fri, 12 Aug 2022 15:21:57 +0530 Subject: [PATCH 37/42] Test for footer metrics --- .../fs/azurebfs/AbfsBackoffMetrics.java | 10 +-- .../hadoop/fs/azurebfs/AbfsConfiguration.java | 18 ----- .../fs/azurebfs/AzureBlobFileSystem.java | 5 -- .../azurebfs/constants/ConfigurationKeys.java | 2 - .../constants/FileSystemConfigurations.java | 2 - .../fs/azurebfs/services/AbfsClient.java | 2 +- .../fs/azurebfs/services/AbfsInputStream.java | 4 +- .../ITestAbfsInputStreamStatistics.java | 3 +- .../azurebfs/ITestAbfsReadFooterMetrics.java | 78 +++++++++++++++++++ .../services/TestAbfsInputStream.java | 2 +- .../services/TestAbfsRestOperation.java | 2 +- .../fs/azurebfs/utils/TestMockHelpers.java | 59 -------------- 12 files changed, 90 insertions(+), 97 deletions(-) create mode 100644 hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsReadFooterMetrics.java diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsBackoffMetrics.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsBackoffMetrics.java index 5babda75540fb..3c55580331e68 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsBackoffMetrics.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsBackoffMetrics.java @@ -155,11 +155,11 @@ public AtomicLong getNumberOfNetworkFailedRequests() { 5.IT :- Number of IOPS throttled requests 6.OT :- Number of Other throttled requests 7.NFR :- Number of requests which failed due to network errors - 7.%RT :- Percentage of requests that are throttled - 8.TRNR :- Total number of requests which succeeded without retrying - 9.TRF :- Total number of requests which failed - 10.TR :- Total number of requests which were made - 11.MRC :- Max retry count across all requests + 8.%RT :- Percentage of requests that are throttled + 9.TRNR :- Total number of requests which succeeded without retrying + 10.TRF :- Total number of requests which failed + 11.TR :- Total number of requests which were made + 12.MRC :- Max retry count across all requests */ @Override public String toString() { diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java index cb25ca3a75a36..ca299b6eca5ee 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AbfsConfiguration.java @@ -112,16 +112,6 @@ public class AbfsConfiguration{ DefaultValue = DEFAULT_READ_SMALL_FILES_COMPLETELY) private boolean readSmallFilesCompletely; - @BooleanConfigurationValidatorAnnotation( - ConfigurationKey = AZURE_ENABLE_BACKOFF_METRIC_COLLECTION, - DefaultValue = DEFAULT_AZURE_ENABLE_BACKOFF_METRIC_COLLECTION) - private boolean enableBackoffMetricCollection; - - @BooleanConfigurationValidatorAnnotation( - ConfigurationKey = AZURE_ENABLE_FOOTER_METRIC_COLLECTION, - DefaultValue = DEFAULT_AZURE_ENABLE_FOOTER_METRIC_COLLECTION) - private boolean enableFooterMetricCollection; - @BooleanConfigurationValidatorAnnotation( ConfigurationKey = AZURE_READ_OPTIMIZE_FOOTER_READ, DefaultValue = DEFAULT_OPTIMIZE_FOOTER_READ) @@ -699,14 +689,6 @@ public String getAppendBlobDirs() { return this.azureAppendBlobDirs; } - public boolean isBackoffMetricCollectionEnabled() { - return enableBackoffMetricCollection; - } - - public boolean isFooterMetricCollectionEnabled() { - return enableFooterMetricCollection; - } - public String getAzureInfiniteLeaseDirs() { return this.azureInfiniteLeaseDirs; } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java index ee8864e33d762..ce911bc5d8c33 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/AzureBlobFileSystem.java @@ -143,10 +143,7 @@ public class AzureBlobFileSystem extends FileSystem private AbfsDelegationTokenManager delegationTokenManager; private AbfsCounters abfsCounters; private String clientCorrelationId; - private boolean sendBackoffMetricsToStore; - private boolean sendFooterMetricsToStore; private TracingHeaderFormat tracingHeaderFormat; - private TracingHeaderFormat tracingMetricHeaderFormat; private Listener listener; @@ -203,8 +200,6 @@ public void initialize(URI uri, Configuration configuration) .getAbfsConfiguration(); clientCorrelationId = TracingContext.validateClientCorrelationID( abfsConfiguration.getClientCorrelationId()); - sendBackoffMetricsToStore = abfsConfiguration.isBackoffMetricCollectionEnabled(); - sendFooterMetricsToStore = abfsConfiguration.isFooterMetricCollectionEnabled(); tracingHeaderFormat = abfsConfiguration.getTracingHeaderFormat(); tracingMetricHeaderFormat = abfsConfiguration.getTracingMetricHeaderFormat(); this.setWorkingDirectory(this.getHomeDirectory()); diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java index f8030a107740f..8c1829550f088 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/ConfigurationKeys.java @@ -42,8 +42,6 @@ public final class ConfigurationKeys { public static final String FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME_REGX = "fs\\.azure\\.account\\.key\\.(.*)"; public static final String FS_AZURE_SECURE_MODE = "fs.azure.secure.mode"; - public static final String AZURE_ENABLE_BACKOFF_METRIC_COLLECTION = "fs.azure.enable.backoff.metric.collection"; - public static final String AZURE_ENABLE_FOOTER_METRIC_COLLECTION = "fs.azure.enable.footer.metric.collection"; // Retry strategy defined by the user public static final String AZURE_MIN_BACKOFF_INTERVAL = "fs.azure.io.retry.min.backoff.interval"; public static final String AZURE_MAX_BACKOFF_INTERVAL = "fs.azure.io.retry.max.backoff.interval"; diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java index 8f84a1ddd0199..63d62a33b1819 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java @@ -60,8 +60,6 @@ public final class FileSystemConfigurations { public static final int DEFAULT_READ_BUFFER_SIZE = 4 * ONE_MB; // 4 MB public static final boolean DEFAULT_READ_SMALL_FILES_COMPLETELY = false; public static final boolean DEFAULT_OPTIMIZE_FOOTER_READ = false; - public static final boolean DEFAULT_AZURE_ENABLE_BACKOFF_METRIC_COLLECTION = false; - public static final boolean DEFAULT_AZURE_ENABLE_FOOTER_METRIC_COLLECTION = false; public static final boolean DEFAULT_ALWAYS_READ_BUFFER_SIZE = false; public static final int DEFAULT_READ_AHEAD_BLOCK_SIZE = 4 * ONE_MB; public static final int DEFAULT_READ_AHEAD_RANGE = 64 * ONE_KB; // 64 KB diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java index 2e797f0b0c4f1..3f99b8517e164 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsClient.java @@ -1266,7 +1266,7 @@ public SASTokenProvider getSasTokenProvider() { * Getter for abfsCounters from AbfsClient. * @return AbfsCounters instance. */ - protected AbfsCounters getAbfsCounters() { + public AbfsCounters getAbfsCounters() { return abfsCounters; } diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java index 27b4281741dcb..5b2c5280588be 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsInputStream.java @@ -245,8 +245,7 @@ public synchronized int read(final byte[] b, final int off, final int len) throw // go back and read from buffer is fCursor - limit. // There maybe case that we read less than requested data. long filePosAtStartOfBuffer = fCursor - limit; - if (firstRead && nextReadPos >= contentLength - 20 * ONE_KB) { - this.collectStreamMetrics = true; + if (abfsReadFooterMetrics != null && firstRead && nextReadPos >= contentLength - 20 * ONE_KB) { this.collectMetricsForNextRead = true; this.offsetOfFirstRead = nextReadPos; this.abfsReadFooterMetrics.setSizeReadByFirstRead(len + "_" + (Math.abs(contentLength - nextReadPos))); @@ -257,6 +256,7 @@ public synchronized int read(final byte[] b, final int off, final int len) throw readReqCount += 1; } if (!firstRead && collectMetricsForNextRead){ + this.collectStreamMetrics = true; this.abfsReadFooterMetrics.setOffsetDiffBetweenFirstAndSecondRead(len + "_" + (Math.abs(nextReadPos - offsetOfFirstRead))); this.collectMetricsForNextRead = false; this.collectLenMetrics = true; diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsInputStreamStatistics.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsInputStreamStatistics.java index d96f1a283609f..88146ca8dbe1d 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsInputStreamStatistics.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsInputStreamStatistics.java @@ -34,7 +34,7 @@ import org.apache.hadoop.fs.statistics.IOStatistics; import org.apache.hadoop.fs.statistics.StoreStatisticNames; import org.apache.hadoop.io.IOUtils; - +import org.apache.hadoop.fs.azurebfs.services.AbfsReadFooterMetrics; import static org.apache.hadoop.fs.statistics.IOStatisticAssertions.extractStatistics; import static org.apache.hadoop.fs.statistics.IOStatisticAssertions.lookupMeanStatistic; import static org.apache.hadoop.fs.statistics.IOStatisticsLogging.ioStatisticsToPrettyString; @@ -263,6 +263,7 @@ public void testWithNullStreamStatistics() throws IOException { getConfiguration().getSasTokenRenewPeriodForStreamsInSeconds()) .withReadBufferSize(getConfiguration().getReadBufferSize()) .withReadAheadQueueDepth(getConfiguration().getReadAheadQueueDepth()) + .withReadFooterMetrics(new AbfsReadFooterMetrics()) .withStreamStatistics(null) .withReadAheadRange(getConfiguration().getReadAheadRange()) .build(); diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsReadFooterMetrics.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsReadFooterMetrics.java new file mode 100644 index 0000000000000..b8a69f0b6e201 --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsReadFooterMetrics.java @@ -0,0 +1,78 @@ +package org.apache.hadoop.fs.azurebfs; + +import static org.apache.hadoop.fs.CommonConfigurationKeys.IOSTATISTICS_LOGGING_LEVEL_INFO; +import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_TRACINGMETRICHEADER_FORMAT; +import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.MIN_BUFFER_SIZE; +import org.junit.Test; +import java.util.Random; +import java.util.List; +import org.apache.hadoop.fs.azurebfs.services.AbfsReadFooterMetrics; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.azurebfs.utils.TracingHeaderFormat; +import org.apache.hadoop.fs.statistics.IOStatisticsLogging; +import org.apache.hadoop.fs.statistics.IOStatisticsSource; +import org.apache.hadoop.fs.azurebfs.services.AbfsInputStream; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.azurebfs.utils.TracingHeaderValidator; +import org.apache.hadoop.fs.azurebfs.constants.FSOperationType; + +public class ITestAbfsReadFooterMetrics extends AbstractAbfsScaleTest { + + public ITestAbfsReadFooterMetrics() throws Exception { + } + + private static final String TEST_PATH = "/testfile"; + + @Test + public void testReadFooterMetrics() throws Exception { + int bufferSize = MIN_BUFFER_SIZE; + final AzureBlobFileSystem fs = getFileSystem(); + final AbfsConfiguration abfsConfiguration = fs.getAbfsStore().getAbfsConfiguration(); + abfsConfiguration.set(FS_AZURE_TRACINGMETRICHEADER_FORMAT, String.valueOf(TracingHeaderFormat.INTERNAL_FOOTER_METRIC_FORMAT)); + abfsConfiguration.setWriteBufferSize(bufferSize); + abfsConfiguration.setReadBufferSize(bufferSize); + + final byte[] b = new byte[2 * bufferSize]; + new Random().nextBytes(b); + + Path testPath = path(TEST_PATH); + FSDataOutputStream stream = fs.create(testPath); + try { + stream.write(b); + } finally{ + stream.close(); + } + IOStatisticsLogging.logIOStatisticsAtLevel(LOG, IOSTATISTICS_LOGGING_LEVEL_INFO, stream); + + final byte[] readBuffer = new byte[2 * bufferSize]; + int result; + IOStatisticsSource statisticsSource = null; + try (FSDataInputStream inputStream = fs.open(testPath)) { + statisticsSource = inputStream; + ((AbfsInputStream) inputStream.getWrappedStream()).registerListener( + new TracingHeaderValidator(abfsConfiguration.getClientCorrelationId(), + fs.getFileSystemId(), FSOperationType.READ, true, 0, + ((AbfsInputStream) inputStream.getWrappedStream()) + .getStreamID())); + inputStream.seek(bufferSize); + result = inputStream.read(readBuffer, bufferSize, bufferSize); + assertNotEquals(-1, result); + + //to test tracingHeader for case with bypassReadAhead == true + inputStream.seek(0); + byte[] temp = new byte[5]; + int t = inputStream.read(temp, 0, 1); + + inputStream.seek(0); + result = inputStream.read(readBuffer, 0, bufferSize); + } + IOStatisticsLogging.logIOStatisticsAtLevel(LOG, IOSTATISTICS_LOGGING_LEVEL_INFO, statisticsSource); + + assertNotEquals("data read in final read()", -1, result); + assertArrayEquals(readBuffer, b); + List abfsReadFooterMetricsList = fs.getAbfsClient().getAbfsCounters().getAbfsReadFooterMetrics(); + String footerMetric = AbfsReadFooterMetrics.getFooterMetrics(abfsReadFooterMetricsList, ""); + assertEquals("NonParquet: #FR=16384.000_16384.000 #SR=1.000_16384.000 #FL=32768.000 #RL=16384.000 ", footerMetric); + } +} diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsInputStream.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsInputStream.java index b5ae9b737842d..3c0576ee6a520 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsInputStream.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsInputStream.java @@ -112,7 +112,7 @@ private AbfsInputStream getAbfsInputStream(AbfsClient mockAbfsClient, null, FORWARD_SLASH + fileName, THREE_KB, - inputStreamContext.withReadBufferSize(ONE_KB).withReadAheadQueueDepth(10).withReadAheadBlockSize(ONE_KB), + inputStreamContext.withReadBufferSize(ONE_KB).withReadAheadQueueDepth(10).withReadAheadBlockSize(ONE_KB).withReadFooterMetrics(new AbfsReadFooterMetrics()), "eTag", getTestTracingContext(null, false)); diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsRestOperation.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsRestOperation.java index 954fcf8cf3aee..9018946da96e9 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsRestOperation.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsRestOperation.java @@ -47,7 +47,7 @@ public void testBackoffRetryMetrics() throws Exception { ArrayList retryCounts = new ArrayList<>(Arrays.asList(35, 28, 31, 45, 10, 2, 9)); int statusCode = HttpURLConnection.HTTP_UNAVAILABLE; - Method getMetrics = AbfsRestOperation.class.getDeclaredMethod("updateDriverMetrics", int.class, int.class); + Method getMetrics = AbfsRestOperation.class.getDeclaredMethod("updateBackoffMetrics", int.class, int.class); getMetrics.setAccessible(true); for (int retryCount : retryCounts) { getMetrics.invoke(op, retryCount, statusCode); diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/TestMockHelpers.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/TestMockHelpers.java index e25a099a00ef3..e69de29bb2d1d 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/TestMockHelpers.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/TestMockHelpers.java @@ -1,59 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs.azurebfs.utils; - -import java.lang.reflect.Field; -import java.lang.reflect.Modifier; - -/** - * Test Mock Helpers. - */ -public final class TestMockHelpers { - - /** - * Sets a class field by reflection. - * @param type - * @param obj - * @param fieldName - * @param fieldObject - * @param - * @return - * @throws Exception - */ - public static T setClassField( - Class type, - final T obj, - final String fieldName, - Object fieldObject) throws Exception { - - Field field = type.getDeclaredField(fieldName); - field.setAccessible(true); - Field modifiersField = Field.class.getDeclaredField("modifiers"); - modifiersField.setAccessible(true); - modifiersField.setInt(field, - field.getModifiers() & ~Modifier.FINAL); - field.set(obj, fieldObject); - - return obj; - } - - private TestMockHelpers() { - // Not called. - For checkstyle: HideUtilityClassConstructor - } -} From 5f680d08bd4bb59386f8b006d3c1f8cc75ad646f Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Fri, 12 Aug 2022 15:24:15 +0530 Subject: [PATCH 38/42] Test for footer metrics --- .../apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java index 14aa650c757b0..175191e90cdb5 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/AbfsRestOperation.java @@ -362,7 +362,7 @@ private boolean executeHttpOperation(final int retryCount, hostname = httpOperation.getHost(); LOG.warn("Unknown host name: {}. Retrying to resolve the host name...", hostname); - abfsBackoffMetrics.getNumberOfRequestsFailed().getAndIncrement(); + abfsBackoffMetrics.getNumberOfNetworkFailedRequests().getAndIncrement(); if (!client.getRetryPolicy().shouldRetry(retryCount, -1)) { updateBackoffMetrics(retryCount, httpOperation.getStatusCode()); throw new InvalidAbfsRestOperationException(ex); From a0d3ef90ef1bc03d8a24475b0ed81728b32aeeda Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Fri, 12 Aug 2022 19:35:42 +0530 Subject: [PATCH 39/42] Checkstyle fixes --- .../azurebfs/utils/TracingMetricContext.java | 18 ++++++ .../azurebfs/ITestAbfsReadFooterMetrics.java | 18 ++++++ .../services/TestAbfsInputStream.java | 3 +- .../fs/azurebfs/utils/TestMockHelpers.java | 59 +++++++++++++++++++ 4 files changed, 97 insertions(+), 1 deletion(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/TracingMetricContext.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/TracingMetricContext.java index dc1a6e04fe044..ba25110893c11 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/TracingMetricContext.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/TracingMetricContext.java @@ -1,3 +1,21 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.hadoop.fs.azurebfs.utils; import org.apache.hadoop.fs.azurebfs.services.AbfsCounters; diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsReadFooterMetrics.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsReadFooterMetrics.java index b8a69f0b6e201..662f9e051644a 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsReadFooterMetrics.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsReadFooterMetrics.java @@ -1,3 +1,21 @@ + +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.hadoop.fs.azurebfs; import static org.apache.hadoop.fs.CommonConfigurationKeys.IOSTATISTICS_LOGGING_LEVEL_INFO; diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsInputStream.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsInputStream.java index 3c0576ee6a520..e031f64a95652 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsInputStream.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestAbfsInputStream.java @@ -112,7 +112,8 @@ private AbfsInputStream getAbfsInputStream(AbfsClient mockAbfsClient, null, FORWARD_SLASH + fileName, THREE_KB, - inputStreamContext.withReadBufferSize(ONE_KB).withReadAheadQueueDepth(10).withReadAheadBlockSize(ONE_KB).withReadFooterMetrics(new AbfsReadFooterMetrics()), + inputStreamContext.withReadBufferSize(ONE_KB).withReadAheadQueueDepth(10).withReadAheadBlockSize(ONE_KB). + withReadFooterMetrics(new AbfsReadFooterMetrics()), "eTag", getTestTracingContext(null, false)); diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/TestMockHelpers.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/TestMockHelpers.java index e69de29bb2d1d..4500403e6b38a 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/TestMockHelpers.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/TestMockHelpers.java @@ -0,0 +1,59 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azurebfs.utils; + +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; + +/** + * Test Mock Helpers. + */ +public final class TestMockHelpers { + + /** + * Sets a class field by reflection. + * @param type + * @param obj + * @param fieldName + * @param fieldObject + * @param + * @return + * @throws Exception + */ + public static T setClassField( + Class type, + final T obj, + final String fieldName, + Object fieldObject) throws Exception { + + Field field = type.getDeclaredField(fieldName); + field.setAccessible(true); + Field modifiersField = Field.class.getDeclaredField("modifiers"); + modifiersField.setAccessible(true); + modifiersField.setInt(field, + field.getModifiers() & ~Modifier.FINAL); + field.set(obj, fieldObject); + + return obj; + } + + private TestMockHelpers() { + // Not called. - For checkstyle: HideUtilityClassConstructor + } +} \ No newline at end of file From 63abb7f3fafa20afcac6713171eeab1caaea261f Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Fri, 12 Aug 2022 19:58:53 +0530 Subject: [PATCH 40/42] Tracing header fix --- .../apache/hadoop/fs/azurebfs/utils/TracingMetricContext.java | 3 --- .../org/apache/hadoop/fs/azurebfs/utils/TestMockHelpers.java | 2 +- .../hadoop/fs/azurebfs/utils/TracingHeaderValidator.java | 3 --- 3 files changed, 1 insertion(+), 7 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/TracingMetricContext.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/TracingMetricContext.java index ba25110893c11..a99307e91158e 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/TracingMetricContext.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/TracingMetricContext.java @@ -79,9 +79,6 @@ public void constructHeader(AbfsHttpOperation httpOperation){ header = ""; break; } - if (listener != null) { //for testing - listener.callTracingHeaderValidator(header, tracingHeaderFormat); - } httpOperation.setRequestProperty(HttpHeaderConfigurations.X_MS_CLIENT_REQUEST_ID, header); } } diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/TestMockHelpers.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/TestMockHelpers.java index 4500403e6b38a..e25a099a00ef3 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/TestMockHelpers.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/TestMockHelpers.java @@ -56,4 +56,4 @@ public static T setClassField( private TestMockHelpers() { // Not called. - For checkstyle: HideUtilityClassConstructor } -} \ No newline at end of file +} diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/TracingHeaderValidator.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/TracingHeaderValidator.java index 4cd21906c077b..e195f1c381a94 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/TracingHeaderValidator.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/TracingHeaderValidator.java @@ -97,9 +97,6 @@ private void validateBasicFormat(String[] idList) { } else if (format == TracingHeaderFormat.TWO_ID_FORMAT) { Assertions.assertThat(idList) .describedAs("header should have 2 elements").hasSize(2); - } else if (format == TracingHeaderFormat.INTERNAL_METRIC_FORMAT) { - Assertions.assertThat(idList) - .describedAs("header should have 4 elements").hasSize(4); } else { Assertions.assertThat(idList).describedAs("header should have 1 element") .hasSize(1); From eb381e143eda068477f0c45dabb08377f70ba967 Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Tue, 16 Aug 2022 10:52:01 +0530 Subject: [PATCH 41/42] Removing unused fields --- .../apache/hadoop/fs/azurebfs/utils/TracingMetricContext.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/TracingMetricContext.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/TracingMetricContext.java index a99307e91158e..143e71745f62b 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/TracingMetricContext.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/TracingMetricContext.java @@ -34,7 +34,6 @@ public class TracingMetricContext extends TracingContext{ private final String clientCorrelationID; // passed over config by client private final String fileSystemID; // GUID for fileSystem instance private String clientRequestId = EMPTY_STRING; - private Listener listener = null; private TracingHeaderFormat tracingHeaderFormat; public TracingMetricContext(String clientCorrelationID, String fileSystemID, @@ -46,7 +45,6 @@ public TracingMetricContext(String clientCorrelationID, String fileSystemID, this.fileSystemID = fileSystemID; this.tracingHeaderFormat = tracingHeaderFormat; this.abfsCounters = abfsCounters; - this.listener = listener; } private String getFooterMetrics(){ From 40868de10dc26228d48b6ca1f5e535bbd173bdcd Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Tue, 16 Aug 2022 12:56:04 +0530 Subject: [PATCH 42/42] Removing unused fields --- .../hadoop/fs/azurebfs/utils/TracingContext.java | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/TracingContext.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/TracingContext.java index 21f6429433886..48a9337003ed5 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/TracingContext.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/utils/TracingContext.java @@ -60,7 +60,6 @@ public class TracingContext { private Listener listener = null; // null except when testing //final concatenated ID list set into x-ms-client-request-id header private String header = EMPTY_STRING; - private String metricResults = EMPTY_STRING; private static final Logger LOG = LoggerFactory.getLogger(AbfsClient.class); public static final int MAX_CLIENT_CORRELATION_ID_LENGTH = 72; public static final String CLIENT_CORRELATION_ID_PATTERN = "[a-zA-Z0-9-]*"; @@ -99,14 +98,6 @@ public TracingContext(String clientCorrelationID, String fileSystemID, } } - public TracingContext(String clientCorrelationID, String fileSystemID, - FSOperationType opType, boolean needsPrimaryReqId, - TracingHeaderFormat tracingHeaderFormat, Listener listener, - String metricResults) { - this(clientCorrelationID, fileSystemID, opType, needsPrimaryReqId, tracingHeaderFormat, listener); - this.metricResults = metricResults; - } - public TracingContext(TracingContext originalTracingContext) { this.fileSystemID = originalTracingContext.fileSystemID; this.streamID = originalTracingContext.streamID; @@ -129,10 +120,6 @@ public static String validateClientCorrelationID(String clientCorrelationID) { return clientCorrelationID; } - public String getMetricResults() { - return metricResults; - } - public void setPrimaryRequestID() { primaryRequestId = UUID.randomUUID().toString(); if (listener != null) {