From 1cd860e66498ddfb36b3f986cbee3717d4b384b8 Mon Sep 17 00:00:00 2001 From: Ankit Saurabh Date: Wed, 19 Oct 2022 17:17:48 +0100 Subject: [PATCH 01/20] Added the range of GetObject Request to the HttpReferer Header --- .../hadoop/fs/audit/AuditConstants.java | 6 +++ .../fs/s3a/audit/impl/LoggingAuditor.java | 41 ++++++++------- .../fs/s3a/audit/AbstractAuditingTest.java | 34 ++++++++---- .../audit/TestHttpReferrerAuditHeader.java | 52 ++++++++++++------- 4 files changed, 85 insertions(+), 48 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/audit/AuditConstants.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/audit/AuditConstants.java index 0929c2be03acf..ee76dd6041058 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/audit/AuditConstants.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/audit/AuditConstants.java @@ -90,6 +90,12 @@ private AuditConstants() { */ public static final String PARAM_PROCESS = "ps"; + /* + * Header : Start of range for GET request data : {@value }. + */ + public static final String RANGE = "r"; + + /** * Task Attempt ID query header: {@value}. */ diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/LoggingAuditor.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/LoggingAuditor.java index da1f5b59bdc36..0dbe4e5881f78 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/LoggingAuditor.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/LoggingAuditor.java @@ -18,16 +18,8 @@ package org.apache.hadoop.fs.s3a.audit.impl; -import javax.annotation.Nullable; -import java.io.IOException; -import java.util.Collection; -import java.util.HashMap; -import java.util.Map; - import com.amazonaws.AmazonWebServiceRequest; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - +import com.amazonaws.services.s3.model.GetObjectRequest; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.audit.AuditConstants; @@ -37,20 +29,20 @@ import org.apache.hadoop.fs.s3a.audit.AuditSpanS3A; import org.apache.hadoop.fs.store.audit.HttpReferrerAuditHeader; import org.apache.hadoop.security.UserGroupInformation; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -import static org.apache.hadoop.fs.audit.AuditConstants.PARAM_FILESYSTEM_ID; -import static org.apache.hadoop.fs.audit.AuditConstants.PARAM_PRINCIPAL; -import static org.apache.hadoop.fs.audit.AuditConstants.PARAM_THREAD0; -import static org.apache.hadoop.fs.audit.AuditConstants.PARAM_TIMESTAMP; +import javax.annotation.Nullable; +import java.io.IOException; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; + +import static org.apache.hadoop.fs.audit.AuditConstants.*; import static org.apache.hadoop.fs.audit.CommonAuditContext.currentAuditContext; import static org.apache.hadoop.fs.audit.CommonAuditContext.currentThreadID; import static org.apache.hadoop.fs.s3a.audit.AWSRequestAnalyzer.isRequestNotAlwaysInSpan; -import static org.apache.hadoop.fs.s3a.audit.S3AAuditConstants.OUTSIDE_SPAN; -import static org.apache.hadoop.fs.s3a.audit.S3AAuditConstants.REFERRER_HEADER_ENABLED; -import static org.apache.hadoop.fs.s3a.audit.S3AAuditConstants.REFERRER_HEADER_ENABLED_DEFAULT; -import static org.apache.hadoop.fs.s3a.audit.S3AAuditConstants.REFERRER_HEADER_FILTER; -import static org.apache.hadoop.fs.s3a.audit.S3AAuditConstants.REJECT_OUT_OF_SPAN_OPERATIONS; -import static org.apache.hadoop.fs.s3a.audit.S3AAuditConstants.UNAUDITED_OPERATION; +import static org.apache.hadoop.fs.s3a.audit.S3AAuditConstants.*; import static org.apache.hadoop.fs.s3a.commit.CommitUtils.extractJobID; import static org.apache.hadoop.fs.s3a.impl.HeaderProcessing.HEADER_REFERRER; @@ -95,6 +87,7 @@ public class LoggingAuditor */ private boolean headerEnabled; + /** * This is the header sent by the last S3 operation through * this auditor. @@ -238,6 +231,7 @@ private LoggingAuditSpan( final CommonAuditContext context, final String path1, final String path2) { + super(spanId, operationName); this.referrer = HttpReferrerAuditHeader.builder() @@ -314,8 +308,16 @@ public void set(final String key, final String value) { @Override public T beforeExecution( final T request) { + // Adding range to the header when there is GetObjectRequest + if(request instanceof GetObjectRequest && ((GetObjectRequest) request).getRange() != null) { + long[] rangeValue = ((GetObjectRequest) request).getRange(); + String rangeKey = "r"; + String combinedRangeValue = "bytes=" + rangeValue[0] + "-" + rangeValue[1]; + referrer.set(rangeKey, combinedRangeValue); + } // build the referrer header final String header = referrer.buildHttpReferrer(); + // update the outer class's field. setLastHeader(header); if (headerEnabled) { @@ -410,7 +412,6 @@ public T requestCreated( @Override public T beforeExecution( final T request) { - String error = "executing a request outside an audit span " + analyzer.analyze(request); final String unaudited = getSpanId() + " " diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/AbstractAuditingTest.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/AbstractAuditingTest.java index c76e3fa968f92..67de979591b9f 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/AbstractAuditingTest.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/AbstractAuditingTest.java @@ -18,15 +18,8 @@ package org.apache.hadoop.fs.s3a.audit; -import java.io.IOException; -import java.util.Map; - import com.amazonaws.services.s3.model.GetObjectMetadataRequest; -import org.junit.After; -import org.junit.Before; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - +import com.amazonaws.services.s3.model.GetObjectRequest; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.s3a.Statistic; import org.apache.hadoop.fs.s3a.api.RequestFactory; @@ -35,10 +28,17 @@ import org.apache.hadoop.fs.statistics.impl.IOStatisticsStore; import org.apache.hadoop.fs.store.audit.AuditSpan; import org.apache.hadoop.test.AbstractHadoopTestBase; +import org.junit.After; +import org.junit.Before; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.Map; import static org.apache.hadoop.fs.s3a.Statistic.INVOCATION_GET_FILE_STATUS; -import static org.apache.hadoop.fs.s3a.audit.S3AAuditConstants.UNAUDITED_OPERATION; import static org.apache.hadoop.fs.s3a.audit.AuditTestSupport.createIOStatisticsStoreForAuditing; +import static org.apache.hadoop.fs.s3a.audit.S3AAuditConstants.UNAUDITED_OPERATION; import static org.apache.hadoop.service.ServiceOperations.stopQuietly; import static org.apache.hadoop.test.LambdaTestUtils.intercept; import static org.assertj.core.api.Assertions.assertThat; @@ -138,6 +138,11 @@ protected GetObjectMetadataRequest head() { requestFactory.newGetObjectMetadataRequest("/")); } + protected GetObjectRequest get() { + return manager.beforeExecution( + requestFactory.newGetObjectRequest("/").withRange(8, 24)); + } + /** * Assert a head request fails as there is no * active span. @@ -210,4 +215,15 @@ protected void assertMapContains(final Map params, .isEqualTo(expected); } + /** + * Assert the map does not contain the key, i.e, it is null. + * @param params map of params + * @param key key + */ + protected void assertMapNotContains(final Map params, final String key) { + assertThat(params.get(key)) + .describedAs(key) + .isNull(); + } + } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/TestHttpReferrerAuditHeader.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/TestHttpReferrerAuditHeader.java index b653d24d4168a..13331d8ce2b10 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/TestHttpReferrerAuditHeader.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/TestHttpReferrerAuditHeader.java @@ -18,37 +18,29 @@ package org.apache.hadoop.fs.s3a.audit; -import java.net.URISyntaxException; -import java.util.Map; -import java.util.regex.Matcher; - import com.amazonaws.services.s3.model.GetObjectMetadataRequest; -import org.junit.Before; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - +import com.amazonaws.services.s3.model.GetObjectRequest; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.audit.CommonAuditContext; import org.apache.hadoop.fs.s3a.audit.impl.LoggingAuditor; import org.apache.hadoop.fs.store.audit.AuditSpan; -import org.apache.hadoop.fs.audit.CommonAuditContext; import org.apache.hadoop.fs.store.audit.HttpReferrerAuditHeader; import org.apache.hadoop.security.UserGroupInformation; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.net.URISyntaxException; +import java.util.Map; +import java.util.regex.Matcher; +import static org.apache.hadoop.fs.audit.AuditConstants.*; import static org.apache.hadoop.fs.s3a.audit.AuditTestSupport.loggingAuditConfig; import static org.apache.hadoop.fs.s3a.audit.S3AAuditConstants.REFERRER_HEADER_FILTER; import static org.apache.hadoop.fs.s3a.audit.S3LogParser.*; import static org.apache.hadoop.fs.s3a.impl.HeaderProcessing.HEADER_REFERRER; import static org.apache.hadoop.fs.store.audit.HttpReferrerAuditHeader.maybeStripWrappedQuotes; -import static org.apache.hadoop.fs.audit.AuditConstants.PARAM_FILESYSTEM_ID; -import static org.apache.hadoop.fs.audit.AuditConstants.PARAM_ID; -import static org.apache.hadoop.fs.audit.AuditConstants.PARAM_OP; -import static org.apache.hadoop.fs.audit.AuditConstants.PARAM_PATH; -import static org.apache.hadoop.fs.audit.AuditConstants.PARAM_PATH2; -import static org.apache.hadoop.fs.audit.AuditConstants.PARAM_PRINCIPAL; -import static org.apache.hadoop.fs.audit.AuditConstants.PARAM_THREAD0; -import static org.apache.hadoop.fs.audit.AuditConstants.PARAM_THREAD1; -import static org.apache.hadoop.fs.audit.AuditConstants.PARAM_TIMESTAMP; import static org.assertj.core.api.Assertions.assertThat; /** @@ -115,6 +107,7 @@ public void testHttpReferrerPatchesTheRequest() throws Throwable { assertThat(span.getTimestamp()) .describedAs("Timestamp of " + span) .isEqualTo(ts); + assertMapNotContains(params, RANGE); assertMapContains(params, PARAM_TIMESTAMP, Long.toString(ts)); @@ -179,6 +172,7 @@ public void testHeaderFiltering() throws Throwable { } + /** * A real log entry. * This is derived from a real log entry on a test run. @@ -309,6 +303,26 @@ public void testStripWrappedQuotes() throws Throwable { expectStrippedField("\"\"\"b\"", "b"); } + /** + * Verify that correct range is getting published in header + */ + @Test + public void testGetObjectRange() throws Throwable { + AuditSpan span = span(); + long ts = span.getTimestamp(); + GetObjectRequest request = get(); + Map headers + = request.getCustomRequestHeaders(); + assertThat(headers) + .describedAs("Custom headers") + .containsKey(HEADER_REFERRER); + String header = headers.get(HEADER_REFERRER); + LOG.info("Header is {}", header); + Map params + = HttpReferrerAuditHeader.extractQueryParameters(header); + assertMapContains(params, RANGE, "bytes=8-24"); + } + /** * Expect a field with quote stripping to match the expected value. * @param str string to strip From cdc06856e7491505756c7fcc297948abc026c5f6 Mon Sep 17 00:00:00 2001 From: Ankit Saurabh Date: Wed, 26 Oct 2022 17:35:28 +0100 Subject: [PATCH 02/20] Rearranged the import statements and a few other minor changes --- .../hadoop/fs/audit/AuditConstants.java | 6 +-- .../fs/s3a/audit/impl/LoggingAuditor.java | 53 ++++++++++++------- .../fs/s3a/audit/AbstractAuditingTest.java | 23 ++++---- .../audit/TestHttpReferrerAuditHeader.java | 37 ++++++++----- 4 files changed, 74 insertions(+), 45 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/audit/AuditConstants.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/audit/AuditConstants.java index ee76dd6041058..82de0d57e0b6a 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/audit/AuditConstants.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/audit/AuditConstants.java @@ -90,10 +90,10 @@ private AuditConstants() { */ public static final String PARAM_PROCESS = "ps"; - /* - * Header : Start of range for GET request data : {@value }. + /** + * Header: Range for GET request data : {@value }. */ - public static final String RANGE = "r"; + public static final String PARAM_RANGE = "r"; /** diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/LoggingAuditor.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/LoggingAuditor.java index 0dbe4e5881f78..e795fb4666e94 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/LoggingAuditor.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/LoggingAuditor.java @@ -18,8 +18,17 @@ package org.apache.hadoop.fs.s3a.audit.impl; +import javax.annotation.Nullable; +import java.io.IOException; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; + import com.amazonaws.AmazonWebServiceRequest; import com.amazonaws.services.s3.model.GetObjectRequest; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.audit.AuditConstants; @@ -29,20 +38,20 @@ import org.apache.hadoop.fs.s3a.audit.AuditSpanS3A; import org.apache.hadoop.fs.store.audit.HttpReferrerAuditHeader; import org.apache.hadoop.security.UserGroupInformation; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.annotation.Nullable; -import java.io.IOException; -import java.util.Collection; -import java.util.HashMap; -import java.util.Map; -import static org.apache.hadoop.fs.audit.AuditConstants.*; +import static org.apache.hadoop.fs.audit.AuditConstants.PARAM_FILESYSTEM_ID; +import static org.apache.hadoop.fs.audit.AuditConstants.PARAM_PRINCIPAL; +import static org.apache.hadoop.fs.audit.AuditConstants.PARAM_THREAD0; +import static org.apache.hadoop.fs.audit.AuditConstants.PARAM_TIMESTAMP; import static org.apache.hadoop.fs.audit.CommonAuditContext.currentAuditContext; import static org.apache.hadoop.fs.audit.CommonAuditContext.currentThreadID; import static org.apache.hadoop.fs.s3a.audit.AWSRequestAnalyzer.isRequestNotAlwaysInSpan; -import static org.apache.hadoop.fs.s3a.audit.S3AAuditConstants.*; +import static org.apache.hadoop.fs.s3a.audit.S3AAuditConstants.OUTSIDE_SPAN; +import static org.apache.hadoop.fs.s3a.audit.S3AAuditConstants.REFERRER_HEADER_ENABLED; +import static org.apache.hadoop.fs.s3a.audit.S3AAuditConstants.REFERRER_HEADER_ENABLED_DEFAULT; +import static org.apache.hadoop.fs.s3a.audit.S3AAuditConstants.REFERRER_HEADER_FILTER; +import static org.apache.hadoop.fs.s3a.audit.S3AAuditConstants.REJECT_OUT_OF_SPAN_OPERATIONS; +import static org.apache.hadoop.fs.s3a.audit.S3AAuditConstants.UNAUDITED_OPERATION; import static org.apache.hadoop.fs.s3a.commit.CommitUtils.extractJobID; import static org.apache.hadoop.fs.s3a.impl.HeaderProcessing.HEADER_REFERRER; @@ -87,7 +96,6 @@ public class LoggingAuditor */ private boolean headerEnabled; - /** * This is the header sent by the last S3 operation through * this auditor. @@ -223,6 +231,18 @@ private class LoggingAuditSpan extends AbstractAuditSpanImpl { private final HttpReferrerAuditHeader referrer; + /** + * Attach Range of data for GetObject Request + * @param request given get object request + */ + private void attachRangeFromRequest(AmazonWebServiceRequest request) { + if (request instanceof GetObjectRequest && ((GetObjectRequest) request).getRange() != null) { + long[] rangeValue = ((GetObjectRequest) request).getRange(); + String combinedRangeValue = String.format("bytes=%d-%d", rangeValue[0], rangeValue[1]); + referrer.set(AuditConstants.PARAM_RANGE, combinedRangeValue); + } + } + private final String description; private LoggingAuditSpan( @@ -231,7 +251,6 @@ private LoggingAuditSpan( final CommonAuditContext context, final String path1, final String path2) { - super(spanId, operationName); this.referrer = HttpReferrerAuditHeader.builder() @@ -308,16 +327,10 @@ public void set(final String key, final String value) { @Override public T beforeExecution( final T request) { - // Adding range to the header when there is GetObjectRequest - if(request instanceof GetObjectRequest && ((GetObjectRequest) request).getRange() != null) { - long[] rangeValue = ((GetObjectRequest) request).getRange(); - String rangeKey = "r"; - String combinedRangeValue = "bytes=" + rangeValue[0] + "-" + rangeValue[1]; - referrer.set(rangeKey, combinedRangeValue); - } + // attach range for GetObject requests + attachRangeFromRequest(request); // build the referrer header final String header = referrer.buildHttpReferrer(); - // update the outer class's field. setLastHeader(header); if (headerEnabled) { diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/AbstractAuditingTest.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/AbstractAuditingTest.java index 67de979591b9f..d299c7fe43a74 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/AbstractAuditingTest.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/AbstractAuditingTest.java @@ -18,8 +18,16 @@ package org.apache.hadoop.fs.s3a.audit; +import java.io.IOException; +import java.util.Map; + import com.amazonaws.services.s3.model.GetObjectMetadataRequest; import com.amazonaws.services.s3.model.GetObjectRequest; + +import org.junit.After; +import org.junit.Before; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.s3a.Statistic; import org.apache.hadoop.fs.s3a.api.RequestFactory; @@ -28,14 +36,6 @@ import org.apache.hadoop.fs.statistics.impl.IOStatisticsStore; import org.apache.hadoop.fs.store.audit.AuditSpan; import org.apache.hadoop.test.AbstractHadoopTestBase; -import org.junit.After; -import org.junit.Before; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.util.Map; - import static org.apache.hadoop.fs.s3a.Statistic.INVOCATION_GET_FILE_STATUS; import static org.apache.hadoop.fs.s3a.audit.AuditTestSupport.createIOStatisticsStoreForAuditing; import static org.apache.hadoop.fs.s3a.audit.S3AAuditConstants.UNAUDITED_OPERATION; @@ -138,7 +138,12 @@ protected GetObjectMetadataRequest head() { requestFactory.newGetObjectMetadataRequest("/")); } - protected GetObjectRequest get() { + /** + * Create a GetObject request with specified range and pass it through the + * manager's beforeExecution() + * @return the processed request + */ + protected GetObjectRequest rangedGet() { return manager.beforeExecution( requestFactory.newGetObjectRequest("/").withRange(8, 24)); } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/TestHttpReferrerAuditHeader.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/TestHttpReferrerAuditHeader.java index 13331d8ce2b10..77b0503f0321e 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/TestHttpReferrerAuditHeader.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/TestHttpReferrerAuditHeader.java @@ -20,12 +20,6 @@ import com.amazonaws.services.s3.model.GetObjectMetadataRequest; import com.amazonaws.services.s3.model.GetObjectRequest; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.audit.CommonAuditContext; -import org.apache.hadoop.fs.s3a.audit.impl.LoggingAuditor; -import org.apache.hadoop.fs.store.audit.AuditSpan; -import org.apache.hadoop.fs.store.audit.HttpReferrerAuditHeader; -import org.apache.hadoop.security.UserGroupInformation; import org.junit.Before; import org.junit.Test; import org.slf4j.Logger; @@ -35,10 +29,29 @@ import java.util.Map; import java.util.regex.Matcher; -import static org.apache.hadoop.fs.audit.AuditConstants.*; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.audit.CommonAuditContext; +import org.apache.hadoop.fs.s3a.audit.impl.LoggingAuditor; +import org.apache.hadoop.fs.store.audit.AuditSpan; +import org.apache.hadoop.fs.store.audit.HttpReferrerAuditHeader; +import org.apache.hadoop.security.UserGroupInformation; + +import static org.apache.hadoop.fs.audit.AuditConstants.PARAM_FILESYSTEM_ID; +import static org.apache.hadoop.fs.audit.AuditConstants.PARAM_ID; +import static org.apache.hadoop.fs.audit.AuditConstants.PARAM_OP; +import static org.apache.hadoop.fs.audit.AuditConstants.PARAM_PATH; +import static org.apache.hadoop.fs.audit.AuditConstants.PARAM_PATH2; +import static org.apache.hadoop.fs.audit.AuditConstants.PARAM_PRINCIPAL; +import static org.apache.hadoop.fs.audit.AuditConstants.PARAM_RANGE; +import static org.apache.hadoop.fs.audit.AuditConstants.PARAM_THREAD0; +import static org.apache.hadoop.fs.audit.AuditConstants.PARAM_THREAD1; +import static org.apache.hadoop.fs.audit.AuditConstants.PARAM_TIMESTAMP; import static org.apache.hadoop.fs.s3a.audit.AuditTestSupport.loggingAuditConfig; import static org.apache.hadoop.fs.s3a.audit.S3AAuditConstants.REFERRER_HEADER_FILTER; -import static org.apache.hadoop.fs.s3a.audit.S3LogParser.*; +import static org.apache.hadoop.fs.s3a.audit.S3LogParser.AWS_LOG_REGEXP_GROUPS; +import static org.apache.hadoop.fs.s3a.audit.S3LogParser.LOG_ENTRY_PATTERN; +import static org.apache.hadoop.fs.s3a.audit.S3LogParser.REFERRER_GROUP; +import static org.apache.hadoop.fs.s3a.audit.S3LogParser.VERB_GROUP; import static org.apache.hadoop.fs.s3a.impl.HeaderProcessing.HEADER_REFERRER; import static org.apache.hadoop.fs.store.audit.HttpReferrerAuditHeader.maybeStripWrappedQuotes; import static org.assertj.core.api.Assertions.assertThat; @@ -107,7 +120,7 @@ public void testHttpReferrerPatchesTheRequest() throws Throwable { assertThat(span.getTimestamp()) .describedAs("Timestamp of " + span) .isEqualTo(ts); - assertMapNotContains(params, RANGE); + assertMapNotContains(params, PARAM_RANGE); assertMapContains(params, PARAM_TIMESTAMP, Long.toString(ts)); @@ -172,7 +185,6 @@ public void testHeaderFiltering() throws Throwable { } - /** * A real log entry. * This is derived from a real log entry on a test run. @@ -309,8 +321,7 @@ public void testStripWrappedQuotes() throws Throwable { @Test public void testGetObjectRange() throws Throwable { AuditSpan span = span(); - long ts = span.getTimestamp(); - GetObjectRequest request = get(); + GetObjectRequest request = rangedGet(); Map headers = request.getCustomRequestHeaders(); assertThat(headers) @@ -320,7 +331,7 @@ public void testGetObjectRange() throws Throwable { LOG.info("Header is {}", header); Map params = HttpReferrerAuditHeader.extractQueryParameters(header); - assertMapContains(params, RANGE, "bytes=8-24"); + assertMapContains(params, PARAM_RANGE, "bytes=8-24"); } /** From 9d2fd4316dbed3ff6fccf00888550add96f04b26 Mon Sep 17 00:00:00 2001 From: Ankit Saurabh Date: Wed, 26 Oct 2022 17:43:37 +0100 Subject: [PATCH 03/20] Removed blank lines --- .../fs/s3a/audit/impl/LoggingAuditor.java | 1 + .../fs/s3a/audit/AbstractAuditingTest.java | 5 ++-- .../audit/TestHttpReferrerAuditHeader.java | 26 +++++++++---------- 3 files changed, 17 insertions(+), 15 deletions(-) diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/LoggingAuditor.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/LoggingAuditor.java index e795fb4666e94..0ca25c476a214 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/LoggingAuditor.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/LoggingAuditor.java @@ -425,6 +425,7 @@ public T requestCreated( @Override public T beforeExecution( final T request) { + String error = "executing a request outside an audit span " + analyzer.analyze(request); final String unaudited = getSpanId() + " " diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/AbstractAuditingTest.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/AbstractAuditingTest.java index d299c7fe43a74..6f1c888f24d58 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/AbstractAuditingTest.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/AbstractAuditingTest.java @@ -23,11 +23,11 @@ import com.amazonaws.services.s3.model.GetObjectMetadataRequest; import com.amazonaws.services.s3.model.GetObjectRequest; - import org.junit.After; import org.junit.Before; import org.slf4j.Logger; import org.slf4j.LoggerFactory; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.s3a.Statistic; import org.apache.hadoop.fs.s3a.api.RequestFactory; @@ -36,9 +36,10 @@ import org.apache.hadoop.fs.statistics.impl.IOStatisticsStore; import org.apache.hadoop.fs.store.audit.AuditSpan; import org.apache.hadoop.test.AbstractHadoopTestBase; + import static org.apache.hadoop.fs.s3a.Statistic.INVOCATION_GET_FILE_STATUS; -import static org.apache.hadoop.fs.s3a.audit.AuditTestSupport.createIOStatisticsStoreForAuditing; import static org.apache.hadoop.fs.s3a.audit.S3AAuditConstants.UNAUDITED_OPERATION; +import static org.apache.hadoop.fs.s3a.audit.AuditTestSupport.createIOStatisticsStoreForAuditing; import static org.apache.hadoop.service.ServiceOperations.stopQuietly; import static org.apache.hadoop.test.LambdaTestUtils.intercept; import static org.assertj.core.api.Assertions.assertThat; diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/TestHttpReferrerAuditHeader.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/TestHttpReferrerAuditHeader.java index 77b0503f0321e..9d81942a2576a 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/TestHttpReferrerAuditHeader.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/TestHttpReferrerAuditHeader.java @@ -18,6 +18,10 @@ package org.apache.hadoop.fs.s3a.audit; +import java.net.URISyntaxException; +import java.util.Map; +import java.util.regex.Matcher; + import com.amazonaws.services.s3.model.GetObjectMetadataRequest; import com.amazonaws.services.s3.model.GetObjectRequest; import org.junit.Before; @@ -25,17 +29,21 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.net.URISyntaxException; -import java.util.Map; -import java.util.regex.Matcher; - import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.audit.CommonAuditContext; import org.apache.hadoop.fs.s3a.audit.impl.LoggingAuditor; import org.apache.hadoop.fs.store.audit.AuditSpan; +import org.apache.hadoop.fs.audit.CommonAuditContext; import org.apache.hadoop.fs.store.audit.HttpReferrerAuditHeader; import org.apache.hadoop.security.UserGroupInformation; +import static org.apache.hadoop.fs.s3a.audit.AuditTestSupport.loggingAuditConfig; +import static org.apache.hadoop.fs.s3a.audit.S3AAuditConstants.REFERRER_HEADER_FILTER; +import static org.apache.hadoop.fs.s3a.audit.S3LogParser.AWS_LOG_REGEXP_GROUPS; +import static org.apache.hadoop.fs.s3a.audit.S3LogParser.LOG_ENTRY_PATTERN; +import static org.apache.hadoop.fs.s3a.audit.S3LogParser.REFERRER_GROUP; +import static org.apache.hadoop.fs.s3a.audit.S3LogParser.VERB_GROUP; +import static org.apache.hadoop.fs.s3a.impl.HeaderProcessing.HEADER_REFERRER; +import static org.apache.hadoop.fs.store.audit.HttpReferrerAuditHeader.maybeStripWrappedQuotes; import static org.apache.hadoop.fs.audit.AuditConstants.PARAM_FILESYSTEM_ID; import static org.apache.hadoop.fs.audit.AuditConstants.PARAM_ID; import static org.apache.hadoop.fs.audit.AuditConstants.PARAM_OP; @@ -46,14 +54,6 @@ import static org.apache.hadoop.fs.audit.AuditConstants.PARAM_THREAD0; import static org.apache.hadoop.fs.audit.AuditConstants.PARAM_THREAD1; import static org.apache.hadoop.fs.audit.AuditConstants.PARAM_TIMESTAMP; -import static org.apache.hadoop.fs.s3a.audit.AuditTestSupport.loggingAuditConfig; -import static org.apache.hadoop.fs.s3a.audit.S3AAuditConstants.REFERRER_HEADER_FILTER; -import static org.apache.hadoop.fs.s3a.audit.S3LogParser.AWS_LOG_REGEXP_GROUPS; -import static org.apache.hadoop.fs.s3a.audit.S3LogParser.LOG_ENTRY_PATTERN; -import static org.apache.hadoop.fs.s3a.audit.S3LogParser.REFERRER_GROUP; -import static org.apache.hadoop.fs.s3a.audit.S3LogParser.VERB_GROUP; -import static org.apache.hadoop.fs.s3a.impl.HeaderProcessing.HEADER_REFERRER; -import static org.apache.hadoop.fs.store.audit.HttpReferrerAuditHeader.maybeStripWrappedQuotes; import static org.assertj.core.api.Assertions.assertThat; /** From 2d3eb86f56a511e1a6f76931fb89f78bb370d92e Mon Sep 17 00:00:00 2001 From: Ankit Saurabh Date: Wed, 26 Oct 2022 17:45:36 +0100 Subject: [PATCH 04/20] Removed blank lines --- .../hadoop/fs/s3a/audit/TestHttpReferrerAuditHeader.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/TestHttpReferrerAuditHeader.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/TestHttpReferrerAuditHeader.java index 9d81942a2576a..cf84269b54e90 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/TestHttpReferrerAuditHeader.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/TestHttpReferrerAuditHeader.java @@ -38,10 +38,7 @@ import static org.apache.hadoop.fs.s3a.audit.AuditTestSupport.loggingAuditConfig; import static org.apache.hadoop.fs.s3a.audit.S3AAuditConstants.REFERRER_HEADER_FILTER; -import static org.apache.hadoop.fs.s3a.audit.S3LogParser.AWS_LOG_REGEXP_GROUPS; -import static org.apache.hadoop.fs.s3a.audit.S3LogParser.LOG_ENTRY_PATTERN; -import static org.apache.hadoop.fs.s3a.audit.S3LogParser.REFERRER_GROUP; -import static org.apache.hadoop.fs.s3a.audit.S3LogParser.VERB_GROUP; +import static org.apache.hadoop.fs.s3a.audit.S3LogParser.*; import static org.apache.hadoop.fs.s3a.impl.HeaderProcessing.HEADER_REFERRER; import static org.apache.hadoop.fs.store.audit.HttpReferrerAuditHeader.maybeStripWrappedQuotes; import static org.apache.hadoop.fs.audit.AuditConstants.PARAM_FILESYSTEM_ID; From 216db7c59816fc38dad1d9781e7554d3bcdce6cc Mon Sep 17 00:00:00 2001 From: Ankit Saurabh Date: Wed, 26 Oct 2022 17:46:34 +0100 Subject: [PATCH 05/20] Removed blank lines --- .../main/java/org/apache/hadoop/fs/audit/AuditConstants.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/audit/AuditConstants.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/audit/AuditConstants.java index 82de0d57e0b6a..0799107efb684 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/audit/AuditConstants.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/audit/AuditConstants.java @@ -94,8 +94,7 @@ private AuditConstants() { * Header: Range for GET request data : {@value }. */ public static final String PARAM_RANGE = "r"; - - + /** * Task Attempt ID query header: {@value}. */ From 74ad5288455a6225e59c1614963c0eefddbc09e7 Mon Sep 17 00:00:00 2001 From: Ankit Saurabh Date: Thu, 27 Oct 2022 11:39:24 +0100 Subject: [PATCH 06/20] Audit Constants added --- .../main/java/org/apache/hadoop/fs/audit/AuditConstants.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/audit/AuditConstants.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/audit/AuditConstants.java index 0799107efb684..231ccc76e9eb2 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/audit/AuditConstants.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/audit/AuditConstants.java @@ -94,7 +94,7 @@ private AuditConstants() { * Header: Range for GET request data : {@value }. */ public static final String PARAM_RANGE = "r"; - + /** * Task Attempt ID query header: {@value}. */ From 7c8b3f18ed72454141c88986477447781e18387e Mon Sep 17 00:00:00 2001 From: Ankit Saurabh Date: Thu, 27 Oct 2022 11:43:20 +0100 Subject: [PATCH 07/20] Removed extra spaces --- .../main/java/org/apache/hadoop/fs/audit/AuditConstants.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/audit/AuditConstants.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/audit/AuditConstants.java index 231ccc76e9eb2..19694c44d2f49 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/audit/AuditConstants.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/audit/AuditConstants.java @@ -91,7 +91,7 @@ private AuditConstants() { public static final String PARAM_PROCESS = "ps"; /** - * Header: Range for GET request data : {@value }. + * Header: Range for GET request data : {@value}. */ public static final String PARAM_RANGE = "r"; From cf06569ff916fa22fde302e460cc4807f52b9cf3 Mon Sep 17 00:00:00 2001 From: Ankit Saurabh Date: Tue, 1 Nov 2022 11:00:13 +0000 Subject: [PATCH 08/20] Added the check for 2 elements in getObjectRequest range --- .../org/apache/hadoop/fs/audit/AuditConstants.java | 2 +- .../hadoop/fs/s3a/audit/impl/LoggingAuditor.java | 10 +++++++++- .../hadoop/fs/s3a/audit/AbstractAuditingTest.java | 2 +- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/audit/AuditConstants.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/audit/AuditConstants.java index 19694c44d2f49..fd0b5acf2ef1c 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/audit/AuditConstants.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/audit/AuditConstants.java @@ -91,7 +91,7 @@ private AuditConstants() { public static final String PARAM_PROCESS = "ps"; /** - * Header: Range for GET request data : {@value}. + * Header: Range for GET request data: {@value}. */ public static final String PARAM_RANGE = "r"; diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/LoggingAuditor.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/LoggingAuditor.java index 0ca25c476a214..f921ca4e2120e 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/LoggingAuditor.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/LoggingAuditor.java @@ -236,7 +236,15 @@ private class LoggingAuditSpan extends AbstractAuditSpanImpl { * @param request given get object request */ private void attachRangeFromRequest(AmazonWebServiceRequest request) { - if (request instanceof GetObjectRequest && ((GetObjectRequest) request).getRange() != null) { + if (request instanceof GetObjectRequest) + { + if (((GetObjectRequest) request).getRange().length != 2) { + try { + throw new Exception("Cannot get a range for the request"); + } catch (Exception e) { + throw new RuntimeException(e); + } + } long[] rangeValue = ((GetObjectRequest) request).getRange(); String combinedRangeValue = String.format("bytes=%d-%d", rangeValue[0], rangeValue[1]); referrer.set(AuditConstants.PARAM_RANGE, combinedRangeValue); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/AbstractAuditingTest.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/AbstractAuditingTest.java index 6f1c888f24d58..f706cec95bc2f 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/AbstractAuditingTest.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/AbstractAuditingTest.java @@ -226,7 +226,7 @@ protected void assertMapContains(final Map params, * @param params map of params * @param key key */ - protected void assertMapNotContains(final Map params, final String key) { + protected void assertMapNotContains(final Map params, final String key) { assertThat(params.get(key)) .describedAs(key) .isNull(); From 5150698bce738a0570c88097f191a565c735967f Mon Sep 17 00:00:00 2001 From: Ankit Saurabh Date: Thu, 3 Nov 2022 19:14:31 +0000 Subject: [PATCH 09/20] Added Consumer Interface to add the range --- .../fs/s3a/audit/impl/LoggingAuditor.java | 22 +++++++++++------- .../fs/s3a/audit/AbstractAuditingTest.java | 14 ++++++----- .../audit/TestHttpReferrerAuditHeader.java | 23 +++++++++++++++++-- 3 files changed, 43 insertions(+), 16 deletions(-) diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/LoggingAuditor.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/LoggingAuditor.java index f921ca4e2120e..559c8989e8356 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/LoggingAuditor.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/LoggingAuditor.java @@ -36,6 +36,7 @@ import org.apache.hadoop.fs.s3a.audit.AWSRequestAnalyzer; import org.apache.hadoop.fs.s3a.audit.AuditFailureException; import org.apache.hadoop.fs.s3a.audit.AuditSpanS3A; +import org.apache.hadoop.fs.store.LogExactlyOnce; import org.apache.hadoop.fs.store.audit.HttpReferrerAuditHeader; import org.apache.hadoop.security.UserGroupInformation; @@ -111,6 +112,14 @@ public class LoggingAuditor */ private Collection filters; + /** + * Log for warning of problems getting the range of GetObjectRequest + * will only log of a problem once per process instance. + * This is to avoid logs being flooded with errors. + */ + private static final LogExactlyOnce WARN_INCORRECT_RANGE = + new LogExactlyOnce(LOG); + /** * Create the auditor. * The UGI current user is used to provide the principal; @@ -236,18 +245,15 @@ private class LoggingAuditSpan extends AbstractAuditSpanImpl { * @param request given get object request */ private void attachRangeFromRequest(AmazonWebServiceRequest request) { - if (request instanceof GetObjectRequest) - { - if (((GetObjectRequest) request).getRange().length != 2) { - try { - throw new Exception("Cannot get a range for the request"); - } catch (Exception e) { - throw new RuntimeException(e); - } + if (request instanceof GetObjectRequest && ((GetObjectRequest) request).getRange()!=null) { + if(((GetObjectRequest) request).getRange().length!=2) { + WARN_INCORRECT_RANGE.warn("Range has unexpected number of elements"); + return; } long[] rangeValue = ((GetObjectRequest) request).getRange(); String combinedRangeValue = String.format("bytes=%d-%d", rangeValue[0], rangeValue[1]); referrer.set(AuditConstants.PARAM_RANGE, combinedRangeValue); + } } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/AbstractAuditingTest.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/AbstractAuditingTest.java index f706cec95bc2f..7675f3f93200e 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/AbstractAuditingTest.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/AbstractAuditingTest.java @@ -20,6 +20,7 @@ import java.io.IOException; import java.util.Map; +import java.util.function.Consumer; import com.amazonaws.services.s3.model.GetObjectMetadataRequest; import com.amazonaws.services.s3.model.GetObjectRequest; @@ -140,13 +141,14 @@ protected GetObjectMetadataRequest head() { } /** - * Create a GetObject request with specified range and pass it through the - * manager's beforeExecution() - * @return the processed request + * Create a GetObject request and modify it before passing it through auditor + * @param modifyRequest Consumer Interface for changing the request before passing to the auditor + * @return the request */ - protected GetObjectRequest rangedGet() { - return manager.beforeExecution( - requestFactory.newGetObjectRequest("/").withRange(8, 24)); + protected GetObjectRequest get(Consumer modifyRequest) { + GetObjectRequest req = requestFactory.newGetObjectRequest("/"); + modifyRequest.accept(req); + return manager.beforeExecution(req); } /** diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/TestHttpReferrerAuditHeader.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/TestHttpReferrerAuditHeader.java index cf84269b54e90..f7a0575d86bf1 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/TestHttpReferrerAuditHeader.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/TestHttpReferrerAuditHeader.java @@ -318,7 +318,7 @@ public void testStripWrappedQuotes() throws Throwable { @Test public void testGetObjectRange() throws Throwable { AuditSpan span = span(); - GetObjectRequest request = rangedGet(); + GetObjectRequest request = get(getObjectRequest -> getObjectRequest.setRange(100, 200)); Map headers = request.getCustomRequestHeaders(); assertThat(headers) @@ -328,7 +328,26 @@ public void testGetObjectRange() throws Throwable { LOG.info("Header is {}", header); Map params = HttpReferrerAuditHeader.extractQueryParameters(header); - assertMapContains(params, PARAM_RANGE, "bytes=8-24"); + assertMapContains(params, PARAM_RANGE, "bytes=100-200"); + } + + /** + * Verify that no range is getting added to the header in request without range + */ + @Test + public void testGetObjectWithoutRange() throws Throwable { + AuditSpan span = span(); + GetObjectRequest request = get(getObjectRequest -> {}); + Map headers + = request.getCustomRequestHeaders(); + assertThat(headers) + .describedAs("Custom headers") + .containsKey(HEADER_REFERRER); + String header = headers.get(HEADER_REFERRER); + LOG.info("Header is {}", header); + Map params + = HttpReferrerAuditHeader.extractQueryParameters(header); + assertMapNotContains(params, PARAM_RANGE); } /** From 6837be9fc5ae7b4001fe782a1e24cee82a17a53c Mon Sep 17 00:00:00 2001 From: Ankit Saurabh Date: Fri, 4 Nov 2022 11:14:28 +0000 Subject: [PATCH 10/20] Added the case for 0 elements in range --- .../hadoop/fs/s3a/audit/impl/LoggingAuditor.java | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/LoggingAuditor.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/LoggingAuditor.java index 559c8989e8356..8d988e69ffcf2 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/LoggingAuditor.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/LoggingAuditor.java @@ -245,15 +245,17 @@ private class LoggingAuditSpan extends AbstractAuditSpanImpl { * @param request given get object request */ private void attachRangeFromRequest(AmazonWebServiceRequest request) { - if (request instanceof GetObjectRequest && ((GetObjectRequest) request).getRange()!=null) { - if(((GetObjectRequest) request).getRange().length!=2) { - WARN_INCORRECT_RANGE.warn("Range has unexpected number of elements"); + if (request instanceof GetObjectRequest) { + long[] rangeValue = ((GetObjectRequest) request).getRange(); + if (rangeValue == null || rangeValue.length == 0) { return; } - long[] rangeValue = ((GetObjectRequest) request).getRange(); + if (rangeValue.length != 2) { + WARN_INCORRECT_RANGE.warn("Expected range to contain 0 or 2 elements. Got " + + rangeValue.length + ". Ignoring"); + } String combinedRangeValue = String.format("bytes=%d-%d", rangeValue[0], rangeValue[1]); referrer.set(AuditConstants.PARAM_RANGE, combinedRangeValue); - } } From 2be27c95a1d7111acd61750d7bb755e88f4eba69 Mon Sep 17 00:00:00 2001 From: Ankit Saurabh Date: Fri, 4 Nov 2022 12:04:37 +0000 Subject: [PATCH 11/20] Handling NullPointer Exception --- .../org/apache/hadoop/fs/s3a/audit/impl/LoggingAuditor.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/LoggingAuditor.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/LoggingAuditor.java index 8d988e69ffcf2..79d2472b8a688 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/LoggingAuditor.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/LoggingAuditor.java @@ -245,9 +245,9 @@ private class LoggingAuditSpan extends AbstractAuditSpanImpl { * @param request given get object request */ private void attachRangeFromRequest(AmazonWebServiceRequest request) { - if (request instanceof GetObjectRequest) { + if (request instanceof GetObjectRequest && ((GetObjectRequest) request).getRange()!=null) { long[] rangeValue = ((GetObjectRequest) request).getRange(); - if (rangeValue == null || rangeValue.length == 0) { + if (rangeValue.length == 0) { return; } if (rangeValue.length != 2) { From 620059c814fc910af75f27d4a2870b6d8e3e656e Mon Sep 17 00:00:00 2001 From: Ankit Saurabh Date: Fri, 4 Nov 2022 13:21:21 +0000 Subject: [PATCH 12/20] Added the return statement in case of unexpected elements in range --- .../org/apache/hadoop/fs/s3a/audit/impl/LoggingAuditor.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/LoggingAuditor.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/LoggingAuditor.java index 79d2472b8a688..66931f6ee8ce2 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/LoggingAuditor.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/LoggingAuditor.java @@ -252,7 +252,8 @@ private void attachRangeFromRequest(AmazonWebServiceRequest request) { } if (rangeValue.length != 2) { WARN_INCORRECT_RANGE.warn("Expected range to contain 0 or 2 elements. Got " - + rangeValue.length + ". Ignoring"); + + rangeValue.length + " elements. Ignoring"); + return; } String combinedRangeValue = String.format("bytes=%d-%d", rangeValue[0], rangeValue[1]); referrer.set(AuditConstants.PARAM_RANGE, combinedRangeValue); From 732fd000f69f3d241a41a92433f9beaf2fbe7151 Mon Sep 17 00:00:00 2001 From: Ankit Saurabh Date: Fri, 4 Nov 2022 15:28:44 +0000 Subject: [PATCH 13/20] Restructured the if statements to less complex --- .../org/apache/hadoop/fs/s3a/audit/impl/LoggingAuditor.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/LoggingAuditor.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/LoggingAuditor.java index 66931f6ee8ce2..f6320c6faa009 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/LoggingAuditor.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/LoggingAuditor.java @@ -245,9 +245,9 @@ private class LoggingAuditSpan extends AbstractAuditSpanImpl { * @param request given get object request */ private void attachRangeFromRequest(AmazonWebServiceRequest request) { - if (request instanceof GetObjectRequest && ((GetObjectRequest) request).getRange()!=null) { + if (request instanceof GetObjectRequest) { long[] rangeValue = ((GetObjectRequest) request).getRange(); - if (rangeValue.length == 0) { + if (rangeValue == null || rangeValue.length == 0) { return; } if (rangeValue.length != 2) { From deddad4394ac39c818e6ce3c1a82b35080071afd Mon Sep 17 00:00:00 2001 From: Ankit Saurabh Date: Mon, 7 Nov 2022 10:32:51 +0000 Subject: [PATCH 14/20] added period according to Checkstyle recommendation. --- .../org/apache/hadoop/fs/s3a/audit/impl/LoggingAuditor.java | 2 +- .../org/apache/hadoop/fs/s3a/audit/AbstractAuditingTest.java | 2 +- .../hadoop/fs/s3a/audit/TestHttpReferrerAuditHeader.java | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/LoggingAuditor.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/LoggingAuditor.java index f6320c6faa009..3fb005494426a 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/LoggingAuditor.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/LoggingAuditor.java @@ -241,7 +241,7 @@ private class LoggingAuditSpan extends AbstractAuditSpanImpl { private final HttpReferrerAuditHeader referrer; /** - * Attach Range of data for GetObject Request + * Attach Range of data for GetObject Request. * @param request given get object request */ private void attachRangeFromRequest(AmazonWebServiceRequest request) { diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/AbstractAuditingTest.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/AbstractAuditingTest.java index 7675f3f93200e..f5e5cd5e954af 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/AbstractAuditingTest.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/AbstractAuditingTest.java @@ -141,7 +141,7 @@ protected GetObjectMetadataRequest head() { } /** - * Create a GetObject request and modify it before passing it through auditor + * Create a GetObject request and modify it before passing it through auditor. * @param modifyRequest Consumer Interface for changing the request before passing to the auditor * @return the request */ diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/TestHttpReferrerAuditHeader.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/TestHttpReferrerAuditHeader.java index f7a0575d86bf1..8f1dab89264de 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/TestHttpReferrerAuditHeader.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/TestHttpReferrerAuditHeader.java @@ -313,7 +313,7 @@ public void testStripWrappedQuotes() throws Throwable { } /** - * Verify that correct range is getting published in header + * Verify that correct range is getting published in header. */ @Test public void testGetObjectRange() throws Throwable { @@ -332,7 +332,7 @@ public void testGetObjectRange() throws Throwable { } /** - * Verify that no range is getting added to the header in request without range + * Verify that no range is getting added to the header in request without range. */ @Test public void testGetObjectWithoutRange() throws Throwable { From 46d371cbe90436cc16f52f2da74ffda10dd8cb3a Mon Sep 17 00:00:00 2001 From: Ankit Saurabh Date: Wed, 23 Nov 2022 13:56:35 +0000 Subject: [PATCH 15/20] Added the documentation for range in referer header and removed bytes= from range --- .../hadoop/fs/audit/AuditConstants.java | 2 +- .../fs/s3a/audit/impl/LoggingAuditor.java | 5 +- .../markdown/tools/hadoop-aws/auditing.md | 54 ++++++++++--------- .../audit/TestHttpReferrerAuditHeader.java | 2 +- 4 files changed, 32 insertions(+), 31 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/audit/AuditConstants.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/audit/AuditConstants.java index fd0b5acf2ef1c..ffca6097c4733 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/audit/AuditConstants.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/audit/AuditConstants.java @@ -93,7 +93,7 @@ private AuditConstants() { /** * Header: Range for GET request data: {@value}. */ - public static final String PARAM_RANGE = "r"; + public static final String PARAM_RANGE = "rg"; /** * Task Attempt ID query header: {@value}. diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/LoggingAuditor.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/LoggingAuditor.java index 3fb005494426a..ed1c078ef42d0 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/LoggingAuditor.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/audit/impl/LoggingAuditor.java @@ -251,11 +251,10 @@ private void attachRangeFromRequest(AmazonWebServiceRequest request) { return; } if (rangeValue.length != 2) { - WARN_INCORRECT_RANGE.warn("Expected range to contain 0 or 2 elements. Got " - + rangeValue.length + " elements. Ignoring"); + WARN_INCORRECT_RANGE.warn("Expected range to contain 0 or 2 elements. Got {} elements. Ignoring.", rangeValue.length); return; } - String combinedRangeValue = String.format("bytes=%d-%d", rangeValue[0], rangeValue[1]); + String combinedRangeValue = String.format("%d-%d", rangeValue[0], rangeValue[1]); referrer.set(AuditConstants.PARAM_RANGE, combinedRangeValue); } } diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/auditing.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/auditing.md index 8ccc36cf83bb1..4932b9b92bf3e 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/auditing.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/auditing.md @@ -205,37 +205,39 @@ This can be parsed (consult AWS documentation for a regular expression) and the http referrer header extracted. ``` -https://audit.example.org/hadoop/1/op_rename/3c0d9b7e-2a63-43d9-a220-3c574d768ef3-3/ - ?op=op_rename - &p1=s3a://alice-london/path1 - &pr=alice - &p2=s3a://alice-london/path2 - &ps=235865a0-d399-4696-9978-64568db1b51c - &id=3c0d9b7e-2a63-43d9-a220-3c574d768ef3-3 - &t0=12 - &fs=af5943a9-b6f6-4eec-9c58-008982fc492a - &t1=12 - &ts=1617116985923 +https://audit.example.org/hadoop/1/op_get_file_status/baa4bd75-f4f5-4fe4-ab83-e85fbe2315a8-00000004/ + ?op=op_get_file_status + &p1=/path1 + &pr=sauraank + &p2=/path2 + &ps=53cf0dec-4ffe-4787-8875-5a6fb4411486 + &rg=100-200 + &id=baa4bd75-f4f5-4fe4-ab83-e85fbe2315a8-00000004 + &t0=13 + &fs=baa4bd75-f4f5-4fe4-ab83-e85fbe2315a8 + &t1=13 + &ts=1669140734400 ``` Here are the fields which may be found in a request. If any of the field values were `null`, the field is omitted. -| Name | Meaning | Example | -|------|---------|---------| -| `cm` | Command | `S3GuardTool$BucketInfo` | -| `fs` | FileSystem ID | `af5943a9-b6f6-4eec-9c58-008982fc492a` | -| `id` | Span ID | `3c0d9b7e-2a63-43d9-a220-3c574d768ef3-3` | -| `ji` | Job ID (S3A committer)| `(Generated by query engine)` | -| `op` | Filesystem API call | `op_rename` | -| `p1` | Path 1 of operation | `s3a://alice-london/path1` | -| `p2` | Path 2 of operation | `s3a://alice-london/path2` | -| `pr` | Principal | `alice` | -| `ps` | Unique process UUID | `235865a0-d399-4696-9978-64568db1b51c` | -| `ta` | Task Attempt ID (S3A committer) | | -| `t0` | Thread 0: thread span was created in | `100` | -| `t1` | Thread 1: thread this operation was executed in | `200` | -| `ts` | Timestamp (UTC epoch millis) | `1617116985923` | +| Name | Meaning | Example | +|------|-------------------------------------------------|-------------------------------------------------| +| `cm` | Command | `S3GuardTool$BucketInfo` | +| `fs` | FileSystem ID | `baa4bd75-f4f5-4fe4-ab83-e85fbe2315a8` | +| `id` | Span ID | `baa4bd75-f4f5-4fe4-ab83-e85fbe2315a8-00000004` | +| `ji` | Job ID (S3A committer) | `(Generated by query engine)` | +| `op` | Filesystem API call | `op_get_file_status` | +| `p1` | Path 1 of operation | `s3a://alice-london/path1` | +| `p2` | Path 2 of operation | `s3a://alice-london/path2` | +| `pr` | Principal | `alice` | +| `ps` | Unique process UUID | `53cf0dec-4ffe-4787-8875-5a6fb4411486` | +| `rg` | Range of GetObject Request | `100-200` | +| `ta` | Task Attempt ID (S3A committer) | | +| `t0` | Thread 0: thread span was created in | `13` | +| `t1` | Thread 1: thread this operation was executed in | `13` | +| `ts` | Timestamp (UTC epoch millis) | `1669140734400` | _Notes_ diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/TestHttpReferrerAuditHeader.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/TestHttpReferrerAuditHeader.java index 8f1dab89264de..af94e1455fc30 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/TestHttpReferrerAuditHeader.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/audit/TestHttpReferrerAuditHeader.java @@ -328,7 +328,7 @@ public void testGetObjectRange() throws Throwable { LOG.info("Header is {}", header); Map params = HttpReferrerAuditHeader.extractQueryParameters(header); - assertMapContains(params, PARAM_RANGE, "bytes=100-200"); + assertMapContains(params, PARAM_RANGE, "100-200"); } /** From c17dc7729e0ec5c4f721baa12439365dfb6c5aad Mon Sep 17 00:00:00 2001 From: Ankit Saurabh Date: Wed, 23 Nov 2022 14:34:20 +0000 Subject: [PATCH 16/20] Added the parameter for referrer header example --- .../src/site/markdown/tools/hadoop-aws/auditing.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/auditing.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/auditing.md index 4932b9b92bf3e..e154c6e787b30 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/auditing.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/auditing.md @@ -207,9 +207,9 @@ and the http referrer header extracted. ``` https://audit.example.org/hadoop/1/op_get_file_status/baa4bd75-f4f5-4fe4-ab83-e85fbe2315a8-00000004/ ?op=op_get_file_status - &p1=/path1 - &pr=sauraank - &p2=/path2 + &p1=s3a://alice-london/path1 + &pr=alice + &p2=s3a://alice-london/path2 &ps=53cf0dec-4ffe-4787-8875-5a6fb4411486 &rg=100-200 &id=baa4bd75-f4f5-4fe4-ab83-e85fbe2315a8-00000004 From 2122eafcd622c30cc0dc8ceeccdd7793743a67e6 Mon Sep 17 00:00:00 2001 From: Ankit Saurabh Date: Wed, 23 Nov 2022 17:11:34 +0000 Subject: [PATCH 17/20] Unchanged the parameters other than range in Auditing.md documentation. --- .../markdown/tools/hadoop-aws/auditing.md | 49 +++++++++---------- 1 file changed, 24 insertions(+), 25 deletions(-) diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/auditing.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/auditing.md index e154c6e787b30..4d9fd3a52c5ea 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/auditing.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/auditing.md @@ -205,40 +205,39 @@ This can be parsed (consult AWS documentation for a regular expression) and the http referrer header extracted. ``` -https://audit.example.org/hadoop/1/op_get_file_status/baa4bd75-f4f5-4fe4-ab83-e85fbe2315a8-00000004/ +https://audit.example.org/hadoop/1/op_rename/3c0d9b7e-2a63-43d9-a220-3c574d768ef3-3/ + ?op=op_rename ?op=op_get_file_status &p1=s3a://alice-london/path1 &pr=alice &p2=s3a://alice-london/path2 - &ps=53cf0dec-4ffe-4787-8875-5a6fb4411486 - &rg=100-200 - &id=baa4bd75-f4f5-4fe4-ab83-e85fbe2315a8-00000004 - &t0=13 - &fs=baa4bd75-f4f5-4fe4-ab83-e85fbe2315a8 - &t1=13 - &ts=1669140734400 + &ps=235865a0-d399-4696-9978-64568db1b51c + &id=3c0d9b7e-2a63-43d9-a220-3c574d768ef3-3 + &t0=12 + &fs=af5943a9-b6f6-4eec-9c58-008982fc492a + &t1=12 + &ts=1617116985923 ``` Here are the fields which may be found in a request. If any of the field values were `null`, the field is omitted. -| Name | Meaning | Example | -|------|-------------------------------------------------|-------------------------------------------------| -| `cm` | Command | `S3GuardTool$BucketInfo` | -| `fs` | FileSystem ID | `baa4bd75-f4f5-4fe4-ab83-e85fbe2315a8` | -| `id` | Span ID | `baa4bd75-f4f5-4fe4-ab83-e85fbe2315a8-00000004` | -| `ji` | Job ID (S3A committer) | `(Generated by query engine)` | -| `op` | Filesystem API call | `op_get_file_status` | -| `p1` | Path 1 of operation | `s3a://alice-london/path1` | -| `p2` | Path 2 of operation | `s3a://alice-london/path2` | -| `pr` | Principal | `alice` | -| `ps` | Unique process UUID | `53cf0dec-4ffe-4787-8875-5a6fb4411486` | -| `rg` | Range of GetObject Request | `100-200` | -| `ta` | Task Attempt ID (S3A committer) | | -| `t0` | Thread 0: thread span was created in | `13` | -| `t1` | Thread 1: thread this operation was executed in | `13` | -| `ts` | Timestamp (UTC epoch millis) | `1669140734400` | - +| Name | Meaning | Example | +|------|--|---------| +| `cm` | Command | `S3GuardTool$BucketInfo` | +| `fs` | FileSystem ID | `af5943a9-b6f6-4eec-9c58-008982fc492a` | +| `id` | Span ID | `3c0d9b7e-2a63-43d9-a220-3c574d768ef3-3` | +| `ji` | Job ID (S3A committer)| `(Generated by query engine)` | +| `op` | Filesystem API call | `op_rename` | +| `p1` | Path 1 of operation | `s3a://alice-london/path1` | +| `p2` | Path 2 of operation | `s3a://alice-london/path2` | +| `pr` | Principal | `alice` | +| `ps` | Unique process UUID | `235865a0-d399-4696-9978-64568db1b51c` | +| `rg` | GetObjectRequest range | `100-200` | +| `ta` | Task Attempt ID (S3A committer) | | +| `t0` | Thread 0: thread span was created in | `100` | +| `t1` | Thread 1: thread this operation was executed in | `200` | +| `ts` | Timestamp (UTC epoch millis) | `1617116985923` | _Notes_ * Thread IDs are from the current thread in the JVM, so can be compared to those in````````` From 9ecfee5abd3a7fd40f14082b700a41c2847082b5 Mon Sep 17 00:00:00 2001 From: Ankit Saurabh Date: Wed, 23 Nov 2022 17:41:30 +0000 Subject: [PATCH 18/20] Formatted the table in auditing documentation back to as it was --- .../hadoop-aws/src/site/markdown/tools/hadoop-aws/auditing.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/auditing.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/auditing.md index 4d9fd3a52c5ea..88dd2607359c7 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/auditing.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/auditing.md @@ -223,7 +223,7 @@ Here are the fields which may be found in a request. If any of the field values were `null`, the field is omitted. | Name | Meaning | Example | -|------|--|---------| +|------|---------|---------| | `cm` | Command | `S3GuardTool$BucketInfo` | | `fs` | FileSystem ID | `af5943a9-b6f6-4eec-9c58-008982fc492a` | | `id` | Span ID | `3c0d9b7e-2a63-43d9-a220-3c574d768ef3-3` | From 783ac9dd33ebcafed83d81b4ed96c4d21d899141 Mon Sep 17 00:00:00 2001 From: Ankit Saurabh Date: Wed, 23 Nov 2022 17:42:48 +0000 Subject: [PATCH 19/20] Formatted the table in auditing documentation back to as it was --- .../hadoop-aws/src/site/markdown/tools/hadoop-aws/auditing.md | 1 - 1 file changed, 1 deletion(-) diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/auditing.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/auditing.md index 88dd2607359c7..291668f087dc2 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/auditing.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/auditing.md @@ -207,7 +207,6 @@ and the http referrer header extracted. ``` https://audit.example.org/hadoop/1/op_rename/3c0d9b7e-2a63-43d9-a220-3c574d768ef3-3/ ?op=op_rename - ?op=op_get_file_status &p1=s3a://alice-london/path1 &pr=alice &p2=s3a://alice-london/path2 From f24228897b4a7c3ac87985ea46587c53738ce987 Mon Sep 17 00:00:00 2001 From: Ankit Saurabh Date: Wed, 23 Nov 2022 17:43:56 +0000 Subject: [PATCH 20/20] Added new line --- .../hadoop-aws/src/site/markdown/tools/hadoop-aws/auditing.md | 1 + 1 file changed, 1 insertion(+) diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/auditing.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/auditing.md index 291668f087dc2..ec22709810c40 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/auditing.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/auditing.md @@ -237,6 +237,7 @@ If any of the field values were `null`, the field is omitted. | `t0` | Thread 0: thread span was created in | `100` | | `t1` | Thread 1: thread this operation was executed in | `200` | | `ts` | Timestamp (UTC epoch millis) | `1617116985923` | + _Notes_ * Thread IDs are from the current thread in the JVM, so can be compared to those in`````````