HADOOP-18311. Upgrade dependencies to address several CVEs

LICENSE-binary

@@ -406,7 +406,7 @@ hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dataTables.bootstrap.css
 hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dataTables.bootstrap.js
 hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dust-full-2.0.0.min.js
 hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dust-helpers-1.1.1.min.js
-hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/jquery-3.5.1.min.js
+hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/jquery-3.6.0.min.js
 hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/jquery.dataTables.min.js
 hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/moment.min.js
 hadoop-tools/hadoop-sls/src/main/html/js/thirdparty/bootstrap.min.js

LICENSE.txt

@@ -243,7 +243,7 @@ hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dataTables.bootstrap.css
 hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dataTables.bootstrap.js
 hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dust-full-2.0.0.min.js
 hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dust-helpers-1.1.1.min.js
-hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/jquery-3.5.1.min.js
+hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/jquery-3.6.0.min.js
 hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/jquery.dataTables.min.js
 hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/moment.min.js
 hadoop-tools/hadoop-sls/src/main/html/js/thirdparty/bootstrap.min.js

NOTICE-binary

@@ -66,7 +66,7 @@ available from http://www.digip.org/jansson/.
 
 
 AWS SDK for Java
-Copyright 2010-2014 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+Copyright 2010-2022 Amazon.com, Inc. or its affiliates. All Rights Reserved.
 
 This product includes software developed by
 Amazon Technologies, Inc (http://www.amazon.com/).

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java

@@ -95,7 +95,7 @@
 import org.eclipse.jetty.server.Server;
 import org.eclipse.jetty.server.ServerConnector;
 import org.eclipse.jetty.server.SslConnectionFactory;
-import org.eclipse.jetty.server.handler.AllowSymLinkAliasChecker;
+import org.eclipse.jetty.server.SymlinkAllowedResourceAliasChecker;
 import org.eclipse.jetty.server.handler.ContextHandlerCollection;
 import org.eclipse.jetty.server.handler.HandlerCollection;
 import org.eclipse.jetty.server.handler.RequestLogHandler;
@@ -859,7 +859,7 @@ protected void addDefaultApps(ContextHandlerCollection parent,
       handler.setHttpOnly(true);
       handler.getSessionCookieConfig().setSecure(true);
       logContext.setSessionHandler(handler);
-      logContext.addAliasCheck(new AllowSymLinkAliasChecker());
+      logContext.addAliasCheck(new SymlinkAllowedResourceAliasChecker(logContext));
       setContextAttributes(logContext, conf);
       addNoCacheFilter(logContext);
       defaultContexts.put(logContext, true);
@@ -878,7 +878,7 @@ protected void addDefaultApps(ContextHandlerCollection parent,
     handler.setHttpOnly(true);
     handler.getSessionCookieConfig().setSecure(true);
     staticContext.setSessionHandler(handler);
-    staticContext.addAliasCheck(new AllowSymLinkAliasChecker());
+    staticContext.addAliasCheck(new SymlinkAllowedResourceAliasChecker(staticContext));
     setContextAttributes(staticContext, conf);
     defaultContexts.put(staticContext, true);
   }

hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/explorer.html

@@ -311,7 +311,7 @@ <h4 class="modal-title" id="delete-modal-title">Delete</h4>
         </ul>
       </p>
     </script>
-    <script type="text/javascript" src="/static/jquery-3.5.1.min.js">
+    <script type="text/javascript" src="/static/jquery-3.6.0.min.js">
     </script><script type="text/javascript" src="/static/jquery.dataTables.min.js">
     </script><script type="text/javascript" src="/static/bootstrap-3.4.1/js/bootstrap.min.js">
     </script><script type="text/javascript" src="/static/bootstrap-3.4.1/js/bootstrap-editable.min.js">

hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/federationhealth.html

@@ -497,7 +497,7 @@
 
 
 
-<script type="text/javascript" src="static/jquery-3.5.1.min.js"></script>
+<script type="text/javascript" src="static/jquery-3.6.0.min.js"></script>
 <script type="text/javascript" src="static/jquery.dataTables.min.js"></script>
 <script type="text/javascript" src="static/bootstrap-3.4.1/js/bootstrap.min.js"></script>
 <script type="text/javascript" src="static/dataTables.bootstrap.js"></script>

hadoop-hdfs-project/hadoop-hdfs/pom.xml

@@ -412,7 +412,7 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
             <exclude>src/main/webapps/static/moment.min.js</exclude>
             <exclude>src/main/webapps/static/dust-full-2.0.0.min.js</exclude>
             <exclude>src/main/webapps/static/dust-helpers-1.1.1.min.js</exclude>
-            <exclude>src/main/webapps/static/jquery-3.5.1.min.js</exclude>
+            <exclude>src/main/webapps/static/jquery-3.6.0.min.js</exclude>
             <exclude>src/main/webapps/static/jquery.dataTables.min.js</exclude>
             <exclude>src/main/webapps/static/json-bignum.js</exclude>
             <exclude>src/main/webapps/static/dataTables.bootstrap.css</exclude>

hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/datanode.html

@@ -126,7 +126,7 @@
   {/dn.VolumeInfo}
 </script>
 
-<script type="text/javascript" src="/static/jquery-3.5.1.min.js"></script>
+<script type="text/javascript" src="/static/jquery-3.6.0.min.js"></script>
 <script type="text/javascript" src="/static/bootstrap-3.4.1/js/bootstrap.min.js"></script>
 <script type="text/javascript" src="/static/moment.min.js"></script>
 <script type="text/javascript" src="/static/dust-full-2.0.0.min.js"></script>

hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html

@@ -492,7 +492,7 @@
 </table>
 </script>
 
-<script type="text/javascript" src="/static/jquery-3.5.1.min.js">
+<script type="text/javascript" src="/static/jquery-3.6.0.min.js">
 </script><script type="text/javascript" src="/static/jquery.dataTables.min.js">
 </script><script type="text/javascript" src="/static/bootstrap-3.4.1/js/bootstrap.min.js">
 </script><script type="text/javascript" src="/static/dataTables.bootstrap.js">

hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.html

@@ -311,7 +311,7 @@ <h4 class="modal-title" id="delete-modal-title">Delete</h4>
         </ul>
       </p>
     </script>
-    <script type="text/javascript" src="/static/jquery-3.5.1.min.js">
+    <script type="text/javascript" src="/static/jquery-3.6.0.min.js">
     </script><script type="text/javascript" src="/static/jquery.dataTables.min.js">
     </script><script type="text/javascript" src="/static/bootstrap-3.4.1/js/bootstrap.min.js">
     </script><script type="text/javascript" src="/static/bootstrap-3.4.1/js/bootstrap-editable.min.js">

hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/journal/journalnode.html

@@ -96,7 +96,7 @@
   </table>
 </script>
 
-<script type="text/javascript" src="/static/jquery-3.5.1.min.js"></script>
+<script type="text/javascript" src="/static/jquery-3.6.0.min.js"></script>
 <script type="text/javascript" src="/static/bootstrap-3.4.1/js/bootstrap.min.js"></script>
 <script type="text/javascript" src="/static/moment.min.js"></script>
 <script type="text/javascript" src="/static/dust-full-2.0.0.min.js"></script>

hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/status.html

@@ -86,7 +86,7 @@
 {/snn}
 </script>
 
-<script type="text/javascript" src="/static/jquery-3.5.1.min.js">
+<script type="text/javascript" src="/static/jquery-3.6.0.min.js">
 </script><script type="text/javascript" src="/static/bootstrap-3.4.1/js/bootstrap.min.js">
 </script><script type="text/javascript" src="/static/moment.min.js">
 </script><script type="text/javascript" src="/static/dust-full-2.0.0.min.js">

hadoop-project/pom.xml

@@ -37,7 +37,7 @@
     <!--Whether to proceed to next module if any test failures exist-->
     <maven.test.failure.ignore>true</maven.test.failure.ignore>
     <maven.test.redirectTestOutputToFile>true</maven.test.redirectTestOutputToFile>
-    <jetty.version>9.4.43.v20210629</jetty.version>
+    <jetty.version>9.4.46.v20220331</jetty.version>
     <test.exclude>_</test.exclude>
     <test.exclude.pattern>_</test.exclude.pattern>
 
@@ -100,7 +100,7 @@
     <hadoop-thirdparty-shaded-protobuf-prefix>${hadoop-thirdparty-shaded-prefix}.protobuf</hadoop-thirdparty-shaded-protobuf-prefix>
     <hadoop-thirdparty-shaded-guava-prefix>${hadoop-thirdparty-shaded-prefix}.com.google.common</hadoop-thirdparty-shaded-guava-prefix>
 
-    <zookeeper.version>3.5.6</zookeeper.version>
+    <zookeeper.version>3.5.10</zookeeper.version>
     <curator.version>4.2.0</curator.version>
     <findbugs.version>3.0.5</findbugs.version>
     <dnsjava.version>2.1.7</dnsjava.version>
@@ -187,7 +187,7 @@
     <exec-maven-plugin.version>1.3.1</exec-maven-plugin.version>
     <make-maven-plugin.version>1.0-beta-1</make-maven-plugin.version>
     <surefire.fork.timeout>900</surefire.fork.timeout>
-    <aws-java-sdk.version>1.11.1026</aws-java-sdk.version>
+    <aws-java-sdk.version>1.12.243</aws-java-sdk.version>
     <hsqldb.version>2.3.4</hsqldb.version>
     <frontend-maven-plugin.version>1.11.2</frontend-maven-plugin.version>
     <jasmine-maven-plugin.version>2.1</jasmine-maven-plugin.version>
@@ -1674,6 +1674,11 @@
             <groupId>log4j</groupId>
             <artifactId>log4j</artifactId>
           </exclusion>
+          <!-- replace htrace-core with hbase-noop-htrace for CVE-2018-7489 -->
+          <exclusion>
+            <groupId>org.apache.htrace</groupId>
+            <artifactId>htrace-core</artifactId>
+          </exclusion>
         </exclusions>
       </dependency>
       <dependency>
@@ -1691,6 +1696,11 @@
             <groupId>org.slf4j</groupId>
             <artifactId>slf4j-log4j12</artifactId>
           </exclusion>
+          <!-- replace htrace-core with hbase-noop-htrace for CVE-2018-7489 -->
+          <exclusion>
+            <groupId>org.apache.htrace</groupId>
+            <artifactId>htrace-core</artifactId>
+          </exclusion>
         </exclusions>
       </dependency>
       <dependency>
@@ -1703,8 +1713,19 @@
             <groupId>jdk.tools</groupId>
             <artifactId>jdk.tools</artifactId>
           </exclusion>
+          <!-- replace htrace-core with hbase-noop-htrace for CVE-2018-7489 -->
+          <exclusion>
+            <groupId>org.apache.htrace</groupId>
+            <artifactId>htrace-core</artifactId>
+          </exclusion>
         </exclusions>
       </dependency>
+      <!-- replace htrace-core with hbase-noop-htrace for CVE-2018-7489 -->
+      <dependency>
+        <groupId>org.apache.hbase.thirdparty</groupId>
+        <artifactId>hbase-noop-htrace</artifactId>
+        <version>4.1.0</version>
+      </dependency>
       <dependency>
         <groupId>org.apache.hbase</groupId>
         <artifactId>hbase-server</artifactId>

hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/ArnResource.java

@@ -21,12 +21,12 @@
 import javax.annotation.Nonnull;
 
 import com.amazonaws.arn.Arn;
-import com.amazonaws.regions.RegionUtils;
 
 /**
  * Represents an Arn Resource, this can be an accesspoint or bucket.
  */
 public final class ArnResource {
+  private final static String ACCESSPOINT_ENDPOINT_FORMAT = "s3-accesspoint.%s.amazonaws.com";
 
   /**
    * Resource name.
@@ -106,8 +106,7 @@ public String getFullArn() {
    * @return resource endpoint.
    */
   public String getEndpoint() {
-    return RegionUtils.getRegion(accessPointRegionKey)
-        .getServiceEndpoint("s3");
+    return String.format(ACCESSPOINT_ENDPOINT_FORMAT, region);
   }
 
   /**

hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md

@@ -1662,6 +1662,7 @@ to AWS services.
 * Try and get other people, especially anyone with their own endpoints,
   apps or different deployment environments, to run their own tests.
 * Run the load tests, especially `ILoadTestS3ABulkDeleteThrottling`.
+* Checkout cloudstore, build it against your version of hadoop, then use its CLI to run some commands (`storediag` etc)
 
 ### Dealing with Deprecated APIs and New Features
 

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestArnResource.java

@@ -19,6 +19,7 @@
 package org.apache.hadoop.fs.s3a;
 
 import com.amazonaws.regions.Regions;
+import org.assertj.core.api.Assertions;
 import org.junit.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -27,40 +28,54 @@
 
 import static org.apache.hadoop.test.LambdaTestUtils.intercept;
 
+/**
+ * Verifies the mapping of ARN declaration of resource to the associated
+ * access point.
+ * The region mapping assertions have been brittle to changes across AWS SDK
+ * versions to only verify partial matches, rather than the FQDN of the
+ * endpoints.
+ *
+ */
 public class TestArnResource extends HadoopTestBase {
   private final static Logger LOG = LoggerFactory.getLogger(TestArnResource.class);
 
+  private final static String MOCK_ACCOUNT = "123456789101";
+
   @Test
   public void parseAccessPointFromArn() throws IllegalArgumentException {
     describe("Parse AccessPoint ArnResource from arn string");
 
     String accessPoint = "testAp";
-    String accountId = "123456789101";
     String[][] regionPartitionEndpoints = new String[][] {
-        {Regions.EU_WEST_1.getName(), "aws", "s3-accesspoint.eu-west-1.amazonaws.com"},
-        {Regions.US_GOV_EAST_1.getName(), "aws-us-gov",
-            "s3-accesspoint.us-gov-east-1.amazonaws.com"},
-        {Regions.CN_NORTH_1.getName(), "aws-cn", "s3-accesspoint.cn-north-1.amazonaws.com.cn"},
+        {Regions.EU_WEST_1.getName(), "aws"},
+        {Regions.US_GOV_EAST_1.getName(), "aws-us-gov"},
+        {Regions.CN_NORTH_1.getName(), "aws-cn"},
     };
 
     for (String[] testPair : regionPartitionEndpoints) {
       String region = testPair[0];
       String partition = testPair[1];
-      String endpoint = testPair[2];
 
-      // arn:partition:service:region:account-id:resource-type/resource-id
-      String arn = String.format("arn:%s:s3:%s:%s:accesspoint/%s", partition, region, accountId,
-          accessPoint);
-
-      ArnResource resource = ArnResource.accessPointFromArn(arn);
-      assertEquals("Arn does not match", arn, resource.getFullArn());
+      ArnResource resource = getArnResourceFrom(partition, region, MOCK_ACCOUNT, accessPoint);
       assertEquals("Access Point name does not match", accessPoint, resource.getName());
-      assertEquals("Account Id does not match", accountId, resource.getOwnerAccountId());
+      assertEquals("Account Id does not match", MOCK_ACCOUNT, resource.getOwnerAccountId());
       assertEquals("Region does not match", region, resource.getRegion());
-      assertEquals("Endpoint does not match", endpoint, resource.getEndpoint());
     }
   }
 
+  @Test
+  public void makeSureEndpointHasTheCorrectFormat() {
+    // Access point (AP) endpoints are different from S3 bucket endpoints, thus when using APs the
+    // endpoints for the client are modified. This test makes sure endpoint is set up correctly.
+    ArnResource accessPoint = getArnResourceFrom("aws", "eu-west-1", MOCK_ACCOUNT,
+        "test");
+    String expected = "s3-accesspoint.eu-west-1.amazonaws.com";
+
+    Assertions.assertThat(accessPoint.getEndpoint())
+        .describedAs("Endpoint has invalid format. Access Point requests will not work")
+        .isEqualTo(expected);
+  }
+
   @Test
   public void invalidARNsMustThrow() throws Exception {
     describe("Using an invalid ARN format must throw when initializing an ArnResource.");
@@ -69,6 +84,23 @@ public void invalidARNsMustThrow() throws Exception {
         ArnResource.accessPointFromArn("invalid:arn:resource"));
   }
 
+  /**
+   * Create an {@link ArnResource} from string components
+   * @param partition - partition for ARN
+   * @param region - region for ARN
+   * @param accountId - accountId for ARN
+   * @param resourceName - ARN resource name
+   * @return ArnResource described by its properties
+   */
+  private ArnResource getArnResourceFrom(String partition, String region, String accountId,
+      String resourceName) {
+    // arn:partition:service:region:account-id:resource-type/resource-id
+    String arn = String.format("arn:%s:s3:%s:%s:accesspoint/%s", partition, region, accountId,
+        resourceName);
+
+    return ArnResource.accessPointFromArn(arn);
+  }
+
   private void describe(String message) {
     LOG.info(message);
   }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml

@@ -214,6 +214,12 @@
         </exclusion>
       </exclusions>
     </dependency>
+    <!-- replace htrace-core with hbase-noop-htrace for CVE-2018-7489 -->
+    <dependency>
+      <groupId>org.apache.hbase.thirdparty</groupId>
+      <artifactId>hbase-noop-htrace</artifactId>
+      <scope>test</scope>
+    </dependency>
 
     <dependency>
       <groupId>org.apache.hbase</groupId>

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/pom.xml

@@ -141,6 +141,11 @@
         </exclusion>
       </exclusions>
     </dependency>
+    <!-- replace htrace-core with hbase-noop-htrace for CVE-2018-7489 -->
+    <dependency>
+      <groupId>org.apache.hbase.thirdparty</groupId>
+      <artifactId>hbase-noop-htrace</artifactId>
+    </dependency>
 
     <dependency>
       <groupId>org.apache.hbase</groupId>

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-common/pom.xml

@@ -83,6 +83,11 @@
         </exclusion>
       </exclusions>
     </dependency>
+    <!-- replace htrace-core with hbase-noop-htrace for CVE-2018-7489 -->
+    <dependency>
+      <groupId>org.apache.hbase.thirdparty</groupId>
+      <artifactId>hbase-noop-htrace</artifactId>
+    </dependency>
 
     <dependency>
       <groupId>org.apache.hadoop.thirdparty</groupId>

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/hadoop-yarn-server-timelineservice-hbase-server-1/pom.xml

@@ -102,6 +102,11 @@
             </exclusion>
           </exclusions>
         </dependency>
+        <!-- replace htrace-core with hbase-noop-htrace for CVE-2018-7489 -->
+        <dependency>
+          <groupId>org.apache.hbase.thirdparty</groupId>
+          <artifactId>hbase-noop-htrace</artifactId>
+        </dependency>
 
         <dependency>
           <groupId>org.apache.hbase</groupId>

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-server/hadoop-yarn-server-timelineservice-hbase-server-2/pom.xml

@@ -102,6 +102,11 @@
             </exclusion>
           </exclusions>
         </dependency>
+        <!-- replace htrace-core with hbase-noop-htrace for CVE-2018-7489 -->
+        <dependency>
+          <groupId>org.apache.hbase.thirdparty</groupId>
+          <artifactId>hbase-noop-htrace</artifactId>
+        </dependency>
 
         <!-- This is to work around the version divergence of
             org.jruby.jcodings:jcodings pulled in by hbase-client -->