From 17b55c705985d99f0a7055ae8e5c4e29f08ed2c5 Mon Sep 17 00:00:00 2001 From: Wei-Chiu Chuang Date: Mon, 4 Jan 2021 09:43:58 -0800 Subject: [PATCH 1/3] HADOOP-17371. Bump Jetty to the latest version 9.4.34. Contributed by Wei-Chiu Chuang. (#2453) (cherry picked from commit 66ee0a6df0dc0dd8242018153fd652a3206e73b5) (cherry picked from commit 6340ac857b7ff3f73bbcf95b59b98aac134f33af) Conflicts: hadoop-client-modules/hadoop-client-minicluster/pom.xml Change-Id: I673ac136922740cb1d426cb9593aa1bd3e9acd32 --- .../hadoop-client-minicluster/pom.xml | 12 ++++++++ hadoop-common-project/hadoop-auth/pom.xml | 4 +++ .../server/AuthenticationFilter.java | 14 +++++++-- .../http/RestCsrfPreventionFilter.java | 5 ++++ .../kms/server/KMSAuthenticationFilter.java | 13 ++++++++ .../hdfs/server/namenode/ImageServlet.java | 30 ++++++++++++------- hadoop-project/pom.xml | 2 +- 7 files changed, 67 insertions(+), 13 deletions(-) diff --git a/hadoop-client-modules/hadoop-client-minicluster/pom.xml b/hadoop-client-modules/hadoop-client-minicluster/pom.xml index baece53395b82..79e8017deced7 100644 --- a/hadoop-client-modules/hadoop-client-minicluster/pom.xml +++ b/hadoop-client-modules/hadoop-client-minicluster/pom.xml @@ -792,6 +792,18 @@ */** + + org.eclipse.jetty:jetty-util-ajax + + */** + + + + org.eclipse.jetty:jetty-server + + jetty-dir.css + + diff --git a/hadoop-common-project/hadoop-auth/pom.xml b/hadoop-common-project/hadoop-auth/pom.xml index 012c335985046..9400071ce868e 100644 --- a/hadoop-common-project/hadoop-auth/pom.xml +++ b/hadoop-common-project/hadoop-auth/pom.xml @@ -188,6 +188,10 @@ guava compile + + org.eclipse.jetty + jetty-server + diff --git a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java index 94d11f48cf2a9..9f40c42d24135 100644 --- a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java +++ b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java @@ -19,6 +19,7 @@ import org.apache.hadoop.security.authentication.client.AuthenticationException; import org.apache.hadoop.security.authentication.client.KerberosAuthenticator; import org.apache.hadoop.security.authentication.util.*; +import org.eclipse.jetty.server.Response; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -619,11 +620,20 @@ && getMaxInactiveInterval() > 0) { KerberosAuthenticator.WWW_AUTHENTICATE))) { errCode = HttpServletResponse.SC_FORBIDDEN; } + // After Jetty 9.4.21, sendError() no longer allows a custom message. + // use setStatusWithReason() to set a custom message. + String reason; if (authenticationEx == null) { - httpResponse.sendError(errCode, "Authentication required"); + reason = "Authentication required"; } else { - httpResponse.sendError(errCode, authenticationEx.getMessage()); + reason = authenticationEx.getMessage(); } + + if (httpResponse instanceof Response) { + ((Response)httpResponse).setStatusWithReason(errCode, reason); + } + + httpResponse.sendError(errCode, reason); } } } diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/http/RestCsrfPreventionFilter.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/http/RestCsrfPreventionFilter.java index 59cb0d6599595..b81ed8e90155e 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/http/RestCsrfPreventionFilter.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/http/RestCsrfPreventionFilter.java @@ -37,6 +37,7 @@ import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; +import org.eclipse.jetty.server.Response; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -271,6 +272,10 @@ public void proceed() throws IOException, ServletException { @Override public void sendError(int code, String message) throws IOException { + if (httpResponse instanceof Response) { + ((Response)httpResponse).setStatusWithReason(code, message); + } + httpResponse.sendError(code, message); } } diff --git a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java index da542ffb191e6..7f114db9ef245 100644 --- a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java +++ b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java @@ -28,6 +28,7 @@ import org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationHandler; import org.apache.hadoop.security.token.delegation.web.KerberosDelegationTokenAuthenticationHandler; import org.apache.hadoop.security.token.delegation.web.PseudoDelegationTokenAuthenticationHandler; +import org.eclipse.jetty.server.Response; import javax.servlet.FilterChain; import javax.servlet.FilterConfig; @@ -113,6 +114,18 @@ public void setStatus(int sc) { public void sendError(int sc, String msg) throws IOException { statusCode = sc; this.msg = msg; + + ServletResponse response = getResponse(); + + // After Jetty 9.4.21, sendError() no longer allows a custom message. + // use setStatusWithReason() to set a custom message. + if (response instanceof Response) { + ((Response) response).setStatusWithReason(sc, msg); + } else { + KMS.LOG.warn("The wrapped response object is instance of {}" + + ", not org.eclipse.jetty.server.Response. Can't set custom error " + + "message", response.getClass()); + } super.sendError(sc, HtmlQuoting.quoteHtmlChars(msg)); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ImageServlet.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ImageServlet.java index 94657cbac1132..8214cfa2ac8e8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ImageServlet.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ImageServlet.java @@ -41,6 +41,7 @@ import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSUtilClient; import org.apache.hadoop.security.SecurityUtil; +import org.eclipse.jetty.server.Response; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.classification.InterfaceAudience; @@ -118,7 +119,7 @@ private FSImage getAndValidateFSImage(ServletContext context, if (nnImage == null) { String errorMsg = "NameNode initialization not yet complete. " + "FSImage has not been set in the NameNode."; - response.sendError(HttpServletResponse.SC_FORBIDDEN, errorMsg); + sendError(response, HttpServletResponse.SC_FORBIDDEN, errorMsg); throw new IOException(errorMsg); } return nnImage; @@ -207,7 +208,7 @@ private void serveFile(File file) throws IOException { } catch (Throwable t) { String errMsg = "GetImage failed. " + StringUtils.stringifyException(t); - response.sendError(HttpServletResponse.SC_GONE, errMsg); + sendError(response, HttpServletResponse.SC_GONE, errMsg); throw new IOException(errMsg); } finally { response.getOutputStream().close(); @@ -223,7 +224,7 @@ private void validateRequest(ServletContext context, Configuration conf, conf)) { String errorMsg = "Only Namenode, Secondary Namenode, and administrators may access " + "this servlet"; - response.sendError(HttpServletResponse.SC_FORBIDDEN, errorMsg); + sendError(response, HttpServletResponse.SC_FORBIDDEN, errorMsg); LOG.warn("Received non-NN/SNN/administrator request for image or edits from " + request.getUserPrincipal().getName() + " at " @@ -236,7 +237,7 @@ private void validateRequest(ServletContext context, Configuration conf, && !myStorageInfoString.equals(theirStorageInfoString)) { String errorMsg = "This namenode has storage info " + myStorageInfoString + " but the secondary expected " + theirStorageInfoString; - response.sendError(HttpServletResponse.SC_FORBIDDEN, errorMsg); + sendError(response, HttpServletResponse.SC_FORBIDDEN, errorMsg); LOG.warn("Received an invalid request file transfer request " + "from a secondary with storage info " + theirStorageInfoString); throw new IOException(errorMsg); @@ -552,7 +553,7 @@ public Void run() throws Exception { // we need a different response type here so the client can differentiate this // from the failure to upload due to (1) security, or (2) other checkpoints already // present - response.sendError(HttpServletResponse.SC_EXPECTATION_FAILED, + sendError(response, HttpServletResponse.SC_EXPECTATION_FAILED, "Nameode "+request.getLocalAddr()+" is currently not in a state which can " + "accept uploads of new fsimages. State: "+state); return null; @@ -567,7 +568,7 @@ public Void run() throws Exception { // if the node is attempting to upload an older transaction, we ignore it SortedSet larger = currentlyDownloadingCheckpoints.tailSet(imageRequest); if (larger.size() > 0) { - response.sendError(HttpServletResponse.SC_CONFLICT, + sendError(response, HttpServletResponse.SC_CONFLICT, "Another checkpointer is already in the process of uploading a" + " checkpoint made up to transaction ID " + larger.last()); return null; @@ -575,7 +576,7 @@ public Void run() throws Exception { //make sure no one else has started uploading one if (!currentlyDownloadingCheckpoints.add(imageRequest)) { - response.sendError(HttpServletResponse.SC_CONFLICT, + sendError(response, HttpServletResponse.SC_CONFLICT, "Either current namenode is checkpointing or another" + " checkpointer is already in the process of " + "uploading a checkpoint made at transaction ID " @@ -622,7 +623,7 @@ public Void run() throws Exception { (txid - lastCheckpointTxid) + " expecting at least " + checkpointTxnCount; LOG.info(message); - response.sendError(HttpServletResponse.SC_CONFLICT, message); + sendError(response, HttpServletResponse.SC_CONFLICT, message); return null; } @@ -632,7 +633,7 @@ public Void run() throws Exception { + "another checkpointer already uploaded an " + "checkpoint for txid " + txid; LOG.info(message); - response.sendError(HttpServletResponse.SC_CONFLICT, message); + sendError(response, HttpServletResponse.SC_CONFLICT, message); return null; } @@ -669,11 +670,20 @@ public Void run() throws Exception { }); } catch (Throwable t) { String errMsg = "PutImage failed. " + StringUtils.stringifyException(t); - response.sendError(HttpServletResponse.SC_GONE, errMsg); + sendError(response, HttpServletResponse.SC_GONE, errMsg); throw new IOException(errMsg); } } + private void sendError(HttpServletResponse response, int code, String message) + throws IOException { + if (response instanceof Response) { + ((Response)response).setStatusWithReason(code, message); + } + + response.sendError(code, message); + } + /* * Params required to handle put image request */ diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index 7231f3fe74ad4..d9106e77192a7 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -35,7 +35,7 @@ false true - 9.4.20.v20190813 + 9.4.35.v20201120 _ _ From f003ff061726640195aefb90c77e34d9f3d4221e Mon Sep 17 00:00:00 2001 From: Wei-Chiu Chuang Date: Tue, 6 Apr 2021 21:55:01 -0700 Subject: [PATCH 2/3] HADOOP-17621. hadoop-auth to remove jetty-server dependency. (#2865) Reviewed-by: Akira Ajisaka (cherry picked from commit dac60b8282013d7776667415a429e7ca35efba66) (cherry picked from commit 1110b03752b45bc4695baaa6d9655e18de67303a) --- hadoop-common-project/hadoop-auth/pom.xml | 6 +----- .../authentication/server/AuthenticationFilter.java | 8 ++------ 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/hadoop-common-project/hadoop-auth/pom.xml b/hadoop-common-project/hadoop-auth/pom.xml index 9400071ce868e..835ec45655b57 100644 --- a/hadoop-common-project/hadoop-auth/pom.xml +++ b/hadoop-common-project/hadoop-auth/pom.xml @@ -188,11 +188,7 @@ guava compile - - org.eclipse.jetty - jetty-server - - + diff --git a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java index 9f40c42d24135..b339a5d5a2450 100644 --- a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java +++ b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/server/AuthenticationFilter.java @@ -19,7 +19,6 @@ import org.apache.hadoop.security.authentication.client.AuthenticationException; import org.apache.hadoop.security.authentication.client.KerberosAuthenticator; import org.apache.hadoop.security.authentication.util.*; -import org.eclipse.jetty.server.Response; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -621,7 +620,7 @@ && getMaxInactiveInterval() > 0) { errCode = HttpServletResponse.SC_FORBIDDEN; } // After Jetty 9.4.21, sendError() no longer allows a custom message. - // use setStatusWithReason() to set a custom message. + // use setStatus() to set a custom message. String reason; if (authenticationEx == null) { reason = "Authentication required"; @@ -629,10 +628,7 @@ && getMaxInactiveInterval() > 0) { reason = authenticationEx.getMessage(); } - if (httpResponse instanceof Response) { - ((Response)httpResponse).setStatusWithReason(errCode, reason); - } - + httpResponse.setStatus(errCode, reason); httpResponse.sendError(errCode, reason); } } From 26dd37ae2040f7659994b494b0219eec43af157f Mon Sep 17 00:00:00 2001 From: Wei-Chiu Chuang Date: Wed, 7 Apr 2021 23:25:11 -0700 Subject: [PATCH 3/3] HADOOP-17625. Update to Jetty 9.4.39. (#2870) Reviewed-by: cxorm (cherry picked from commit 6040e86e99aae5e29c17b03fddb0a805da8fcae8) (cherry picked from commit 7f7535534d5541e10b6b14dee3aa38a00058f201) --- hadoop-project/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index d9106e77192a7..8f4c9756f09b7 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -35,7 +35,7 @@ false true - 9.4.35.v20201120 + 9.4.39.v20210325 _ _