diff --git a/dev/.rat-excludes b/dev/.rat-excludes index 59e619ba109b..ccf266c3218d 100644 --- a/dev/.rat-excludes +++ b/dev/.rat-excludes @@ -115,3 +115,5 @@ structured-streaming/* kafka-source-initial-offset-version-2.1.0.bin kafka-source-initial-offset-future-version.bin vote.tmpl +SessionManager.java +SessionHandler.java diff --git a/resource-managers/yarn/pom.xml b/resource-managers/yarn/pom.xml index df910c159787..0e5df14e060d 100644 --- a/resource-managers/yarn/pom.xml +++ b/resource-managers/yarn/pom.xml @@ -29,7 +29,7 @@ Spark Project YARN yarn - 1.9 + 1.19 @@ -166,6 +166,12 @@ test ${jersey-1.version} + + com.sun.jersey + jersey-servlet + test + ${jersey-1.version} + diff --git a/resource-managers/yarn/src/test/java/org/apache/hadoop/net/ServerSocketUtil.java b/resource-managers/yarn/src/test/java/org/apache/hadoop/net/ServerSocketUtil.java new file mode 100644 index 000000000000..df0ebcc9871a --- /dev/null +++ b/resource-managers/yarn/src/test/java/org/apache/hadoop/net/ServerSocketUtil.java @@ -0,0 +1,132 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.net; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.net.ServerSocket; +import java.util.Random; + +/** + * Copied from + * hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/ServerSocketUtil.java + * for Hadoop-3.x testing + */ +public class ServerSocketUtil { + + private static final Logger LOG = LoggerFactory.getLogger(ServerSocketUtil.class); + private static Random rand = new Random(); + + /** + * Port scan & allocate is how most other apps find ports + * + * @param port given port + * @param retries number of retries + * @return + * @throws IOException + */ + public static int getPort(int port, int retries) throws IOException { + int tryPort = port; + int tries = 0; + while (true) { + if (tries > 0 || tryPort == 0) { + tryPort = port + rand.nextInt(65535 - port); + } + if (tryPort == 0) { + continue; + } + try (ServerSocket s = new ServerSocket(tryPort)) { + LOG.info("Using port " + tryPort); + return tryPort; + } catch (IOException e) { + tries++; + if (tries >= retries) { + LOG.info("Port is already in use; giving up"); + throw e; + } else { + LOG.info("Port is already in use; trying again"); + } + } + } + } + + /** + * Check whether port is available or not. + * + * @param port given port + * @return + */ + private static boolean isPortAvailable(int port) { + try (ServerSocket s = new ServerSocket(port)) { + return true; + } catch (IOException e) { + return false; + } + } + + /** + * Wait till the port available. + * + * @param port given port + * @param retries number of retries for given port + * @return + * @throws InterruptedException + * @throws IOException + */ + public static int waitForPort(int port, int retries) + throws InterruptedException, IOException { + int tries = 0; + while (true) { + if (isPortAvailable(port)) { + return port; + } else { + tries++; + if (tries >= retries) { + throw new IOException( + "Port is already in use; giving up after " + tries + " times."); + } + Thread.sleep(1000); + } + } + } + + /** + * Find the specified number of unique ports available. + * The ports are all closed afterwards, + * so other network services started may grab those same ports. + * + * @param numPorts number of required port nubmers + * @return array of available port numbers + * @throws IOException + */ + public static int[] getPorts(int numPorts) throws IOException { + ServerSocket[] sockets = new ServerSocket[numPorts]; + int[] ports = new int[numPorts]; + for (int i = 0; i < numPorts; i++) { + ServerSocket sock = new ServerSocket(0); + sockets[i] = sock; + ports[i] = sock.getLocalPort(); + } + for (ServerSocket sock : sockets) { + sock.close(); + } + return ports; + } +} diff --git a/resource-managers/yarn/src/test/java/org/eclipse/jetty/server/SessionManager.java b/resource-managers/yarn/src/test/java/org/eclipse/jetty/server/SessionManager.java new file mode 100644 index 000000000000..bf89f8d7c804 --- /dev/null +++ b/resource-managers/yarn/src/test/java/org/eclipse/jetty/server/SessionManager.java @@ -0,0 +1,290 @@ +// +// ======================================================================== +// Copyright (c) 1995-2018 Mort Bay Consulting Pty. Ltd. +// ------------------------------------------------------------------------ +// All rights reserved. This program and the accompanying materials +// are made available under the terms of the Eclipse Public License v1.0 +// and Apache License v2.0 which accompanies this distribution. +// +// The Eclipse Public License is available at +// http://www.eclipse.org/legal/epl-v10.html +// +// The Apache License v2.0 is available at +// http://www.opensource.org/licenses/apache2.0.php +// +// You may elect to redistribute this code under either of these licenses. +// ======================================================================== +// + +package org.eclipse.jetty.server; + +import javax.servlet.SessionCookieConfig; +import javax.servlet.SessionTrackingMode; +import javax.servlet.http.Cookie; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpSession; +import java.util.EventListener; +import java.util.Set; + +import org.eclipse.jetty.http.HttpCookie; +import org.eclipse.jetty.server.session.SessionHandler; +import org.eclipse.jetty.util.component.LifeCycle; + +/** + * Adapted from https://github.com/eclipse/jetty.project/blob/jetty-9.3.25.v20180904/ + * jetty-server/src/main/java/org/eclipse/jetty/server/SessionManager.java + */ +public interface SessionManager extends LifeCycle { + /** + * Session cookie name. + * Defaults to JSESSIONID, but can be set with the + * org.eclipse.jetty.servlet.SessionCookie context init parameter. + */ + String __SessionCookieProperty = "org.eclipse.jetty.servlet.SessionCookie"; + String __DefaultSessionCookie = "JSESSIONID"; + + /** + * Session id path parameter name. + * Defaults to jsessionid, but can be set with the + * org.eclipse.jetty.servlet.SessionIdPathParameterName context init parameter. + * If set to null or "none" no URL rewriting will be done. + */ + String __SessionIdPathParameterNameProperty = + "org.eclipse.jetty.servlet.SessionIdPathParameterName"; + String __DefaultSessionIdPathParameterName = "jsessionid"; + String __CheckRemoteSessionEncoding = "org.eclipse.jetty.servlet.CheckingRemoteSessionIdEncoding"; + + /** + * Session Domain. + * If this property is set as a ServletContext InitParam, then it is + * used as the domain for session cookies. If it is not set, then + * no domain is specified for the session cookie. + */ + String __SessionDomainProperty = "org.eclipse.jetty.servlet.SessionDomain"; + String __DefaultSessionDomain = null; + + /** + * Session Path. + * If this property is set as a ServletContext InitParam, then it is + * used as the path for the session cookie. If it is not set, then + * the context path is used as the path for the cookie. + */ + String __SessionPathProperty = "org.eclipse.jetty.servlet.SessionPath"; + + /** + * Session Max Age. + * If this property is set as a ServletContext InitParam, then it is + * used as the max age for the session cookie. If it is not set, then + * a max age of -1 is used. + */ + String __MaxAgeProperty = "org.eclipse.jetty.servlet.MaxAge"; + + /** + * Returns the HttpSession with the given session id + * + * @param id the session id + * @return the HttpSession with the corresponding id + * or null if no session with the given id exists + */ + HttpSession getHttpSession(String id); + + /** + * Creates a new HttpSession. + * + * @param request the HttpServletRequest containing the requested session id + * @return the new HttpSession + */ + HttpSession newHttpSession(HttpServletRequest request); + + /** + * @return true if session cookies should be HTTP-only (Microsoft extension) + * @see HttpCookie#isHttpOnly() + */ + boolean getHttpOnly(); + + /** + * @return the max period of inactivity, after which the session is invalidated, in seconds. + * @see #setMaxInactiveInterval(int) + */ + int getMaxInactiveInterval(); + + /** + * Sets the max period of inactivity, after which the session is invalidated, in seconds. + * + * @param seconds the max inactivity period, in seconds. + * @see #getMaxInactiveInterval() + */ + void setMaxInactiveInterval(int seconds); + + /** + * Sets the {@link SessionHandler}. + * + * @param handler the SessionHandler object + */ + void setSessionHandler(SessionHandler handler); + + /** + * Adds an event listener for session-related events. + * + * @param listener the session event listener to add + * Individual SessionManagers implementations may accept arbitrary listener types, + * but they are expected to at least handle HttpSessionActivationListener, + * HttpSessionAttributeListener, + * HttpSessionBindingListener and HttpSessionListener. + * @see #removeEventListener(EventListener) + */ + void addEventListener(EventListener listener); + + /** + * Removes an event listener for for session-related events. + * + * @param listener the session event listener to remove + * @see #addEventListener(EventListener) + */ + void removeEventListener(EventListener listener); + + /** + * Removes all event listeners for session-related events. + * + * @see #removeEventListener(EventListener) + */ + void clearEventListeners(); + + /** + * Gets a Cookie for a session. + * + * @param session the session to which the cookie should refer. + * @param contextPath the context to which the cookie should be linked. + * The client will only send the cookie value when + * requesting resources under this path. + * @param requestIsSecure whether the client is accessing the server over + * a secure protocol (i.e. HTTPS). + * @return if this SessionManager uses cookies, then this method will return a new + * {@link Cookie cookie object} that should be set on the client + * in order to link future HTTP requests + * with the session. If cookies are not in use, + * this method returns null. + */ + HttpCookie getSessionCookie(HttpSession session, String contextPath, boolean requestIsSecure); + + /** + * @return the cross context session id manager. + * @see #setSessionIdManager(SessionIdManager) + */ + SessionIdManager getSessionIdManager(); + + /** + * @return the cross context session id manager. + * @deprecated use {@link #getSessionIdManager()} + */ + @Deprecated + SessionIdManager getMetaManager(); + + /** + * Sets the cross context session id manager + * + * @param idManager the cross context session id manager. + * @see #getSessionIdManager() + */ + void setSessionIdManager(SessionIdManager idManager); + + /** + * @param session the session to test for validity + * @return whether the given session is valid, that is, it has not been invalidated. + */ + boolean isValid(HttpSession session); + + /** + * @param session the session object + * @return the unique id of the session within the cluster, extended with an optional node id. + * @see #getClusterId(HttpSession) + */ + String getNodeId(HttpSession session); + + /** + * @param session the session object + * @return the unique id of the session within the cluster (without a node id extension) + * @see #getNodeId(HttpSession) + */ + String getClusterId(HttpSession session); + + /** + * Called by the {@link SessionHandler} when a session is first accessed by a request. + * + * @param session the session object + * @param secure whether the request is secure or not + * @return the session cookie. If not null, + * this cookie should be set on the response to either migrate + * the session or to refresh a session cookie that may expire. + * @see #complete(HttpSession) + */ + HttpCookie access(HttpSession session, boolean secure); + + /** + * Called by the {@link SessionHandler} when a session is last accessed by a request. + * + * @param session the session object + * @see #access(HttpSession, boolean) + */ + void complete(HttpSession session); + + /** + * Sets the session id URL path parameter name. + * + * @param parameterName the URL path parameter name + * for session id URL rewriting (null or "none" for no rewriting). + * @see #getSessionIdPathParameterName() + * @see #getSessionIdPathParameterNamePrefix() + */ + void setSessionIdPathParameterName(String parameterName); + + /** + * @return the URL path parameter name for session id URL rewriting, by default "jsessionid". + * @see #setSessionIdPathParameterName(String) + */ + String getSessionIdPathParameterName(); + + /** + * @return a formatted version of {@link #getSessionIdPathParameterName()}, by default + * ";" + sessionIdParameterName + "=", for easier lookup in URL strings. + * @see #getSessionIdPathParameterName() + */ + String getSessionIdPathParameterNamePrefix(); + + /** + * @return whether the session management is handled via cookies. + */ + boolean isUsingCookies(); + + /** + * @return whether the session management is handled via URLs. + */ + boolean isUsingURLs(); + + Set getDefaultSessionTrackingModes(); + + Set getEffectiveSessionTrackingModes(); + + void setSessionTrackingModes(Set sessionTrackingModes); + + SessionCookieConfig getSessionCookieConfig(); + + /** + * @return True if absolute URLs are check for remoteness before being session encoded. + */ + boolean isCheckingRemoteSessionIdEncoding(); + + /** + * @param remote True if absolute URLs are check for remoteness before being session encoded. + */ + void setCheckingRemoteSessionIdEncoding(boolean remote); + + /** Change the existing session id. + * + * @param oldClusterId the old cluster id + * @param oldNodeId the old node id + * @param newClusterId the new cluster id + * @param newNodeId the new node id + */ + void renewSessionId(String oldClusterId, String oldNodeId, String newClusterId, String newNodeId); +} diff --git a/resource-managers/yarn/src/test/java/org/eclipse/jetty/server/session/SessionHandler.java b/resource-managers/yarn/src/test/java/org/eclipse/jetty/server/session/SessionHandler.java new file mode 100644 index 000000000000..f8bcf8de82f6 --- /dev/null +++ b/resource-managers/yarn/src/test/java/org/eclipse/jetty/server/session/SessionHandler.java @@ -0,0 +1,90 @@ +// +// ======================================================================== +// Copyright (c) 1995-2018 Mort Bay Consulting Pty. Ltd. +// ------------------------------------------------------------------------ +// All rights reserved. This program and the accompanying materials +// are made available under the terms of the Eclipse Public License v1.0 +// and Apache License v2.0 which accompanies this distribution. +// +// The Eclipse Public License is available at +// http://www.eclipse.org/legal/epl-v10.html +// +// The Apache License v2.0 is available at +// http://www.opensource.org/licenses/apache2.0.php +// +// You may elect to redistribute this code under either of these licenses. +// ======================================================================== +// + +package org.eclipse.jetty.server.session; + +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.io.IOException; + +import org.eclipse.jetty.server.Request; +import org.eclipse.jetty.server.SessionManager; +import org.eclipse.jetty.server.handler.ScopedHandler; + +/** + * Adapted from https://github.com/eclipse/jetty.project/blob/jetty-9.3.25.v20180904/ + * jetty-server/src/main/java/org/eclipse/jetty/server/session/SessionHandler.java + */ +public class SessionHandler extends ScopedHandler { + private SessionManager _sessionManager; + + public SessionHandler() { + } + + /** + * @param manager + * The session manager + */ + public SessionHandler(SessionManager manager) { + setSessionManager(manager); + } + + /** + * @return Returns the sessionManager. + */ + public SessionManager getSessionManager() { + return _sessionManager; + } + + /** + * @param sessionManager + * The sessionManager to set. + */ + public void setSessionManager(SessionManager sessionManager) { + if (isStarted()) { + throw new IllegalStateException(); + } + if (sessionManager != null) { + updateBean(_sessionManager,sessionManager); + _sessionManager=sessionManager; + } + } + + /* + * @see org.eclipse.jetty.server.Handler#handle(javax.servlet.http.HttpServletRequest, + * javax.servlet.http.HttpServletResponse, int) + */ + @Override + public void doHandle(String target, Request baseRequest, HttpServletRequest request, + HttpServletResponse response) throws IOException, ServletException { + // start manual inline of nextHandle(target,baseRequest,request,response); + if (_nextScope != null && _nextScope == _handler) { + _nextScope.doHandle(target,baseRequest,request,response); + } else if (_handler != null) { + _handler.handle(target,baseRequest,request,response); + // end manual inline + } + } + + public void clearEventListeners() { + if (_sessionManager != null) { + _sessionManager.clearEventListeners(); + } + } +}