From 3f6386a3d8a98bc2db5f194d315a8a68eac1500b Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Wed, 4 Jan 2023 03:51:29 -0800 Subject: [PATCH 01/18] Add retry for HTTP 429 and HTTP 410 --- .../services/ExponentialRetryPolicy.java | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ExponentialRetryPolicy.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ExponentialRetryPolicy.java index bcf94651acad9..560a5a0c59d95 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ExponentialRetryPolicy.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ExponentialRetryPolicy.java @@ -56,6 +56,20 @@ public class ExponentialRetryPolicy { */ private static final double MAX_RANDOM_RATIO = 1.2; + /** + * Qualifies for retry based on + * https://docs.microsoft.com/en-in/azure/virtual-machines/linux/ + * instance-metadata-service?tabs=windows#errors-and-debugging + */ + private static final int HTTP_GONE = 410; + + /** + * Qualifies for retry based on + * https://learn.microsoft.com/en-us/azure/active-directory/ + * managed-identities-azure-resources/how-to-use-vm-token#error-handling + */ + private static final int HTTP_TOO_MANY_REQUESTS = 429; + /** * Holds the random number generator used to calculate randomized backoff intervals */ @@ -128,9 +142,11 @@ public boolean shouldRetry(final int retryCount, final int statusCode) { return retryCount < this.retryCount && (statusCode == -1 || statusCode == HttpURLConnection.HTTP_CLIENT_TIMEOUT + || statusCode == HTTP_GONE + || statusCode == HTTP_TOO_MANY_REQUESTS || (statusCode >= HttpURLConnection.HTTP_INTERNAL_ERROR - && statusCode != HttpURLConnection.HTTP_NOT_IMPLEMENTED - && statusCode != HttpURLConnection.HTTP_VERSION)); + && statusCode != HttpURLConnection.HTTP_NOT_IMPLEMENTED + && statusCode != HttpURLConnection.HTTP_VERSION)); } /** From 56c6f7de6ead9500d2c449ba768d2301fc7e4d85 Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Wed, 4 Jan 2023 03:55:33 -0800 Subject: [PATCH 02/18] Add retry for HTTP 429 and HTTP 410 --- .../hadoop/fs/azurebfs/services/ExponentialRetryPolicy.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ExponentialRetryPolicy.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ExponentialRetryPolicy.java index 560a5a0c59d95..816cdc3a64402 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ExponentialRetryPolicy.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ExponentialRetryPolicy.java @@ -145,8 +145,8 @@ public boolean shouldRetry(final int retryCount, final int statusCode) { || statusCode == HTTP_GONE || statusCode == HTTP_TOO_MANY_REQUESTS || (statusCode >= HttpURLConnection.HTTP_INTERNAL_ERROR - && statusCode != HttpURLConnection.HTTP_NOT_IMPLEMENTED - && statusCode != HttpURLConnection.HTTP_VERSION)); + && statusCode != HttpURLConnection.HTTP_NOT_IMPLEMENTED + && statusCode != HttpURLConnection.HTTP_VERSION)); } /** From 2c25b39b727a259ca0feb754705adbcfe5e80331 Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Wed, 4 Jan 2023 04:04:15 -0800 Subject: [PATCH 03/18] Add retry for HTTP 429 and HTTP 410 --- .../fs/azurebfs/services/ExponentialRetryPolicy.java | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ExponentialRetryPolicy.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ExponentialRetryPolicy.java index 816cdc3a64402..1b84877b1b64b 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ExponentialRetryPolicy.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ExponentialRetryPolicy.java @@ -56,13 +56,6 @@ public class ExponentialRetryPolicy { */ private static final double MAX_RANDOM_RATIO = 1.2; - /** - * Qualifies for retry based on - * https://docs.microsoft.com/en-in/azure/virtual-machines/linux/ - * instance-metadata-service?tabs=windows#errors-and-debugging - */ - private static final int HTTP_GONE = 410; - /** * Qualifies for retry based on * https://learn.microsoft.com/en-us/azure/active-directory/ @@ -133,6 +126,9 @@ public ExponentialRetryPolicy(final int retryCount, final int minBackoff, final /** * Returns if a request should be retried based on the retry count, current response, * and the current strategy. + * HTTP status code 410 qualifies for retry based on + * https://docs.microsoft.com/en-in/azure/virtual-machines/linux/ + * instance-metadata-service?tabs=windows#errors-and-debugging * * @param retryCount The current retry attempt count. * @param statusCode The status code of the response, or -1 for socket error. @@ -142,7 +138,7 @@ public boolean shouldRetry(final int retryCount, final int statusCode) { return retryCount < this.retryCount && (statusCode == -1 || statusCode == HttpURLConnection.HTTP_CLIENT_TIMEOUT - || statusCode == HTTP_GONE + || statusCode == HttpURLConnection.HTTP_GONE || statusCode == HTTP_TOO_MANY_REQUESTS || (statusCode >= HttpURLConnection.HTTP_INTERNAL_ERROR && statusCode != HttpURLConnection.HTTP_NOT_IMPLEMENTED From 88e51b674ffcf624f811c5f1d27397f1c134ad41 Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Wed, 4 Jan 2023 21:51:01 -0800 Subject: [PATCH 04/18] Added tests to verify retry --- hadoop-tools/hadoop-azure/pom.xml | 15 +++++ .../azurebfs/oauth2/AzureADAuthenticator.java | 5 +- .../azurebfs/ITestAbfsMsiTokenProvider.java | 62 ++++++++++++++++++- 3 files changed, 79 insertions(+), 3 deletions(-) diff --git a/hadoop-tools/hadoop-azure/pom.xml b/hadoop-tools/hadoop-azure/pom.xml index c313fa28a3ab4..55599b05ea025 100644 --- a/hadoop-tools/hadoop-azure/pom.xml +++ b/hadoop-tools/hadoop-azure/pom.xml @@ -308,8 +308,23 @@ org.mockito mockito-core + 4.11.0 test + + + org.mockito + mockito-inline + 4.11.0 + test + + + org.mockito + mockito-core + + + + org.apache.hadoop hadoop-minikdc diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/AzureADAuthenticator.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/AzureADAuthenticator.java index dd4ec7c2009c2..94c9b0b65f782 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/AzureADAuthenticator.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/AzureADAuthenticator.java @@ -29,6 +29,7 @@ import java.util.Hashtable; import java.util.Map; +import org.apache.hadoop.classification.VisibleForTesting; import org.apache.hadoop.util.Preconditions; import com.fasterxml.jackson.core.JsonFactory; @@ -213,7 +214,7 @@ public String getRequestId() { return this.requestId; } - protected HttpException( + public HttpException( final int httpErrorCode, final String requestId, final String message, @@ -341,7 +342,7 @@ private static boolean isRecoverableFailure(IOException e) { || e instanceof FileNotFoundException); } - private static AzureADToken getTokenSingleCall(String authEndpoint, + public static AzureADToken getTokenSingleCall(String authEndpoint, String payload, Hashtable headers, String httpMethod, boolean isMsi) throws IOException { diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsMsiTokenProvider.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsMsiTokenProvider.java index d871befa43005..6c59e25308cd2 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsMsiTokenProvider.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsMsiTokenProvider.java @@ -19,15 +19,21 @@ package org.apache.hadoop.fs.azurebfs; import java.io.IOException; +import java.lang.reflect.Field; import java.util.Date; - import org.junit.Test; +import org.mockito.MockedStatic; +import org.mockito.Mockito; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.fs.azurebfs.oauth2.AccessTokenProvider; +import org.apache.hadoop.fs.azurebfs.oauth2.AzureADAuthenticator; import org.apache.hadoop.fs.azurebfs.oauth2.AzureADToken; import org.apache.hadoop.fs.azurebfs.oauth2.MsiTokenProvider; +import org.apache.hadoop.fs.azurebfs.services.ExponentialRetryPolicy; +import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_AZURE_OAUTH_TOKEN_FETCH_RETRY_MAX_ATTEMPTS; +import static org.apache.hadoop.test.LambdaTestUtils.intercept; import static org.junit.Assume.assumeThat; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.CoreMatchers.not; @@ -40,6 +46,7 @@ import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_OAUTH_MSI_AUTHORITY; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_OAUTH_MSI_ENDPOINT; import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_OAUTH_MSI_TENANT; +import static org.mockito.Mockito.times; /** * Test MsiTokenProvider. @@ -90,4 +97,57 @@ private String getTrimmedPasswordString(AbfsConfiguration conf, String key, return value.trim(); } + /** + * Test to verify that token fetch is retried for throttling errors (too many requests 429). + * @throws Exception + */ + @Test + public void testRetryForThrottling() throws Exception { + final int HTTP_TOO_MANY_REQUESTS = 429; + AbfsConfiguration conf = getConfiguration(); + + // Exception to be thrown with throttling error code 429. + AzureADAuthenticator.HttpException httpException + = new AzureADAuthenticator.HttpException(HTTP_TOO_MANY_REQUESTS, + "abc", "abc", "abc", "abc", "abc"); + + String tenantGuid = "abcd"; + String clientId = "abcd"; + String authEndpoint = getTrimmedPasswordString(conf, + FS_AZURE_ACCOUNT_OAUTH_MSI_ENDPOINT, + DEFAULT_FS_AZURE_ACCOUNT_OAUTH_MSI_ENDPOINT); + String authority = getTrimmedPasswordString(conf, + FS_AZURE_ACCOUNT_OAUTH_MSI_AUTHORITY, + DEFAULT_FS_AZURE_ACCOUNT_OAUTH_MSI_AUTHORITY); + + // Mock the getTokenSingleCall to throw exception so the retry logic comes into place. + try (MockedStatic adAuthenticator = Mockito.mockStatic( + AzureADAuthenticator.class, Mockito.CALLS_REAL_METHODS)) { + adAuthenticator.when( + () -> AzureADAuthenticator.getTokenSingleCall(Mockito.anyString(), + Mockito.anyString(), Mockito.any(), Mockito.anyString(), + Mockito.anyBoolean())).thenThrow(httpException); + + // Mock the tokenFetchRetryPolicy to verify retries. + ExponentialRetryPolicy exponentialRetryPolicy = Mockito.spy( + conf.getOauthTokenFetchRetryPolicy()); + Field tokenFetchRetryPolicy = AzureADAuthenticator.class.getDeclaredField( + "tokenFetchRetryPolicy"); + tokenFetchRetryPolicy.setAccessible(true); + tokenFetchRetryPolicy.set(ExponentialRetryPolicy.class, + exponentialRetryPolicy); + + AccessTokenProvider tokenProvider = new MsiTokenProvider(authEndpoint, + tenantGuid, clientId, authority); + AzureADToken token = null; + intercept(AzureADAuthenticator.HttpException.class, + tokenProvider::getToken); + + // If the status code doesn't qualify for retry shouldRetry returns false and the loop ends. + // It being called multiple times verifies that the retry was done for the throttling status code 429. + Mockito.verify(exponentialRetryPolicy, + times(DEFAULT_AZURE_OAUTH_TOKEN_FETCH_RETRY_MAX_ATTEMPTS + 1)) + .shouldRetry(Mockito.anyInt(), Mockito.anyInt()); + } + } } From 3be0b00a59ca2e0286bdd44dbb20e3d0f76b8d4d Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Wed, 4 Jan 2023 21:53:31 -0800 Subject: [PATCH 05/18] Removing unused imports --- .../apache/hadoop/fs/azurebfs/oauth2/AzureADAuthenticator.java | 1 - 1 file changed, 1 deletion(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/AzureADAuthenticator.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/AzureADAuthenticator.java index 94c9b0b65f782..eaa6d1d9c2661 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/AzureADAuthenticator.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/AzureADAuthenticator.java @@ -29,7 +29,6 @@ import java.util.Hashtable; import java.util.Map; -import org.apache.hadoop.classification.VisibleForTesting; import org.apache.hadoop.util.Preconditions; import com.fasterxml.jackson.core.JsonFactory; From e04011ca7ca13cb0ab3146ee54e5c9bbf7aefddb Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Thu, 5 Jan 2023 00:18:59 -0800 Subject: [PATCH 06/18] Adding negative tests --- .../azurebfs/ITestAbfsMsiTokenProvider.java | 57 ++++++++++++++++++- 1 file changed, 56 insertions(+), 1 deletion(-) diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsMsiTokenProvider.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsMsiTokenProvider.java index 6c59e25308cd2..c93561005db7e 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsMsiTokenProvider.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsMsiTokenProvider.java @@ -20,6 +20,7 @@ import java.io.IOException; import java.lang.reflect.Field; +import java.net.HttpURLConnection; import java.util.Date; import org.junit.Test; import org.mockito.MockedStatic; @@ -54,6 +55,8 @@ public final class ITestAbfsMsiTokenProvider extends AbstractAbfsIntegrationTest { + private static final int HTTP_TOO_MANY_REQUESTS = 429; + public ITestAbfsMsiTokenProvider() throws Exception { super(); } @@ -103,7 +106,6 @@ private String getTrimmedPasswordString(AbfsConfiguration conf, String key, */ @Test public void testRetryForThrottling() throws Exception { - final int HTTP_TOO_MANY_REQUESTS = 429; AbfsConfiguration conf = getConfiguration(); // Exception to be thrown with throttling error code 429. @@ -150,4 +152,57 @@ public void testRetryForThrottling() throws Exception { .shouldRetry(Mockito.anyInt(), Mockito.anyInt()); } } + + /** + * Test to verify that token fetch is not retried for resource not found errors. + * @throws Exception + */ + @Test + public void testNoRetryForResourceNotFound() throws Exception { + AbfsConfiguration conf = getConfiguration(); + + // Exception to be thrown 404 error code. + AzureADAuthenticator.HttpException httpException + = new AzureADAuthenticator.HttpException(HttpURLConnection.HTTP_NOT_FOUND, + "abc", "abc", "abc", "abc", "abc"); + + String tenantGuid = "abcd"; + String clientId = "abcd"; + String authEndpoint = getTrimmedPasswordString(conf, + FS_AZURE_ACCOUNT_OAUTH_MSI_ENDPOINT, + DEFAULT_FS_AZURE_ACCOUNT_OAUTH_MSI_ENDPOINT); + String authority = getTrimmedPasswordString(conf, + FS_AZURE_ACCOUNT_OAUTH_MSI_AUTHORITY, + DEFAULT_FS_AZURE_ACCOUNT_OAUTH_MSI_AUTHORITY); + + // Mock the getTokenSingleCall to throw exception. + try (MockedStatic adAuthenticator = Mockito.mockStatic( + AzureADAuthenticator.class, Mockito.CALLS_REAL_METHODS)) { + adAuthenticator.when( + () -> AzureADAuthenticator.getTokenSingleCall(Mockito.anyString(), + Mockito.anyString(), Mockito.any(), Mockito.anyString(), + Mockito.anyBoolean())).thenThrow(httpException); + + // Mock the tokenFetchRetryPolicy to verify no retries. + ExponentialRetryPolicy exponentialRetryPolicy = Mockito.spy( + conf.getOauthTokenFetchRetryPolicy()); + Field tokenFetchRetryPolicy = AzureADAuthenticator.class.getDeclaredField( + "tokenFetchRetryPolicy"); + tokenFetchRetryPolicy.setAccessible(true); + tokenFetchRetryPolicy.set(ExponentialRetryPolicy.class, + exponentialRetryPolicy); + + AccessTokenProvider tokenProvider = new MsiTokenProvider(authEndpoint, + tenantGuid, clientId, authority); + AzureADToken token = null; + intercept(AzureADAuthenticator.HttpException.class, + tokenProvider::getToken); + + // If the status code doesn't qualify for retry shouldRetry returns false and the loop ends. + // It being called only once verifies that retry doesn't come into place.. + Mockito.verify(exponentialRetryPolicy, + times(1)) + .shouldRetry(Mockito.anyInt(), Mockito.anyInt()); + } + } } From e6740a37b2d5d5f4b77c5b81ed2bfdb8a80bde16 Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Tue, 22 Aug 2023 00:34:01 -0700 Subject: [PATCH 07/18] Resolving PR comments --- hadoop-project/pom.xml | 14 ++++++++++- hadoop-tools/hadoop-azure/pom.xml | 8 ------ .../azurebfs/constants/AbfsHttpConstants.java | 6 +++++ .../azurebfs/oauth2/AzureADAuthenticator.java | 25 +++++++++++++++++++ .../services/ExponentialRetryPolicy.java | 8 +----- .../azurebfs/ITestAbfsMsiTokenProvider.java | 6 ++--- 6 files changed, 48 insertions(+), 19 deletions(-) diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index c837a6fc0d5df..b886926233899 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -1308,7 +1308,19 @@ org.mockito mockito-core - 2.28.2 + 4.11.0 + + + org.mockito + mockito-inline + 4.11.0 + test + + + org.mockito + mockito-core + + org.mockito diff --git a/hadoop-tools/hadoop-azure/pom.xml b/hadoop-tools/hadoop-azure/pom.xml index 67b22173e0537..7dcd4bf1e6a2d 100644 --- a/hadoop-tools/hadoop-azure/pom.xml +++ b/hadoop-tools/hadoop-azure/pom.xml @@ -321,21 +321,13 @@ org.mockito mockito-core - 4.11.0 test org.mockito mockito-inline - 4.11.0 test - - - org.mockito - mockito-core - - diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/AbfsHttpConstants.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/AbfsHttpConstants.java index 7e4ddfa675a4c..e7b56383374e5 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/AbfsHttpConstants.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/AbfsHttpConstants.java @@ -111,6 +111,12 @@ public final class AbfsHttpConstants { // The HTTP 100 Continue informational status response code indicates that everything so far // is OK and that the client should continue with the request or ignore it if it is already finished. public static final String HUNDRED_CONTINUE = "100-continue"; + /** + * HTTP status code indicating that the server has received too many requests and the client should + * qualify for retrying the operation, as described in the Microsoft Azure documentation. + * {@link "https://learn.microsoft.com/en-us/azure/active-directory/managed-identities-azure-resources/how-to-use-vm-token#error-handling"}. + */ + public static final int HTTP_TOO_MANY_REQUESTS = 429; public static final char CHAR_FORWARD_SLASH = '/'; public static final char CHAR_EXCLAMATION_POINT = '!'; diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/AzureADAuthenticator.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/AzureADAuthenticator.java index eaa6d1d9c2661..7f0e9a1c8effc 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/AzureADAuthenticator.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/AzureADAuthenticator.java @@ -213,6 +213,18 @@ public String getRequestId() { return this.requestId; } + /** + Constructs an instance of HttpException with detailed information about an HTTP error response. + This exception is designed to encapsulate details of an HTTP error response, providing context about the error + encountered during an HTTP operation. It includes the HTTP error code, the associated request ID, an error message, + the URL that triggered the error, the content type of the response, and the response body. + @param httpErrorCode The HTTP error code indicating the nature of the encountered error. + @param requestId The unique identifier associated with the corresponding HTTP request. + @param message A descriptive error message providing additional information about the encountered error. + @param url The URL that resulted in the HTTP error response. + @param contentType The content type of the HTTP response. + @param body The body of the HTTP response, containing more details about the error. + */ public HttpException( final int httpErrorCode, final String requestId, @@ -341,6 +353,19 @@ private static boolean isRecoverableFailure(IOException e) { || e instanceof FileNotFoundException); } +/** + Retrieves an Azure OAuth token for authentication through a single API call. + This method facilitates the acquisition of an OAuth token from Azure Active Directory + to enable secure authentication for various services. It supports both Managed Service Identity (MSI) + tokens and non-MSI tokens based on the provided parameters. + @param authEndpoint The URL endpoint for OAuth token retrieval. + @param payload The payload to be included in the token request. This typically contains grant type and + any required parameters for token acquisition. + @param headers A Hashtable containing additional HTTP headers to be included in the token request. + @param httpMethod The HTTP method to be used for the token request (e.g., GET, POST). + @param isMsi A boolean flag indicating whether to request a Managed Service Identity (MSI) token or not. + @return An AzureADToken object containing the acquired OAuth token and associated metadata. + */ public static AzureADToken getTokenSingleCall(String authEndpoint, String payload, Hashtable headers, String httpMethod, boolean isMsi) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ExponentialRetryPolicy.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ExponentialRetryPolicy.java index 2be44a6903b4f..d4a9e704a2219 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ExponentialRetryPolicy.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ExponentialRetryPolicy.java @@ -25,6 +25,7 @@ import org.apache.hadoop.classification.VisibleForTesting; import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.HTTP_CONTINUE; +import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.HTTP_TOO_MANY_REQUESTS; /** * Retry policy used by AbfsClient. @@ -58,13 +59,6 @@ public class ExponentialRetryPolicy { */ private static final double MAX_RANDOM_RATIO = 1.2; - /** - * Qualifies for retry based on - * https://learn.microsoft.com/en-us/azure/active-directory/ - * managed-identities-azure-resources/how-to-use-vm-token#error-handling - */ - private static final int HTTP_TOO_MANY_REQUESTS = 429; - /** * Holds the random number generator used to calculate randomized backoff intervals */ diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsMsiTokenProvider.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsMsiTokenProvider.java index c93561005db7e..35710febc0631 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsMsiTokenProvider.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsMsiTokenProvider.java @@ -22,6 +22,8 @@ import java.lang.reflect.Field; import java.net.HttpURLConnection; import java.util.Date; + +import org.junit.Assume; import org.junit.Test; import org.mockito.MockedStatic; import org.mockito.Mockito; @@ -33,6 +35,7 @@ import org.apache.hadoop.fs.azurebfs.oauth2.MsiTokenProvider; import org.apache.hadoop.fs.azurebfs.services.ExponentialRetryPolicy; +import static org.apache.hadoop.fs.azurebfs.constants.AbfsHttpConstants.HTTP_TOO_MANY_REQUESTS; import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_AZURE_OAUTH_TOKEN_FETCH_RETRY_MAX_ATTEMPTS; import static org.apache.hadoop.test.LambdaTestUtils.intercept; import static org.junit.Assume.assumeThat; @@ -55,8 +58,6 @@ public final class ITestAbfsMsiTokenProvider extends AbstractAbfsIntegrationTest { - private static final int HTTP_TOO_MANY_REQUESTS = 429; - public ITestAbfsMsiTokenProvider() throws Exception { super(); } @@ -102,7 +103,6 @@ private String getTrimmedPasswordString(AbfsConfiguration conf, String key, /** * Test to verify that token fetch is retried for throttling errors (too many requests 429). - * @throws Exception */ @Test public void testRetryForThrottling() throws Exception { From 0688809c43766c8d001a39c972e263d102b2df82 Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Tue, 22 Aug 2023 08:57:44 -0700 Subject: [PATCH 08/18] Mockito fix --- .../org/apache/hadoop/ipc/TestServer.java | 2 +- .../security/http/TestCrossOriginFilter.java | 9 +++---- .../http/TestRestCsrfPreventionFilter.java | 10 +++---- hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml | 5 ++++ .../federation/router/TestRouterAdmin.java | 16 +++++++----- .../router/TestRouterRpcMultiDestination.java | 4 +-- .../server/namenode/TestCacheDirectives.java | 2 +- .../namenode/TestSnapshotPathINodes.java | 2 +- hadoop-project/pom.xml | 1 + .../containerlaunch/TestAbstractLauncher.java | 4 +-- .../resources/TestCGroupsHandlerImpl.java | 8 +++--- .../gpu/TestGpuResourceAllocator.java | 4 +-- .../runtime/TestDockerContainerRuntime.java | 4 +-- .../com/nec/TestNECVEPlugin.java | 6 ++--- ...TestFSConfigToCSConfigArgumentHandler.java | 6 ++--- .../TestFSConfigToCSConfigConverter.java | 4 +-- .../TestQueuePlacementConverter.java | 26 +++++++++---------- hadoop-yarn-project/pom.xml | 5 ++++ 18 files changed, 65 insertions(+), 53 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestServer.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestServer.java index 748d99e2a0d34..b6e696fddeefa 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestServer.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestServer.java @@ -151,7 +151,7 @@ public Writable call( // Nothing should be logged for a suppressed exception. server.logException(logger, new TestException1(), dummyCall); - verifyZeroInteractions(logger); + verifyNoInteractions(logger); // No stack trace should be logged for a terse exception. server.logException(logger, new TestException2(), dummyCall); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/http/TestCrossOriginFilter.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/http/TestCrossOriginFilter.java index 0b396be48f983..9f7a6114b0ac3 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/http/TestCrossOriginFilter.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/http/TestCrossOriginFilter.java @@ -58,8 +58,7 @@ public void testSameOrigin() throws ServletException, IOException { CrossOriginFilter filter = new CrossOriginFilter(); filter.init(filterConfig); filter.doFilter(mockReq, mockRes, mockChain); - - Mockito.verifyZeroInteractions(mockRes); + Mockito.verifyNoInteractions(mockRes); Mockito.verify(mockChain).doFilter(mockReq, mockRes); } @@ -224,7 +223,7 @@ public void testDisallowedOrigin() throws ServletException, IOException { filter.init(filterConfig); filter.doFilter(mockReq, mockRes, mockChain); - Mockito.verifyZeroInteractions(mockRes); + Mockito.verifyNoInteractions(mockRes); Mockito.verify(mockChain).doFilter(mockReq, mockRes); } @@ -252,7 +251,7 @@ public void testDisallowedMethod() throws ServletException, IOException { filter.init(filterConfig); filter.doFilter(mockReq, mockRes, mockChain); - Mockito.verifyZeroInteractions(mockRes); + Mockito.verifyNoInteractions(mockRes); Mockito.verify(mockChain).doFilter(mockReq, mockRes); } @@ -283,7 +282,7 @@ public void testDisallowedHeader() throws ServletException, IOException { filter.init(filterConfig); filter.doFilter(mockReq, mockRes, mockChain); - Mockito.verifyZeroInteractions(mockRes); + Mockito.verifyNoInteractions(mockRes); Mockito.verify(mockChain).doFilter(mockReq, mockRes); } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/http/TestRestCsrfPreventionFilter.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/http/TestRestCsrfPreventionFilter.java index 6052ef059a732..f125ffac1e191 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/http/TestRestCsrfPreventionFilter.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/http/TestRestCsrfPreventionFilter.java @@ -75,7 +75,7 @@ public void testNoHeaderDefaultConfigBadRequest() verify(mockRes, atLeastOnce()).sendError( HttpServletResponse.SC_BAD_REQUEST, EXPECTED_MESSAGE); - Mockito.verifyZeroInteractions(mockChain); + Mockito.verifyNoInteractions(mockChain); } @Test @@ -110,7 +110,7 @@ public void testNoHeaderCustomAgentConfigBadRequest() verify(mockRes, atLeastOnce()).sendError( HttpServletResponse.SC_BAD_REQUEST, EXPECTED_MESSAGE); - Mockito.verifyZeroInteractions(mockChain); + Mockito.verifyNoInteractions(mockChain); } @Test @@ -228,7 +228,7 @@ public void testMissingHeaderWithCustomHeaderConfigBadRequest() filter.init(filterConfig); filter.doFilter(mockReq, mockRes, mockChain); - Mockito.verifyZeroInteractions(mockChain); + Mockito.verifyNoInteractions(mockChain); } @Test @@ -260,7 +260,7 @@ public void testMissingHeaderNoMethodsToIgnoreConfigBadRequest() filter.init(filterConfig); filter.doFilter(mockReq, mockRes, mockChain); - Mockito.verifyZeroInteractions(mockChain); + Mockito.verifyNoInteractions(mockChain); } @Test @@ -356,6 +356,6 @@ public void testMissingHeaderMultipleIgnoreMethodsConfigBadRequest() filter.init(filterConfig); filter.doFilter(mockReq, mockRes, mockChain); - Mockito.verifyZeroInteractions(mockChain); + Mockito.verifyNoInteractions(mockChain); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml index e3bb52365fe82..8e2336698d675 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml @@ -177,6 +177,11 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd"> junit-jupiter-params test + + org.mockito + mockito-core + test + diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdmin.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdmin.java index c2eaddc17a2a0..18999a8d7f587 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdmin.java @@ -25,6 +25,7 @@ import static org.junit.Assert.assertTrue; import java.io.IOException; +import java.lang.reflect.Field; import java.security.PrivilegedExceptionAction; import java.util.Collections; import java.util.HashMap; @@ -62,13 +63,13 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.test.LambdaTestUtils; import org.apache.hadoop.util.Lists; +import org.apache.hadoop.util.ReflectionUtils; import org.apache.hadoop.util.Time; import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; import org.mockito.Mockito; -import org.mockito.internal.util.reflection.FieldSetter; /** * The administrator interface of the {@link Router} implemented by @@ -118,18 +119,19 @@ public static void globalSetUp() throws Exception { * @throws IOException * @throws NoSuchFieldException */ - private static void setUpMocks() throws IOException, NoSuchFieldException { + private static void setUpMocks() throws IOException, NoSuchFieldException, IllegalAccessException { RouterRpcServer spyRpcServer = Mockito.spy(routerContext.getRouter().createRpcServer()); - FieldSetter.setField(routerContext.getRouter(), - Router.class.getDeclaredField("rpcServer"), spyRpcServer); + Field rpcServerField = Router.class.getDeclaredField("rpcServer"); + rpcServerField.setAccessible(true); + rpcServerField.set(routerContext.getRouter(), spyRpcServer); Mockito.doReturn(null).when(spyRpcServer).getFileInfo(Mockito.anyString()); // mock rpc client for destination check when editing mount tables. mockRpcClient = Mockito.spy(spyRpcServer.getRPCClient()); - FieldSetter.setField(spyRpcServer, - RouterRpcServer.class.getDeclaredField("rpcClient"), - mockRpcClient); + Field rpcClientField = RouterRpcServer.class.getDeclaredField("rpcClient"); + rpcClientField.setAccessible(true); + rpcClientField.set(spyRpcServer, mockRpcClient); RemoteLocation remoteLocation0 = new RemoteLocation("ns0", "/testdir", null); RemoteLocation remoteLocation1 = diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpcMultiDestination.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpcMultiDestination.java index 336ea3913859e..e730a7978b1c8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpcMultiDestination.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpcMultiDestination.java @@ -24,7 +24,6 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; -import static org.mockito.Matchers.any; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.apache.hadoop.test.Whitebox.getInternalState; @@ -73,6 +72,7 @@ import org.apache.hadoop.ipc.StandbyException; import org.apache.hadoop.test.GenericTestUtils; import org.junit.Test; +import org.mockito.Mockito; import org.slf4j.event.Level; /** @@ -423,7 +423,7 @@ public void testSubclusterDown() throws Exception { FSNamesystem ns0 = nn0.getNamesystem(); HAContext nn0haCtx = (HAContext)getInternalState(ns0, "haContext"); HAContext mockCtx = mock(HAContext.class); - doThrow(new StandbyException("Mock")).when(mockCtx).checkOperation(any()); + doThrow(new StandbyException("Mock")).when(mockCtx).checkOperation(Mockito.any()); setInternalState(ns0, "haContext", mockCtx); // router0 should throw an exception diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java index 1331c50e80b3a..48c1527de1031 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java @@ -1575,7 +1575,7 @@ public void testNoLookupsWhenNotUsed() throws Exception { CacheManager cm = cluster.getNamesystem().getCacheManager(); LocatedBlocks locations = Mockito.mock(LocatedBlocks.class); cm.setCachedLocations(locations); - Mockito.verifyZeroInteractions(locations); + Mockito.verifyNoInteractions(locations); } @Test(timeout=120000) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSnapshotPathINodes.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSnapshotPathINodes.java index b62a4180d43ba..cda5b39f1d03e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSnapshotPathINodes.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSnapshotPathINodes.java @@ -447,6 +447,6 @@ public void testShortCircuitSnapshotSearch() throws SnapshotException { INodesInPath iip = Mockito.mock(INodesInPath.class); List snapDirs = new ArrayList<>(); FSDirSnapshotOp.checkSnapshot(fsn.getFSDirectory(), iip, snapDirs); - Mockito.verifyZeroInteractions(iip); + Mockito.verifyNoInteractions(iip); } } diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index b886926233899..09940b9c16d23 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -1309,6 +1309,7 @@ org.mockito mockito-core 4.11.0 + compile org.mockito diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/containerlaunch/TestAbstractLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/containerlaunch/TestAbstractLauncher.java index 31ca38297c856..6b3da713964cc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/containerlaunch/TestAbstractLauncher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/containerlaunch/TestAbstractLauncher.java @@ -43,7 +43,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.reset; import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyZeroInteractions; +import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; /** @@ -90,7 +90,7 @@ public void testContainerRetries() throws Exception { providerService.buildContainerRetry(mockLauncher, getConfig(), componentLaunchContext, componentInstance); - verifyZeroInteractions(mockLauncher); + verifyNoInteractions(mockLauncher); //OnFailure restart policy diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsHandlerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsHandlerImpl.java index b1e8989213ba1..fc5293a06d136 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsHandlerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsHandlerImpl.java @@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources; import org.apache.commons.io.FileUtils; +import org.mockito.Mockito; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; @@ -52,7 +53,6 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoMoreInteractions; -import static org.mockito.Mockito.verifyZeroInteractions; /** * Tests for the CGroups handler implementation. @@ -192,7 +192,7 @@ public void testMountController() throws IOException { assertTrue("cgroup dir should be cerated", cgroup.mkdirs()); //Since we enabled (deferred) cgroup controller mounting, no interactions //should have occurred, with this mock - verifyZeroInteractions(privilegedOperationExecutorMock); + Mockito.verifyNoInteractions(privilegedOperationExecutorMock); File emptyMtab = createEmptyCgroups(); try { @@ -238,7 +238,7 @@ public void testMountController() throws IOException { public void testCGroupPaths() throws IOException { //As per junit behavior, we expect a new mock object to be available //in this test. - verifyZeroInteractions(privilegedOperationExecutorMock); + Mockito.verifyNoInteractions(privilegedOperationExecutorMock); CGroupsHandler cGroupsHandler = null; File mtab = createEmptyCgroups(); @@ -281,7 +281,7 @@ public void testCGroupPaths() throws IOException { public void testCGroupOperations() throws IOException { //As per junit behavior, we expect a new mock object to be available //in this test. - verifyZeroInteractions(privilegedOperationExecutorMock); + Mockito.verifyNoInteractions(privilegedOperationExecutorMock); CGroupsHandler cGroupsHandler = null; File mtab = createEmptyCgroups(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/TestGpuResourceAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/TestGpuResourceAllocator.java index ba8a9309d02e9..427b7dce656b9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/TestGpuResourceAllocator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/TestGpuResourceAllocator.java @@ -27,7 +27,6 @@ import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyZeroInteractions; import static org.mockito.Mockito.when; import java.io.IOException; @@ -57,6 +56,7 @@ import org.mockito.ArgumentMatcher; import org.mockito.Captor; import org.mockito.Mock; +import org.mockito.Mockito; import org.mockito.MockitoAnnotations; /** @@ -210,7 +210,7 @@ private void assertAllocatedGpus(int gpus, int deniedGpus, private void assertNoAllocation(GpuAllocation allocation) { assertEquals(1, allocation.getDeniedGPUs().size()); assertEquals(0, allocation.getAllowedGPUs().size()); - verifyZeroInteractions(nmStateStore); + Mockito.verifyNoInteractions(nmStateStore); } private void assertAssignmentInStateStore(GpuDevice expectedGpu, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java index ea7c213809330..6d033f97e1891 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java @@ -1272,7 +1272,7 @@ public void testCGroupParent() throws ContainerExecutionException { command); //no --cgroup-parent should be added here - Mockito.verifyZeroInteractions(command); + Mockito.verifyNoInteractions(command); String resourceOptionsCpu = "/sys/fs/cgroup/cpu/" + hierarchy + containerIdStr; @@ -1296,7 +1296,7 @@ public void testCGroupParent() throws ContainerExecutionException { command); //no --cgroup-parent should be added in either case - Mockito.verifyZeroInteractions(command); + Mockito.verifyNoInteractions(command); } @Test diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/com/nec/TestNECVEPlugin.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/com/nec/TestNECVEPlugin.java index 86ef9058f26fb..a9b01bca6fac7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/com/nec/TestNECVEPlugin.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/com/nec/TestNECVEPlugin.java @@ -25,7 +25,7 @@ import static org.mockito.Mockito.when; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyZeroInteractions; +import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.ArgumentMatchers.anyString; @@ -394,7 +394,7 @@ public void testFindDevicesWithUdev() assertEquals("No. of devices", 1, devices.size()); Device device = devices.iterator().next(); assertSame("Device", device, testDevice); - verifyZeroInteractions(mockCommandExecutor); + verifyNoInteractions(mockCommandExecutor); verify(mockEnvProvider).apply(eq("NEC_USE_UDEV")); verifyNoMoreInteractions(mockEnvProvider); } @@ -442,6 +442,6 @@ private Device getTestDevice(int id) { private void verifyBinaryPathSet(Path expectedPath) { assertEquals("Binary path", expectedPath.toString(), plugin.getBinaryPath()); - verifyZeroInteractions(udevUtil); + verifyNoInteractions(udevUtil); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigArgumentHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigArgumentHandler.java index cb8cc587f68f7..33cc3f0dba266 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigArgumentHandler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigArgumentHandler.java @@ -23,7 +23,7 @@ import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyZeroInteractions; +import static org.mockito.Mockito.verifyNoInteractions; import java.io.File; import java.io.IOException; @@ -666,7 +666,7 @@ public void testValidationSkippedWhenCmdLineSwitchIsDefined() FSConfigConverterTestCommons.FS_ALLOC_FILE, "-s"); argumentHandler.parseAndConvert(args); - verifyZeroInteractions(mockValidator); + verifyNoInteractions(mockValidator); } @Test @@ -681,7 +681,7 @@ public void testValidationSkippedWhenOutputIsConsole() throws Exception { FSConfigConverterTestCommons.FS_ALLOC_FILE, "-s", "-p"); argumentHandler.parseAndConvert(args); - verifyZeroInteractions(mockValidator); + verifyNoInteractions(mockValidator); } @Test diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java index 55c43666cdb5b..530c6ddc55301 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java @@ -36,7 +36,7 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyZeroInteractions; +import static org.mockito.Mockito.verifyNoInteractions; import java.io.ByteArrayInputStream; import java.io.File; @@ -673,7 +673,7 @@ public void testPlacementRulesConversionDisabled() throws Exception { converter.setPlacementConverter(placementConverter); converter.convert(params); - verifyZeroInteractions(placementConverter); + verifyNoInteractions(placementConverter); } @Test diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestQueuePlacementConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestQueuePlacementConverter.java index 6599080aab59c..6871444baa03c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestQueuePlacementConverter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestQueuePlacementConverter.java @@ -21,7 +21,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoMoreInteractions; -import static org.mockito.Mockito.verifyZeroInteractions; +import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; import java.util.List; @@ -85,7 +85,7 @@ public void testConvertUserRule() { MappingRulesDescription description = convert(); assertEquals("Number of rules", 1, description.getRules().size()); verifyRule(description.getRules().get(0), Policy.USER); - verifyZeroInteractions(ruleHandler); + verifyNoInteractions(ruleHandler); } @Test @@ -96,7 +96,7 @@ public void testConvertSpecifiedRule() { MappingRulesDescription description = convert(); assertEquals("Number of rules", 1, description.getRules().size()); verifyRule(description.getRules().get(0), Policy.SPECIFIED); - verifyZeroInteractions(ruleHandler); + verifyNoInteractions(ruleHandler); } @Test @@ -108,7 +108,7 @@ public void testConvertPrimaryGroupRule() { assertEquals("Number of rules", 1, description.getRules().size()); verifyRule(description.getRules().get(0), Policy.PRIMARY_GROUP); - verifyZeroInteractions(ruleHandler); + verifyNoInteractions(ruleHandler); } @Test @@ -120,7 +120,7 @@ public void testConvertSecondaryGroupRule() { assertEquals("Number of rules", 1, description.getRules().size()); verifyRule(description.getRules().get(0), Policy.SECONDARY_GROUP); - verifyZeroInteractions(ruleHandler); + verifyNoInteractions(ruleHandler); } @Test @@ -134,7 +134,7 @@ public void testConvertDefaultRuleWithQueueName() { assertEquals("Number of rules", 1, description.getRules().size()); verifyRule(description.getRules().get(0), Policy.CUSTOM); - verifyZeroInteractions(ruleHandler); + verifyNoInteractions(ruleHandler); } @Test @@ -147,7 +147,7 @@ public void testConvertDefaultRule() { assertEquals("Number of rules", 1, description.getRules().size()); verifyRule(description.getRules().get(0), Policy.DEFAULT_QUEUE); - verifyZeroInteractions(ruleHandler); + verifyNoInteractions(ruleHandler); } @Test(expected = IllegalArgumentException.class) @@ -168,7 +168,7 @@ public void testConvertRejectRule() { assertEquals("Number of rules", 1, description.getRules().size()); verifyRule(description.getRules().get(0), Policy.REJECT); - verifyZeroInteractions(ruleHandler); + verifyNoInteractions(ruleHandler); } @Test @@ -182,7 +182,7 @@ public void testConvertNestedPrimaryGroupRule() { assertEquals("Number of rules", 1, description.getRules().size()); verifyRule(description.getRules().get(0), Policy.PRIMARY_GROUP_USER); - verifyZeroInteractions(ruleHandler); + verifyNoInteractions(ruleHandler); } @Test @@ -197,7 +197,7 @@ public void testConvertNestedSecondaryGroupRule() { assertEquals("Number of rules", 1, description.getRules().size()); verifyRule(description.getRules().get(0), Policy.SECONDARY_GROUP_USER); - verifyZeroInteractions(ruleHandler); + verifyNoInteractions(ruleHandler); } @Test @@ -215,7 +215,7 @@ public void testConvertNestedDefaultRule() { Rule rule = description.getRules().get(0); verifyRule(description.getRules().get(0), Policy.USER); assertEquals("Parent path", "root.abc", rule.getParentQueue()); - verifyZeroInteractions(ruleHandler); + verifyNoInteractions(ruleHandler); } @Test(expected = IllegalArgumentException.class) @@ -245,7 +245,7 @@ public void testConvertMultiplePlacementRules() { verifyRule(description.getRules().get(0), Policy.USER); verifyRule(description.getRules().get(1), Policy.PRIMARY_GROUP); verifyRule(description.getRules().get(2), Policy.SECONDARY_GROUP); - verifyZeroInteractions(ruleHandler); + verifyNoInteractions(ruleHandler); } @Test @@ -363,7 +363,7 @@ private void testConvertNestedRuleCreateFlagInWeightMode( any(Policy.class)); verifyNoMoreInteractions(ruleHandler); } else { - verifyZeroInteractions(ruleHandler); + verifyNoInteractions(ruleHandler); } } diff --git a/hadoop-yarn-project/pom.xml b/hadoop-yarn-project/pom.xml index 241e3bc237a0e..78f09a3971069 100644 --- a/hadoop-yarn-project/pom.xml +++ b/hadoop-yarn-project/pom.xml @@ -90,6 +90,11 @@ hadoop-yarn-applications-catalog-webapp war + + org.mockito + mockito-core + test + From 115e88a3f22437d3ba8e9ffc05523e5593b8bb61 Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Tue, 22 Aug 2023 22:48:51 -0700 Subject: [PATCH 09/18] Added in other PR --- hadoop-project/pom.xml | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index b886926233899..9fca0fa159b87 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -1300,27 +1300,10 @@ jackson-dataformat-cbor ${jackson2.version} - - com.fasterxml.jackson.dataformat - jackson-dataformat-yaml - ${jackson2.version} - org.mockito mockito-core - 4.11.0 - - - org.mockito - mockito-inline - 4.11.0 - test - - - org.mockito - mockito-core - - + 2.28.2 org.mockito From 01ec633658bd3daa7a405f49ae7684753375a2c7 Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Tue, 22 Aug 2023 22:54:03 -0700 Subject: [PATCH 10/18] Removed new version changes --- hadoop-project/pom.xml | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index b886926233899..9fca0fa159b87 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -1300,27 +1300,10 @@ jackson-dataformat-cbor ${jackson2.version} - - com.fasterxml.jackson.dataformat - jackson-dataformat-yaml - ${jackson2.version} - org.mockito mockito-core - 4.11.0 - - - org.mockito - mockito-inline - 4.11.0 - test - - - org.mockito - mockito-core - - + 2.28.2 org.mockito From 24c0483e1bedc7985f345314480ec23fbcc3c928 Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Tue, 22 Aug 2023 22:58:43 -0700 Subject: [PATCH 11/18] Reverted --- .../azurebfs/ITestAbfsMsiTokenProvider.java | 55 +------------------ 1 file changed, 1 insertion(+), 54 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsMsiTokenProvider.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsMsiTokenProvider.java index 35710febc0631..3435e65f0fd2f 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsMsiTokenProvider.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsMsiTokenProvider.java @@ -20,7 +20,6 @@ import java.io.IOException; import java.lang.reflect.Field; -import java.net.HttpURLConnection; import java.util.Date; import org.junit.Assume; @@ -106,6 +105,7 @@ private String getTrimmedPasswordString(AbfsConfiguration conf, String key, */ @Test public void testRetryForThrottling() throws Exception { + final int HTTP_TOO_MANY_REQUESTS = 429; AbfsConfiguration conf = getConfiguration(); // Exception to be thrown with throttling error code 429. @@ -152,57 +152,4 @@ public void testRetryForThrottling() throws Exception { .shouldRetry(Mockito.anyInt(), Mockito.anyInt()); } } - - /** - * Test to verify that token fetch is not retried for resource not found errors. - * @throws Exception - */ - @Test - public void testNoRetryForResourceNotFound() throws Exception { - AbfsConfiguration conf = getConfiguration(); - - // Exception to be thrown 404 error code. - AzureADAuthenticator.HttpException httpException - = new AzureADAuthenticator.HttpException(HttpURLConnection.HTTP_NOT_FOUND, - "abc", "abc", "abc", "abc", "abc"); - - String tenantGuid = "abcd"; - String clientId = "abcd"; - String authEndpoint = getTrimmedPasswordString(conf, - FS_AZURE_ACCOUNT_OAUTH_MSI_ENDPOINT, - DEFAULT_FS_AZURE_ACCOUNT_OAUTH_MSI_ENDPOINT); - String authority = getTrimmedPasswordString(conf, - FS_AZURE_ACCOUNT_OAUTH_MSI_AUTHORITY, - DEFAULT_FS_AZURE_ACCOUNT_OAUTH_MSI_AUTHORITY); - - // Mock the getTokenSingleCall to throw exception. - try (MockedStatic adAuthenticator = Mockito.mockStatic( - AzureADAuthenticator.class, Mockito.CALLS_REAL_METHODS)) { - adAuthenticator.when( - () -> AzureADAuthenticator.getTokenSingleCall(Mockito.anyString(), - Mockito.anyString(), Mockito.any(), Mockito.anyString(), - Mockito.anyBoolean())).thenThrow(httpException); - - // Mock the tokenFetchRetryPolicy to verify no retries. - ExponentialRetryPolicy exponentialRetryPolicy = Mockito.spy( - conf.getOauthTokenFetchRetryPolicy()); - Field tokenFetchRetryPolicy = AzureADAuthenticator.class.getDeclaredField( - "tokenFetchRetryPolicy"); - tokenFetchRetryPolicy.setAccessible(true); - tokenFetchRetryPolicy.set(ExponentialRetryPolicy.class, - exponentialRetryPolicy); - - AccessTokenProvider tokenProvider = new MsiTokenProvider(authEndpoint, - tenantGuid, clientId, authority); - AzureADToken token = null; - intercept(AzureADAuthenticator.HttpException.class, - tokenProvider::getToken); - - // If the status code doesn't qualify for retry shouldRetry returns false and the loop ends. - // It being called only once verifies that retry doesn't come into place.. - Mockito.verify(exponentialRetryPolicy, - times(1)) - .shouldRetry(Mockito.anyInt(), Mockito.anyInt()); - } - } } From 99e66fa1ec82f4e16b48e0339416f2d6357e2e1b Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Tue, 22 Aug 2023 23:25:08 -0700 Subject: [PATCH 12/18] Reverted mockito changes --- .../org/apache/hadoop/ipc/TestServer.java | 2 +- .../security/http/TestCrossOriginFilter.java | 9 ++++--- .../http/TestRestCsrfPreventionFilter.java | 10 +++---- hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml | 5 ---- .../federation/router/TestRouterAdmin.java | 16 +++++------- .../router/TestRouterRpcMultiDestination.java | 4 +-- .../server/namenode/TestCacheDirectives.java | 2 +- .../namenode/TestSnapshotPathINodes.java | 2 +- .../containerlaunch/TestAbstractLauncher.java | 4 +-- .../resources/TestCGroupsHandlerImpl.java | 8 +++--- .../gpu/TestGpuResourceAllocator.java | 4 +-- .../runtime/TestDockerContainerRuntime.java | 4 +-- .../com/nec/TestNECVEPlugin.java | 6 ++--- ...TestFSConfigToCSConfigArgumentHandler.java | 6 ++--- .../TestFSConfigToCSConfigConverter.java | 4 +-- .../TestQueuePlacementConverter.java | 26 +++++++++---------- hadoop-yarn-project/pom.xml | 5 ---- 17 files changed, 53 insertions(+), 64 deletions(-) diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestServer.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestServer.java index b6e696fddeefa..748d99e2a0d34 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestServer.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestServer.java @@ -151,7 +151,7 @@ public Writable call( // Nothing should be logged for a suppressed exception. server.logException(logger, new TestException1(), dummyCall); - verifyNoInteractions(logger); + verifyZeroInteractions(logger); // No stack trace should be logged for a terse exception. server.logException(logger, new TestException2(), dummyCall); diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/http/TestCrossOriginFilter.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/http/TestCrossOriginFilter.java index 9f7a6114b0ac3..0b396be48f983 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/http/TestCrossOriginFilter.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/http/TestCrossOriginFilter.java @@ -58,7 +58,8 @@ public void testSameOrigin() throws ServletException, IOException { CrossOriginFilter filter = new CrossOriginFilter(); filter.init(filterConfig); filter.doFilter(mockReq, mockRes, mockChain); - Mockito.verifyNoInteractions(mockRes); + + Mockito.verifyZeroInteractions(mockRes); Mockito.verify(mockChain).doFilter(mockReq, mockRes); } @@ -223,7 +224,7 @@ public void testDisallowedOrigin() throws ServletException, IOException { filter.init(filterConfig); filter.doFilter(mockReq, mockRes, mockChain); - Mockito.verifyNoInteractions(mockRes); + Mockito.verifyZeroInteractions(mockRes); Mockito.verify(mockChain).doFilter(mockReq, mockRes); } @@ -251,7 +252,7 @@ public void testDisallowedMethod() throws ServletException, IOException { filter.init(filterConfig); filter.doFilter(mockReq, mockRes, mockChain); - Mockito.verifyNoInteractions(mockRes); + Mockito.verifyZeroInteractions(mockRes); Mockito.verify(mockChain).doFilter(mockReq, mockRes); } @@ -282,7 +283,7 @@ public void testDisallowedHeader() throws ServletException, IOException { filter.init(filterConfig); filter.doFilter(mockReq, mockRes, mockChain); - Mockito.verifyNoInteractions(mockRes); + Mockito.verifyZeroInteractions(mockRes); Mockito.verify(mockChain).doFilter(mockReq, mockRes); } diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/http/TestRestCsrfPreventionFilter.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/http/TestRestCsrfPreventionFilter.java index f125ffac1e191..6052ef059a732 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/http/TestRestCsrfPreventionFilter.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/http/TestRestCsrfPreventionFilter.java @@ -75,7 +75,7 @@ public void testNoHeaderDefaultConfigBadRequest() verify(mockRes, atLeastOnce()).sendError( HttpServletResponse.SC_BAD_REQUEST, EXPECTED_MESSAGE); - Mockito.verifyNoInteractions(mockChain); + Mockito.verifyZeroInteractions(mockChain); } @Test @@ -110,7 +110,7 @@ public void testNoHeaderCustomAgentConfigBadRequest() verify(mockRes, atLeastOnce()).sendError( HttpServletResponse.SC_BAD_REQUEST, EXPECTED_MESSAGE); - Mockito.verifyNoInteractions(mockChain); + Mockito.verifyZeroInteractions(mockChain); } @Test @@ -228,7 +228,7 @@ public void testMissingHeaderWithCustomHeaderConfigBadRequest() filter.init(filterConfig); filter.doFilter(mockReq, mockRes, mockChain); - Mockito.verifyNoInteractions(mockChain); + Mockito.verifyZeroInteractions(mockChain); } @Test @@ -260,7 +260,7 @@ public void testMissingHeaderNoMethodsToIgnoreConfigBadRequest() filter.init(filterConfig); filter.doFilter(mockReq, mockRes, mockChain); - Mockito.verifyNoInteractions(mockChain); + Mockito.verifyZeroInteractions(mockChain); } @Test @@ -356,6 +356,6 @@ public void testMissingHeaderMultipleIgnoreMethodsConfigBadRequest() filter.init(filterConfig); filter.doFilter(mockReq, mockRes, mockChain); - Mockito.verifyNoInteractions(mockChain); + Mockito.verifyZeroInteractions(mockChain); } } diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml index 8e2336698d675..e3bb52365fe82 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml @@ -177,11 +177,6 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd"> junit-jupiter-params test - - org.mockito - mockito-core - test - diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdmin.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdmin.java index 18999a8d7f587..c2eaddc17a2a0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdmin.java @@ -25,7 +25,6 @@ import static org.junit.Assert.assertTrue; import java.io.IOException; -import java.lang.reflect.Field; import java.security.PrivilegedExceptionAction; import java.util.Collections; import java.util.HashMap; @@ -63,13 +62,13 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.test.LambdaTestUtils; import org.apache.hadoop.util.Lists; -import org.apache.hadoop.util.ReflectionUtils; import org.apache.hadoop.util.Time; import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; import org.mockito.Mockito; +import org.mockito.internal.util.reflection.FieldSetter; /** * The administrator interface of the {@link Router} implemented by @@ -119,19 +118,18 @@ public static void globalSetUp() throws Exception { * @throws IOException * @throws NoSuchFieldException */ - private static void setUpMocks() throws IOException, NoSuchFieldException, IllegalAccessException { + private static void setUpMocks() throws IOException, NoSuchFieldException { RouterRpcServer spyRpcServer = Mockito.spy(routerContext.getRouter().createRpcServer()); - Field rpcServerField = Router.class.getDeclaredField("rpcServer"); - rpcServerField.setAccessible(true); - rpcServerField.set(routerContext.getRouter(), spyRpcServer); + FieldSetter.setField(routerContext.getRouter(), + Router.class.getDeclaredField("rpcServer"), spyRpcServer); Mockito.doReturn(null).when(spyRpcServer).getFileInfo(Mockito.anyString()); // mock rpc client for destination check when editing mount tables. mockRpcClient = Mockito.spy(spyRpcServer.getRPCClient()); - Field rpcClientField = RouterRpcServer.class.getDeclaredField("rpcClient"); - rpcClientField.setAccessible(true); - rpcClientField.set(spyRpcServer, mockRpcClient); + FieldSetter.setField(spyRpcServer, + RouterRpcServer.class.getDeclaredField("rpcClient"), + mockRpcClient); RemoteLocation remoteLocation0 = new RemoteLocation("ns0", "/testdir", null); RemoteLocation remoteLocation1 = diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpcMultiDestination.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpcMultiDestination.java index e730a7978b1c8..336ea3913859e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpcMultiDestination.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpcMultiDestination.java @@ -24,6 +24,7 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import static org.mockito.Matchers.any; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.apache.hadoop.test.Whitebox.getInternalState; @@ -72,7 +73,6 @@ import org.apache.hadoop.ipc.StandbyException; import org.apache.hadoop.test.GenericTestUtils; import org.junit.Test; -import org.mockito.Mockito; import org.slf4j.event.Level; /** @@ -423,7 +423,7 @@ public void testSubclusterDown() throws Exception { FSNamesystem ns0 = nn0.getNamesystem(); HAContext nn0haCtx = (HAContext)getInternalState(ns0, "haContext"); HAContext mockCtx = mock(HAContext.class); - doThrow(new StandbyException("Mock")).when(mockCtx).checkOperation(Mockito.any()); + doThrow(new StandbyException("Mock")).when(mockCtx).checkOperation(any()); setInternalState(ns0, "haContext", mockCtx); // router0 should throw an exception diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java index 48c1527de1031..1331c50e80b3a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCacheDirectives.java @@ -1575,7 +1575,7 @@ public void testNoLookupsWhenNotUsed() throws Exception { CacheManager cm = cluster.getNamesystem().getCacheManager(); LocatedBlocks locations = Mockito.mock(LocatedBlocks.class); cm.setCachedLocations(locations); - Mockito.verifyNoInteractions(locations); + Mockito.verifyZeroInteractions(locations); } @Test(timeout=120000) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSnapshotPathINodes.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSnapshotPathINodes.java index cda5b39f1d03e..b62a4180d43ba 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSnapshotPathINodes.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSnapshotPathINodes.java @@ -447,6 +447,6 @@ public void testShortCircuitSnapshotSearch() throws SnapshotException { INodesInPath iip = Mockito.mock(INodesInPath.class); List snapDirs = new ArrayList<>(); FSDirSnapshotOp.checkSnapshot(fsn.getFSDirectory(), iip, snapDirs); - Mockito.verifyNoInteractions(iip); + Mockito.verifyZeroInteractions(iip); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/containerlaunch/TestAbstractLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/containerlaunch/TestAbstractLauncher.java index 6b3da713964cc..31ca38297c856 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/containerlaunch/TestAbstractLauncher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/containerlaunch/TestAbstractLauncher.java @@ -43,7 +43,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.reset; import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.verifyZeroInteractions; import static org.mockito.Mockito.when; /** @@ -90,7 +90,7 @@ public void testContainerRetries() throws Exception { providerService.buildContainerRetry(mockLauncher, getConfig(), componentLaunchContext, componentInstance); - verifyNoInteractions(mockLauncher); + verifyZeroInteractions(mockLauncher); //OnFailure restart policy diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsHandlerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsHandlerImpl.java index fc5293a06d136..b1e8989213ba1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsHandlerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsHandlerImpl.java @@ -21,7 +21,6 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources; import org.apache.commons.io.FileUtils; -import org.mockito.Mockito; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; @@ -53,6 +52,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.verifyZeroInteractions; /** * Tests for the CGroups handler implementation. @@ -192,7 +192,7 @@ public void testMountController() throws IOException { assertTrue("cgroup dir should be cerated", cgroup.mkdirs()); //Since we enabled (deferred) cgroup controller mounting, no interactions //should have occurred, with this mock - Mockito.verifyNoInteractions(privilegedOperationExecutorMock); + verifyZeroInteractions(privilegedOperationExecutorMock); File emptyMtab = createEmptyCgroups(); try { @@ -238,7 +238,7 @@ public void testMountController() throws IOException { public void testCGroupPaths() throws IOException { //As per junit behavior, we expect a new mock object to be available //in this test. - Mockito.verifyNoInteractions(privilegedOperationExecutorMock); + verifyZeroInteractions(privilegedOperationExecutorMock); CGroupsHandler cGroupsHandler = null; File mtab = createEmptyCgroups(); @@ -281,7 +281,7 @@ public void testCGroupPaths() throws IOException { public void testCGroupOperations() throws IOException { //As per junit behavior, we expect a new mock object to be available //in this test. - Mockito.verifyNoInteractions(privilegedOperationExecutorMock); + verifyZeroInteractions(privilegedOperationExecutorMock); CGroupsHandler cGroupsHandler = null; File mtab = createEmptyCgroups(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/TestGpuResourceAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/TestGpuResourceAllocator.java index 427b7dce656b9..ba8a9309d02e9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/TestGpuResourceAllocator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/TestGpuResourceAllocator.java @@ -27,6 +27,7 @@ import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyZeroInteractions; import static org.mockito.Mockito.when; import java.io.IOException; @@ -56,7 +57,6 @@ import org.mockito.ArgumentMatcher; import org.mockito.Captor; import org.mockito.Mock; -import org.mockito.Mockito; import org.mockito.MockitoAnnotations; /** @@ -210,7 +210,7 @@ private void assertAllocatedGpus(int gpus, int deniedGpus, private void assertNoAllocation(GpuAllocation allocation) { assertEquals(1, allocation.getDeniedGPUs().size()); assertEquals(0, allocation.getAllowedGPUs().size()); - Mockito.verifyNoInteractions(nmStateStore); + verifyZeroInteractions(nmStateStore); } private void assertAssignmentInStateStore(GpuDevice expectedGpu, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java index 6d033f97e1891..ea7c213809330 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java @@ -1272,7 +1272,7 @@ public void testCGroupParent() throws ContainerExecutionException { command); //no --cgroup-parent should be added here - Mockito.verifyNoInteractions(command); + Mockito.verifyZeroInteractions(command); String resourceOptionsCpu = "/sys/fs/cgroup/cpu/" + hierarchy + containerIdStr; @@ -1296,7 +1296,7 @@ public void testCGroupParent() throws ContainerExecutionException { command); //no --cgroup-parent should be added in either case - Mockito.verifyNoInteractions(command); + Mockito.verifyZeroInteractions(command); } @Test diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/com/nec/TestNECVEPlugin.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/com/nec/TestNECVEPlugin.java index a9b01bca6fac7..86ef9058f26fb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/com/nec/TestNECVEPlugin.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/com/nec/TestNECVEPlugin.java @@ -25,7 +25,7 @@ import static org.mockito.Mockito.when; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.verifyZeroInteractions; import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.ArgumentMatchers.anyString; @@ -394,7 +394,7 @@ public void testFindDevicesWithUdev() assertEquals("No. of devices", 1, devices.size()); Device device = devices.iterator().next(); assertSame("Device", device, testDevice); - verifyNoInteractions(mockCommandExecutor); + verifyZeroInteractions(mockCommandExecutor); verify(mockEnvProvider).apply(eq("NEC_USE_UDEV")); verifyNoMoreInteractions(mockEnvProvider); } @@ -442,6 +442,6 @@ private Device getTestDevice(int id) { private void verifyBinaryPathSet(Path expectedPath) { assertEquals("Binary path", expectedPath.toString(), plugin.getBinaryPath()); - verifyNoInteractions(udevUtil); + verifyZeroInteractions(udevUtil); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigArgumentHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigArgumentHandler.java index 33cc3f0dba266..cb8cc587f68f7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigArgumentHandler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigArgumentHandler.java @@ -23,7 +23,7 @@ import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.verifyZeroInteractions; import java.io.File; import java.io.IOException; @@ -666,7 +666,7 @@ public void testValidationSkippedWhenCmdLineSwitchIsDefined() FSConfigConverterTestCommons.FS_ALLOC_FILE, "-s"); argumentHandler.parseAndConvert(args); - verifyNoInteractions(mockValidator); + verifyZeroInteractions(mockValidator); } @Test @@ -681,7 +681,7 @@ public void testValidationSkippedWhenOutputIsConsole() throws Exception { FSConfigConverterTestCommons.FS_ALLOC_FILE, "-s", "-p"); argumentHandler.parseAndConvert(args); - verifyNoInteractions(mockValidator); + verifyZeroInteractions(mockValidator); } @Test diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java index 530c6ddc55301..55c43666cdb5b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSConfigToCSConfigConverter.java @@ -36,7 +36,7 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.verifyZeroInteractions; import java.io.ByteArrayInputStream; import java.io.File; @@ -673,7 +673,7 @@ public void testPlacementRulesConversionDisabled() throws Exception { converter.setPlacementConverter(placementConverter); converter.convert(params); - verifyNoInteractions(placementConverter); + verifyZeroInteractions(placementConverter); } @Test diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestQueuePlacementConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestQueuePlacementConverter.java index 6871444baa03c..6599080aab59c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestQueuePlacementConverter.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestQueuePlacementConverter.java @@ -21,7 +21,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoMoreInteractions; -import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.verifyZeroInteractions; import static org.mockito.Mockito.when; import java.util.List; @@ -85,7 +85,7 @@ public void testConvertUserRule() { MappingRulesDescription description = convert(); assertEquals("Number of rules", 1, description.getRules().size()); verifyRule(description.getRules().get(0), Policy.USER); - verifyNoInteractions(ruleHandler); + verifyZeroInteractions(ruleHandler); } @Test @@ -96,7 +96,7 @@ public void testConvertSpecifiedRule() { MappingRulesDescription description = convert(); assertEquals("Number of rules", 1, description.getRules().size()); verifyRule(description.getRules().get(0), Policy.SPECIFIED); - verifyNoInteractions(ruleHandler); + verifyZeroInteractions(ruleHandler); } @Test @@ -108,7 +108,7 @@ public void testConvertPrimaryGroupRule() { assertEquals("Number of rules", 1, description.getRules().size()); verifyRule(description.getRules().get(0), Policy.PRIMARY_GROUP); - verifyNoInteractions(ruleHandler); + verifyZeroInteractions(ruleHandler); } @Test @@ -120,7 +120,7 @@ public void testConvertSecondaryGroupRule() { assertEquals("Number of rules", 1, description.getRules().size()); verifyRule(description.getRules().get(0), Policy.SECONDARY_GROUP); - verifyNoInteractions(ruleHandler); + verifyZeroInteractions(ruleHandler); } @Test @@ -134,7 +134,7 @@ public void testConvertDefaultRuleWithQueueName() { assertEquals("Number of rules", 1, description.getRules().size()); verifyRule(description.getRules().get(0), Policy.CUSTOM); - verifyNoInteractions(ruleHandler); + verifyZeroInteractions(ruleHandler); } @Test @@ -147,7 +147,7 @@ public void testConvertDefaultRule() { assertEquals("Number of rules", 1, description.getRules().size()); verifyRule(description.getRules().get(0), Policy.DEFAULT_QUEUE); - verifyNoInteractions(ruleHandler); + verifyZeroInteractions(ruleHandler); } @Test(expected = IllegalArgumentException.class) @@ -168,7 +168,7 @@ public void testConvertRejectRule() { assertEquals("Number of rules", 1, description.getRules().size()); verifyRule(description.getRules().get(0), Policy.REJECT); - verifyNoInteractions(ruleHandler); + verifyZeroInteractions(ruleHandler); } @Test @@ -182,7 +182,7 @@ public void testConvertNestedPrimaryGroupRule() { assertEquals("Number of rules", 1, description.getRules().size()); verifyRule(description.getRules().get(0), Policy.PRIMARY_GROUP_USER); - verifyNoInteractions(ruleHandler); + verifyZeroInteractions(ruleHandler); } @Test @@ -197,7 +197,7 @@ public void testConvertNestedSecondaryGroupRule() { assertEquals("Number of rules", 1, description.getRules().size()); verifyRule(description.getRules().get(0), Policy.SECONDARY_GROUP_USER); - verifyNoInteractions(ruleHandler); + verifyZeroInteractions(ruleHandler); } @Test @@ -215,7 +215,7 @@ public void testConvertNestedDefaultRule() { Rule rule = description.getRules().get(0); verifyRule(description.getRules().get(0), Policy.USER); assertEquals("Parent path", "root.abc", rule.getParentQueue()); - verifyNoInteractions(ruleHandler); + verifyZeroInteractions(ruleHandler); } @Test(expected = IllegalArgumentException.class) @@ -245,7 +245,7 @@ public void testConvertMultiplePlacementRules() { verifyRule(description.getRules().get(0), Policy.USER); verifyRule(description.getRules().get(1), Policy.PRIMARY_GROUP); verifyRule(description.getRules().get(2), Policy.SECONDARY_GROUP); - verifyNoInteractions(ruleHandler); + verifyZeroInteractions(ruleHandler); } @Test @@ -363,7 +363,7 @@ private void testConvertNestedRuleCreateFlagInWeightMode( any(Policy.class)); verifyNoMoreInteractions(ruleHandler); } else { - verifyNoInteractions(ruleHandler); + verifyZeroInteractions(ruleHandler); } } diff --git a/hadoop-yarn-project/pom.xml b/hadoop-yarn-project/pom.xml index 78f09a3971069..241e3bc237a0e 100644 --- a/hadoop-yarn-project/pom.xml +++ b/hadoop-yarn-project/pom.xml @@ -90,11 +90,6 @@ hadoop-yarn-applications-catalog-webapp war - - org.mockito - mockito-core - test - From 1472eb86bdd9e106ee3e293b33bbd88cc1fc9497 Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Wed, 23 Aug 2023 02:07:00 -0700 Subject: [PATCH 13/18] POM --- hadoop-tools/hadoop-azure/pom.xml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/hadoop-tools/hadoop-azure/pom.xml b/hadoop-tools/hadoop-azure/pom.xml index 7dcd4bf1e6a2d..67b22173e0537 100644 --- a/hadoop-tools/hadoop-azure/pom.xml +++ b/hadoop-tools/hadoop-azure/pom.xml @@ -321,13 +321,21 @@ org.mockito mockito-core + 4.11.0 test org.mockito mockito-inline + 4.11.0 test + + + org.mockito + mockito-core + + From 462a3b6a0c7aadefe39960218092651bad10c979 Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Tue, 29 Aug 2023 22:55:47 -0700 Subject: [PATCH 14/18] Checkstyle fix --- .../apache/hadoop/fs/azurebfs/ITestAbfsMsiTokenProvider.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsMsiTokenProvider.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsMsiTokenProvider.java index 3435e65f0fd2f..d4f4c4d9ce5ae 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsMsiTokenProvider.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsMsiTokenProvider.java @@ -22,7 +22,6 @@ import java.lang.reflect.Field; import java.util.Date; -import org.junit.Assume; import org.junit.Test; import org.mockito.MockedStatic; import org.mockito.Mockito; @@ -105,7 +104,6 @@ private String getTrimmedPasswordString(AbfsConfiguration conf, String key, */ @Test public void testRetryForThrottling() throws Exception { - final int HTTP_TOO_MANY_REQUESTS = 429; AbfsConfiguration conf = getConfiguration(); // Exception to be thrown with throttling error code 429. From 78329dece95a9e2d53d4f523a95c3325b09a2d6d Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Thu, 31 Aug 2023 05:36:03 -0700 Subject: [PATCH 15/18] PR comments --- hadoop-tools/hadoop-azure/pom.xml | 8 -------- .../azurebfs/oauth2/AzureADAuthenticator.java | 6 ++++++ .../services/ExponentialRetryPolicy.java | 2 +- .../azurebfs/ITestAbfsMsiTokenProvider.java | 19 ++++++++----------- 4 files changed, 15 insertions(+), 20 deletions(-) diff --git a/hadoop-tools/hadoop-azure/pom.xml b/hadoop-tools/hadoop-azure/pom.xml index 67b22173e0537..7dcd4bf1e6a2d 100644 --- a/hadoop-tools/hadoop-azure/pom.xml +++ b/hadoop-tools/hadoop-azure/pom.xml @@ -321,21 +321,13 @@ org.mockito mockito-core - 4.11.0 test org.mockito mockito-inline - 4.11.0 test - - - org.mockito - mockito-core - - diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/AzureADAuthenticator.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/AzureADAuthenticator.java index 7f0e9a1c8effc..782489416b89e 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/AzureADAuthenticator.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/oauth2/AzureADAuthenticator.java @@ -29,6 +29,7 @@ import java.util.Hashtable; import java.util.Map; +import org.apache.hadoop.classification.VisibleForTesting; import org.apache.hadoop.util.Preconditions; import com.fasterxml.jackson.core.JsonFactory; @@ -70,6 +71,11 @@ public static void init(AbfsConfiguration abfsConfiguration) { tokenFetchRetryPolicy = abfsConfiguration.getOauthTokenFetchRetryPolicy(); } + @VisibleForTesting + public static void setTokenFetchRetryPolicy(ExponentialRetryPolicy retryPolicy) { + tokenFetchRetryPolicy = retryPolicy; + } + /** * gets Azure Active Directory token using the user ID and password of * a service principal (that is, Web App in Azure Active Directory). diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ExponentialRetryPolicy.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ExponentialRetryPolicy.java index d4a9e704a2219..82a33ef8f7096 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ExponentialRetryPolicy.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/ExponentialRetryPolicy.java @@ -163,7 +163,7 @@ public long getRetryInterval(final int retryCount) { } @VisibleForTesting - int getRetryCount() { + public int getRetryCount() { return this.retryCount; } diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsMsiTokenProvider.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsMsiTokenProvider.java index d4f4c4d9ce5ae..5fc4d67b450a7 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsMsiTokenProvider.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsMsiTokenProvider.java @@ -22,6 +22,7 @@ import java.lang.reflect.Field; import java.util.Date; +import org.assertj.core.api.Assertions; import org.junit.Test; import org.mockito.MockedStatic; import org.mockito.Mockito; @@ -128,14 +129,8 @@ public void testRetryForThrottling() throws Exception { Mockito.anyString(), Mockito.any(), Mockito.anyString(), Mockito.anyBoolean())).thenThrow(httpException); - // Mock the tokenFetchRetryPolicy to verify retries. - ExponentialRetryPolicy exponentialRetryPolicy = Mockito.spy( - conf.getOauthTokenFetchRetryPolicy()); - Field tokenFetchRetryPolicy = AzureADAuthenticator.class.getDeclaredField( - "tokenFetchRetryPolicy"); - tokenFetchRetryPolicy.setAccessible(true); - tokenFetchRetryPolicy.set(ExponentialRetryPolicy.class, - exponentialRetryPolicy); + ExponentialRetryPolicy exponentialRetryPolicy = conf.getOauthTokenFetchRetryPolicy(); + AzureADAuthenticator.setTokenFetchRetryPolicy(exponentialRetryPolicy); AccessTokenProvider tokenProvider = new MsiTokenProvider(authEndpoint, tenantGuid, clientId, authority); @@ -145,9 +140,11 @@ public void testRetryForThrottling() throws Exception { // If the status code doesn't qualify for retry shouldRetry returns false and the loop ends. // It being called multiple times verifies that the retry was done for the throttling status code 429. - Mockito.verify(exponentialRetryPolicy, - times(DEFAULT_AZURE_OAUTH_TOKEN_FETCH_RETRY_MAX_ATTEMPTS + 1)) - .shouldRetry(Mockito.anyInt(), Mockito.anyInt()); + int actualRetries = exponentialRetryPolicy.getRetryCount(); + Assertions.assertThat(actualRetries) + .describedAs("Number of retries should be equal to " + + "max attempts for token fetch.") + .isEqualTo(DEFAULT_AZURE_OAUTH_TOKEN_FETCH_RETRY_MAX_ATTEMPTS); } } } From 240965dd76c3590b59143f633459b65cd675cbc2 Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Tue, 14 Nov 2023 22:14:28 -0800 Subject: [PATCH 16/18] Correct default value for backoff interval --- .../hadoop/fs/azurebfs/constants/FileSystemConfigurations.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java index 32f9966e30ae9..03b4d23e4159d 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java @@ -48,7 +48,7 @@ public final class FileSystemConfigurations { public static final int DEFAULT_AZURE_OAUTH_TOKEN_FETCH_RETRY_MAX_ATTEMPTS = 5; public static final int DEFAULT_AZURE_OAUTH_TOKEN_FETCH_RETRY_MIN_BACKOFF_INTERVAL = 0; public static final int DEFAULT_AZURE_OAUTH_TOKEN_FETCH_RETRY_MAX_BACKOFF_INTERVAL = SIXTY_SECONDS; - public static final int DEFAULT_AZURE_OAUTH_TOKEN_FETCH_RETRY_DELTA_BACKOFF = 2; + public static final int DEFAULT_AZURE_OAUTH_TOKEN_FETCH_RETRY_DELTA_BACKOFF = 2 * 1000; public static final int ONE_KB = 1024; public static final int ONE_MB = ONE_KB * ONE_KB; From 588ffb98e286465b31aa7320df7504568825b74d Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Thu, 23 Oct 2025 09:44:56 -0700 Subject: [PATCH 17/18] Checkstyle and mockito fix --- hadoop-tools/hadoop-azure/pom.xml | 7 ------- .../hadoop/fs/azurebfs/ITestAbfsMsiTokenProvider.java | 3 ++- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/hadoop-tools/hadoop-azure/pom.xml b/hadoop-tools/hadoop-azure/pom.xml index 162a2afafcad4..5ce3a58441342 100644 --- a/hadoop-tools/hadoop-azure/pom.xml +++ b/hadoop-tools/hadoop-azure/pom.xml @@ -318,13 +318,6 @@ mockito-core test - - - org.mockito - mockito-inline - test - - org.apache.hadoop hadoop-minikdc diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsMsiTokenProvider.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsMsiTokenProvider.java index afb29db50fcd0..3f628ddac11d4 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsMsiTokenProvider.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAbfsMsiTokenProvider.java @@ -20,6 +20,7 @@ import java.io.IOException; import java.util.Date; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import org.assertj.core.api.Assertions; @@ -134,7 +135,7 @@ public AzureADToken getToken() throws IOException { // Return a valid fake token AzureADToken token = new AzureADToken(); token.setAccessToken("fake-token"); - token.setExpiry(new Date(System.currentTimeMillis() + 3600_000)); + token.setExpiry(new Date(System.currentTimeMillis() + TimeUnit.HOURS.toMillis(1))); return token; } } From fd260666ade501f736fe7af3476e96f6231f2f0b Mon Sep 17 00:00:00 2001 From: Anmol Asrani Date: Thu, 23 Oct 2025 09:47:03 -0700 Subject: [PATCH 18/18] fix config value --- .../hadoop/fs/azurebfs/constants/FileSystemConfigurations.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java index e749eee562c09..52dfc2360041d 100644 --- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java +++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/constants/FileSystemConfigurations.java @@ -68,7 +68,7 @@ public final class FileSystemConfigurations { public static final int DEFAULT_AZURE_OAUTH_TOKEN_FETCH_RETRY_MAX_ATTEMPTS = 5; public static final int DEFAULT_AZURE_OAUTH_TOKEN_FETCH_RETRY_MIN_BACKOFF_INTERVAL = 0; public static final int DEFAULT_AZURE_OAUTH_TOKEN_FETCH_RETRY_MAX_BACKOFF_INTERVAL = SIXTY_SECONDS; - public static final int DEFAULT_AZURE_OAUTH_TOKEN_FETCH_RETRY_DELTA_BACKOFF = 2 * 1000; + public static final int DEFAULT_AZURE_OAUTH_TOKEN_FETCH_RETRY_DELTA_BACKOFF = 2_000; public static final int ONE_KB = 1024; public static final int ONE_MB = ONE_KB * ONE_KB;