Integrate api keys with the authentication service (#35555)

This commit integrates usage of the api key service into the
authentication service for verification of api keys. A bug was fixed in
the validation of api keys where the structure of the document was not
being used properly. Additionally, unit tests have been added for
authentication with api keys.

Author: jaymode

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java

@@ -459,7 +459,7 @@ Collection<Object> createComponents(Client client, ThreadPool threadPool, Cluste
         final AuthenticationFailureHandler failureHandler = createAuthenticationFailureHandler(realms);
 
         authcService.set(new AuthenticationService(settings, realms, auditTrailService, failureHandler, threadPool,
-                anonymousUser, tokenService));
+                anonymousUser, tokenService, apiKeyService));
         components.add(authcService.get());
 
         final NativePrivilegeStore privilegeStore = new NativePrivilegeStore(settings, client, securityIndex.get());

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java

@@ -178,11 +178,13 @@ void authenticateWithApiKeyIfPresent(ThreadContext ctx, ActionListener<Authentic
                         }
                     } else {
                         credentials.close();
-                        listener.onResponse(AuthenticationResult.unsuccessful("unable to authenticate", null));
+                        listener.onResponse(
+                            AuthenticationResult.unsuccessful("unable to find apikey with id " + credentials.getId(), null));
                     }
                 }, e -> {
                     credentials.close();
-                    listener.onResponse(AuthenticationResult.unsuccessful("apikey auth encountered a failure", e));
+                    listener.onResponse(AuthenticationResult.unsuccessful("apikey authentication for id " + credentials.getId() +
+                        " encountered a failure", e));
                 }), client::get);
             } else {
                 listener.onResponse(AuthenticationResult.notHandled());
@@ -209,8 +211,9 @@ static void validateApiKeyCredentials(Map<String, Object> source, ApiKeyCredenti
         if (verified) {
             final Long expirationEpochMilli = (Long) source.get("expiration_time");
             if (expirationEpochMilli == null || Instant.ofEpochMilli(expirationEpochMilli).isAfter(clock.instant())) {
-                final String principal = Objects.requireNonNull((String) source.get("principal"));
-                final Map<String, Object> metadata = (Map<String, Object>) source.get("metadata");
+                final Map<String, Object> creator = Objects.requireNonNull((Map<String, Object>) source.get("creator"));
+                final String principal = Objects.requireNonNull((String) creator.get("principal"));
+                final Map<String, Object> metadata = (Map<String, Object>) creator.get("metadata");
                 final List<Map<String, Object>> roleDescriptors = (List<Map<String, Object>>) source.get("role_descriptors");
                 final String[] roleNames = roleDescriptors.stream()
                     .map(rdSource -> (String) rdSource.get("name"))
@@ -219,7 +222,7 @@ static void validateApiKeyCredentials(Map<String, Object> source, ApiKeyCredenti
                 final User apiKeyUser = new User(principal, roleNames, null, null, metadata, true);
                 listener.onResponse(AuthenticationResult.success(apiKeyUser));
             } else {
-                listener.onResponse(AuthenticationResult.unsuccessful("api key is expired", null));
+                listener.onResponse(AuthenticationResult.terminate("api key is expired", null));
             }
         } else {
             listener.onResponse(AuthenticationResult.unsuccessful("invalid credentials", null));

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/AuthenticationService.java

@@ -61,12 +61,13 @@ public class AuthenticationService {
     private final String nodeName;
     private final AnonymousUser anonymousUser;
     private final TokenService tokenService;
+    private final ApiKeyService apiKeyService;
     private final boolean runAsEnabled;
     private final boolean isAnonymousUserEnabled;
 
     public AuthenticationService(Settings settings, Realms realms, AuditTrailService auditTrail,
                                  AuthenticationFailureHandler failureHandler, ThreadPool threadPool,
-                                 AnonymousUser anonymousUser, TokenService tokenService) {
+                                 AnonymousUser anonymousUser, TokenService tokenService, ApiKeyService apiKeyService) {
         this.nodeName = Node.NODE_NAME_SETTING.get(settings);
         this.realms = realms;
         this.auditTrail = auditTrail;
@@ -76,6 +77,7 @@ public AuthenticationService(Settings settings, Realms realms, AuditTrailService
         this.runAsEnabled = AuthenticationServiceField.RUN_AS_ENABLED.get(settings);
         this.isAnonymousUserEnabled = AnonymousUser.isAnonymousEnabled(settings);
         this.tokenService = tokenService;
+        this.apiKeyService = apiKeyService;
     }
 
     /**
@@ -181,7 +183,7 @@ private void authenticateAsync() {
                         if (userToken != null) {
                             writeAuthToContext(userToken.getAuthentication());
                         } else {
-                            extractToken(this::consumeToken);
+                            checkForApiKey();
                         }
                     }, e -> {
                         if (e instanceof ElasticsearchSecurityException &&
@@ -196,6 +198,31 @@ private void authenticateAsync() {
             });
         }
 
+        private void checkForApiKey() {
+            apiKeyService.authenticateWithApiKeyIfPresent(threadContext, ActionListener.wrap(authResult -> {
+                    if (authResult.isAuthenticated()) {
+                        final User user = authResult.getUser();
+                        authenticatedBy = new RealmRef("_es_api_key", "_es_api_key", nodeName);
+                        writeAuthToContext(new Authentication(user, authenticatedBy, null));
+                    } else if (authResult.getStatus() == AuthenticationResult.Status.TERMINATE) {
+                        Exception e = (authResult.getException() != null) ? authResult.getException()
+                            : Exceptions.authenticationError(authResult.getMessage());
+                        listener.onFailure(e);
+                    } else {
+                        if (authResult.getMessage() != null) {
+                            if (authResult.getException() != null) {
+                                logger.warn(new ParameterizedMessage("Authentication using apikey failed - {}", authResult.getMessage()),
+                                    authResult.getException());
+                            } else {
+                                logger.warn("Authentication using apikey failed - {}", authResult.getMessage());
+                            }
+                        }
+                        extractToken(this::consumeToken);
+                    }
+                },
+                e -> listener.onFailure(request.exceptionProcessingRequest(e, null))));
+        }
+
         /**
          * Looks to see if the request contains an existing {@link Authentication} and if so, that authentication will be used. The
          * consumer is called if no exception was thrown while trying to read the authentication and may be called with a {@code null}

x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ApiKeyServiceTests.java

@@ -69,10 +69,11 @@ public void testValidateApiKey() throws Exception {
 
         Map<String, Object> sourceMap = new HashMap<>();
         sourceMap.put("api_key_hash", new String(hash));
-        sourceMap.put("principal", "test_user");
-        sourceMap.put("metadata", Collections.emptyMap());
         sourceMap.put("role_descriptors", Collections.singletonList(Collections.singletonMap("name", "a role")));
-
+        Map<String, Object> creatorMap = new HashMap<>();
+        creatorMap.put("principal", "test_user");
+        creatorMap.put("metadata", Collections.emptyMap());
+        sourceMap.put("creator", creatorMap);
 
         ApiKeyService.ApiKeyCredentials creds =
             new ApiKeyService.ApiKeyCredentials(randomAlphaOfLength(12), new SecureString(apiKey.toCharArray()));