Implement lookup of permissions for API keys (#35970)

This change implements a lookup of permissions for API keys when a
request moves to authorization. In order to support this, the
authentication of an API key will attach values as metadata on the
authentication result. The values attached will include the source
of the role descriptors. The authentication service will then copy
this metadata to the authentication object and set the authentication
type to API_KEY. The authorization service will use the authentication
type to make a decision on how the roles should be obtained.

Author: jaymode

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Authentication.java

@@ -18,6 +18,8 @@
 
 import java.io.IOException;
 import java.util.Base64;
+import java.util.Collections;
+import java.util.Map;
 import java.util.Objects;
 
 // TODO(hub-cap) Clean this up after moving User over - This class can re-inherit its field AUTHENTICATION_KEY in AuthenticationField.
@@ -28,16 +30,25 @@ public class Authentication implements ToXContentObject {
     private final RealmRef authenticatedBy;
     private final RealmRef lookedUpBy;
     private final Version version;
+    private final AuthenticationType type;
+    private final Map<String, Object> metadata;
 
     public Authentication(User user, RealmRef authenticatedBy, RealmRef lookedUpBy) {
         this(user, authenticatedBy, lookedUpBy, Version.CURRENT);
     }
 
     public Authentication(User user, RealmRef authenticatedBy, RealmRef lookedUpBy, Version version) {
+        this(user, authenticatedBy, lookedUpBy, version, AuthenticationType.REALM, Collections.emptyMap());
+    }
+
+    public Authentication(User user, RealmRef authenticatedBy, RealmRef lookedUpBy, Version version,
+                          AuthenticationType type, Map<String, Object> metadata) {
         this.user = Objects.requireNonNull(user);
         this.authenticatedBy = Objects.requireNonNull(authenticatedBy);
         this.lookedUpBy = lookedUpBy;
         this.version = version;
+        this.type = type;
+        this.metadata = metadata;
     }
 
     public Authentication(StreamInput in) throws IOException {
@@ -49,6 +60,13 @@ public Authentication(StreamInput in) throws IOException {
             this.lookedUpBy = null;
         }
         this.version = in.getVersion();
+        if (in.getVersion().onOrAfter(Version.V_7_0_0)) { // TODO change to V6_6 after backport
+            type = AuthenticationType.values()[in.readVInt()];
+            metadata = in.readMap();
+        } else {
+            type = AuthenticationType.REALM;
+            metadata = Collections.emptyMap();
+        }
     }
 
     public User getUser() {
@@ -67,8 +85,15 @@ public Version getVersion() {
         return version;
     }
 
-    public static Authentication readFromContext(ThreadContext ctx)
-            throws IOException, IllegalArgumentException {
+    public AuthenticationType getAuthenticationType() {
+        return type;
+    }
+
+    public Map<String, Object> getMetadata() {
+        return metadata;
+    }
+
+    public static Authentication readFromContext(ThreadContext ctx) throws IOException, IllegalArgumentException {
         Authentication authentication = ctx.getTransient(AuthenticationField.AUTHENTICATION_KEY);
         if (authentication != null) {
             assert ctx.getHeader(AuthenticationField.AUTHENTICATION_KEY) != null;
@@ -107,8 +132,7 @@ public static Authentication decode(String header) throws IOException {
      * Writes the authentication to the context. There must not be an existing authentication in the context and if there is an
      * {@link IllegalStateException} will be thrown
      */
-    public void writeToContext(ThreadContext ctx)
-            throws IOException, IllegalArgumentException {
+    public void writeToContext(ThreadContext ctx) throws IOException, IllegalArgumentException {
         ensureContextDoesNotContainAuthentication(ctx);
         String header = encode();
         ctx.putTransient(AuthenticationField.AUTHENTICATION_KEY, this);
@@ -141,28 +165,28 @@ public void writeTo(StreamOutput out) throws IOException {
         } else {
             out.writeBoolean(false);
         }
+        if (out.getVersion().onOrAfter(Version.V_7_0_0)) { // TODO change to V6_6 after backport
+            out.writeVInt(type.ordinal());
+            out.writeMap(metadata);
+        }
     }
 
     @Override
     public boolean equals(Object o) {
         if (this == o) return true;
         if (o == null || getClass() != o.getClass()) return false;
-
         Authentication that = (Authentication) o;
-
-        if (!user.equals(that.user)) return false;
-        if (!authenticatedBy.equals(that.authenticatedBy)) return false;
-        if (lookedUpBy != null ? !lookedUpBy.equals(that.lookedUpBy) : that.lookedUpBy != null) return false;
-        return version.equals(that.version);
+        return user.equals(that.user) &&
+            authenticatedBy.equals(that.authenticatedBy) &&
+            Objects.equals(lookedUpBy, that.lookedUpBy) &&
+            version.equals(that.version) &&
+            type == that.type &&
+            metadata.equals(that.metadata);
     }
 
     @Override
     public int hashCode() {
-        int result = user.hashCode();
-        result = 31 * result + authenticatedBy.hashCode();
-        result = 31 * result + (lookedUpBy != null ? lookedUpBy.hashCode() : 0);
-        result = 31 * result + version.hashCode();
-        return result;
+        return Objects.hash(user, authenticatedBy, lookedUpBy, version, type, metadata);
     }
 
     @Override
@@ -246,5 +270,11 @@ public int hashCode() {
             return result;
         }
     }
+
+    public enum AuthenticationType {
+        REALM,
+        API_KEY,
+        TOKEN
+    }
 }
 

x-pack/plugin/core/src/main/resources/security-index-template.json

@@ -156,7 +156,7 @@
         },
         "role_descriptors" : {
           "type" : "object",
-          "dynamic" : true
+          "enabled": false
         },
         "version" : {
           "type" : "integer"

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java

@@ -479,7 +479,7 @@ Collection<Object> createComponents(Client client, ThreadPool threadPool, Cluste
         // minimal
         getLicenseState().addListener(allRolesStore::invalidateAll);
         final AuthorizationService authzService = new AuthorizationService(settings, allRolesStore, clusterService,
-            auditTrailService, failureHandler, threadPool, anonymousUser);
+            auditTrailService, failureHandler, threadPool, anonymousUser, apiKeyService);
         components.add(nativeRolesStore); // used by roles actions
         components.add(reservedRolesStore); // used by roles actions
         components.add(allRolesStore); // for SecurityFeatureSet and clear roles cache

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportGetUserPrivilegesAction.java

@@ -60,13 +60,14 @@ public TransportGetUserPrivilegesAction(ThreadPool threadPool, TransportService
     protected void doExecute(Task task, GetUserPrivilegesRequest request, ActionListener<GetUserPrivilegesResponse> listener) {
         final String username = request.username();
 
-        final User user = Authentication.getAuthentication(threadPool.getThreadContext()).getUser();
+        final Authentication authentication = Authentication.getAuthentication(threadPool.getThreadContext());
+        final User user = authentication.getUser();
         if (user.principal().equals(username) == false) {
             listener.onFailure(new IllegalArgumentException("users may only list the privileges of their own account"));
             return;
         }
 
-        authorizationService.roles(user, ActionListener.wrap(
+        authorizationService.roles(user, authentication, ActionListener.wrap(
             role -> listener.onResponse(buildResponseObject(role)),
             listener::onFailure));
     }

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportHasPrivilegesAction.java

@@ -68,13 +68,14 @@ public TransportHasPrivilegesAction(ThreadPool threadPool, TransportService tran
     protected void doExecute(Task task, HasPrivilegesRequest request, ActionListener<HasPrivilegesResponse> listener) {
         final String username = request.username();
 
-        final User user = Authentication.getAuthentication(threadPool.getThreadContext()).getUser();
+        final Authentication authentication = Authentication.getAuthentication(threadPool.getThreadContext());
+        final User user = authentication.getUser();
         if (user.principal().equals(username) == false) {
             listener.onFailure(new IllegalArgumentException("users may only check the privileges of their own account"));
             return;
         }
 
-        authorizationService.roles(user, ActionListener.wrap(
+        authorizationService.roles(user, authentication, ActionListener.wrap(
             role -> resolveApplicationPrivileges(request, ActionListener.wrap(
                 applicationPrivilegeLookup -> checkPrivileges(request, role, applicationPrivilegeLookup, listener),
                 listener::onFailure)),

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java

@@ -19,29 +19,41 @@
 import org.elasticsearch.common.CharArrays;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.UUIDs;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.settings.SecureString;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.common.xcontent.DeprecationHandler;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.xpack.core.XPackSettings;
 import org.elasticsearch.xpack.core.security.action.CreateApiKeyRequest;
 import org.elasticsearch.xpack.core.security.action.CreateApiKeyResponse;
 import org.elasticsearch.xpack.core.security.authc.Authentication;
 import org.elasticsearch.xpack.core.security.authc.AuthenticationResult;
 import org.elasticsearch.xpack.core.security.authc.support.Hasher;
+import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
+import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissionsCache;
+import org.elasticsearch.xpack.core.security.authz.permission.Role;
 import org.elasticsearch.xpack.core.security.user.User;
+import org.elasticsearch.xpack.security.authz.store.CompositeRolesStore;
 import org.elasticsearch.xpack.security.support.SecurityIndexManager;
 
 import javax.crypto.SecretKeyFactory;
 import java.io.Closeable;
 import java.io.IOException;
+import java.io.UncheckedIOException;
 import java.security.NoSuchAlgorithmException;
 import java.time.Clock;
 import java.time.Instant;
 import java.util.Arrays;
 import java.util.Base64;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
@@ -55,7 +67,12 @@
 public class ApiKeyService {
 
     private static final Logger logger = LogManager.getLogger(ApiKeyService.class);
+    private static final DeprecationLogger deprecationLogger = new DeprecationLogger(logger);
     private static final String TYPE = "doc";
+    static final String API_KEY_ID_KEY = "_security_api_key_id";
+    static final String API_KEY_ROLE_DESCRIPTORS_KEY = "_security_api_key_role_descriptors";
+    static final String API_KEY_ROLE_KEY = "_security_api_key_role";
+
     public static final Setting<String> PASSWORD_HASHING_ALGORITHM = new Setting<>(
         "xpack.security.authc.api_key_hashing.algorithm", "pbkdf2", Function.identity(), (v, s) -> {
         if (Hasher.getAvailableAlgoStoredHash().contains(v.toLowerCase(Locale.ROOT)) == false) {
@@ -126,8 +143,12 @@ public void createApiKey(Authentication authentication, CreateApiKeyRequest requ
                     }
                 }
 
-                builder.array("role_descriptors", request.getRoleDescriptors())
-                    .field("name", request.getName())
+                builder.startObject("role_descriptors");
+                for (RoleDescriptor descriptor : request.getRoleDescriptors()) {
+                    builder.field(descriptor.getName(), (contentBuilder, params) -> descriptor.toXContent(contentBuilder, params, true));
+                }
+                builder.endObject();
+                builder.field("name", request.getName())
                     .field("version", version.id)
                     .startObject("creator")
                     .field("principal", authentication.getUser().principal())
@@ -174,7 +195,8 @@ void authenticateWithApiKeyIfPresent(ThreadContext ctx, ActionListener<Authentic
                 executeAsyncWithOrigin(ctx, SECURITY_ORIGIN, getRequest, ActionListener.<GetResponse>wrap(response -> {
                     if (response.isExists()) {
                         try (ApiKeyCredentials ignore = credentials) {
-                            validateApiKeyCredentials(response.getSource(), credentials, clock, listener);
+                            final Map<String, Object> source = response.getSource();
+                            validateApiKeyCredentials(source, credentials, clock, listener);
                         }
                     } else {
                         credentials.close();
@@ -194,6 +216,56 @@ void authenticateWithApiKeyIfPresent(ThreadContext ctx, ActionListener<Authentic
         }
     }
 
+    /**
+     * The current request has been authenticated by an API key and this method enables the
+     * retrieval of role descriptors that are associated with the api key and triggers the building
+     * of the {@link Role} to authorize the request.
+     */
+    public void getRoleForApiKey(Authentication authentication, ThreadContext threadContext, CompositeRolesStore rolesStore,
+                                 FieldPermissionsCache fieldPermissionsCache, ActionListener<Role> listener) {
+        if (authentication.getAuthenticationType() != Authentication.AuthenticationType.API_KEY) {
+            throw new IllegalStateException("authentication type must be api key but is " + authentication.getAuthenticationType());
+        }
+
+        final Map<String, Object> metadata = authentication.getMetadata();
+        final String apiKeyId = (String) metadata.get(API_KEY_ID_KEY);
+        final String contextKeyId = threadContext.getTransient(API_KEY_ID_KEY);
+        if (apiKeyId.equals(contextKeyId)) {
+            final Role preBuiltRole = threadContext.getTransient(API_KEY_ROLE_KEY);
+            if (preBuiltRole != null) {
+                listener.onResponse(preBuiltRole);
+                return;
+            }
+        } else if (contextKeyId != null) {
+            throw new IllegalStateException("authentication api key id [" + apiKeyId + "] does not match context value [" +
+                contextKeyId + "]");
+        }
+
+        final Map<String, Object> roleDescriptors = (Map<String, Object>) metadata.get(API_KEY_ROLE_DESCRIPTORS_KEY);
+        final List<RoleDescriptor> roleDescriptorList = roleDescriptors.entrySet().stream()
+            .map(entry -> {
+                final String name = entry.getKey();
+                final Map<String, Object> rdMap = (Map<String, Object>) entry.getValue();
+                try (XContentBuilder builder = XContentBuilder.builder(XContentType.JSON.xContent())) {
+                    builder.map(rdMap);
+                    try (XContentParser parser = XContentType.JSON.xContent().createParser(NamedXContentRegistry.EMPTY,
+                        new ApiKeyLoggingDeprecationHandler(deprecationLogger, apiKeyId),
+                        BytesReference.bytes(builder).streamInput())) {
+                        return RoleDescriptor.parse(name, parser, false);
+                    }
+                } catch (IOException e) {
+                    throw new UncheckedIOException(e);
+                }
+            }).collect(Collectors.toList());
+
+        rolesStore.buildRoleFromDescriptors(roleDescriptorList, fieldPermissionsCache, ActionListener.wrap(role -> {
+            threadContext.putTransient(API_KEY_ID_KEY, apiKeyId);
+            threadContext.putTransient(API_KEY_ROLE_KEY, role);
+            listener.onResponse(role);
+        }, listener::onFailure));
+
+    }
+
     /**
      * Validates the ApiKey using the source map
      * @param source the source map from a get of the ApiKey document
@@ -214,13 +286,13 @@ static void validateApiKeyCredentials(Map<String, Object> source, ApiKeyCredenti
                 final Map<String, Object> creator = Objects.requireNonNull((Map<String, Object>) source.get("creator"));
                 final String principal = Objects.requireNonNull((String) creator.get("principal"));
                 final Map<String, Object> metadata = (Map<String, Object>) creator.get("metadata");
-                final List<Map<String, Object>> roleDescriptors = (List<Map<String, Object>>) source.get("role_descriptors");
-                final String[] roleNames = roleDescriptors.stream()
-                    .map(rdSource -> (String) rdSource.get("name"))
-                    .collect(Collectors.toList())
-                    .toArray(Strings.EMPTY_ARRAY);
+                final Map<String, Object> roleDescriptors = (Map<String, Object>) source.get("role_descriptors");
+                final String[] roleNames = roleDescriptors.keySet().toArray(Strings.EMPTY_ARRAY);
                 final User apiKeyUser = new User(principal, roleNames, null, null, metadata, true);
-                listener.onResponse(AuthenticationResult.success(apiKeyUser));
+                final Map<String, Object> authResultMetadata = new HashMap<>();
+                authResultMetadata.put(API_KEY_ROLE_DESCRIPTORS_KEY, roleDescriptors);
+                authResultMetadata.put(API_KEY_ID_KEY, credentials.getId());
+                listener.onResponse(AuthenticationResult.success(apiKeyUser, authResultMetadata));
             } else {
                 listener.onResponse(AuthenticationResult.terminate("api key is expired", null));
             }
@@ -310,4 +382,27 @@ public void close() {
             key.close();
         }
     }
+
+    private static class ApiKeyLoggingDeprecationHandler implements DeprecationHandler {
+
+        private final DeprecationLogger deprecationLogger;
+        private final String apiKeyId;
+
+        private ApiKeyLoggingDeprecationHandler(DeprecationLogger logger, String apiKeyId) {
+            this.deprecationLogger = logger;
+            this.apiKeyId = apiKeyId;
+        }
+
+        @Override
+        public void usedDeprecatedName(String usedName, String modernName) {
+            deprecationLogger.deprecated("Deprecated field [{}] used in api key [{}], expected [{}] instead",
+                usedName, apiKeyId, modernName);
+        }
+
+        @Override
+        public void usedDeprecatedField(String usedName, String replacedWith) {
+            deprecationLogger.deprecated("Deprecated field [{}] used in api key [{}], replaced by [{}]",
+                usedName, apiKeyId, replacedWith);
+        }
+    }
 }

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/AuthenticationService.java

@@ -10,6 +10,7 @@
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.apache.logging.log4j.util.Supplier;
 import org.elasticsearch.ElasticsearchSecurityException;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ContextPreservingActionListener;
 import org.elasticsearch.common.Nullable;
@@ -205,7 +206,8 @@ private void checkForApiKey() {
                     if (authResult.isAuthenticated()) {
                         final User user = authResult.getUser();
                         authenticatedBy = new RealmRef("_es_api_key", "_es_api_key", nodeName);
-                        writeAuthToContext(new Authentication(user, authenticatedBy, null));
+                        writeAuthToContext(new Authentication(user, authenticatedBy, null, Version.CURRENT,
+                            Authentication.AuthenticationType.API_KEY, authResult.getMetadata()));
                     } else if (authResult.getStatus() == AuthenticationResult.Status.TERMINATE) {
                         Exception e = (authResult.getException() != null) ? authResult.getException()
                             : Exceptions.authenticationError(authResult.getMessage());