Move role validation to REST layer instead of transport

libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContent.java

@@ -88,4 +88,8 @@ XContentParser createParserForCompatibility(NamedXContentRegistry xContentRegist
                                 DeprecationHandler deprecationHandler, byte[] data, int offset, int length,
                                                 RestApiVersion restApiVersion) throws IOException;
 
+    XContentParser createParserForCompatibility(NamedXContentRegistry xContentRegistry,
+                                                DeprecationHandler deprecationHandler, String content,
+                                                RestApiVersion restApiVersion) throws IOException;
+
 }

libs/x-content/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContent.java

@@ -112,4 +112,9 @@ public XContentParser createParserForCompatibility(NamedXContentRegistry xConten
         return new CborXContentParser(xContentRegistry, deprecationHandler, cborFactory.createParser(data, offset, length), restApiVersion);
     }
 
+    @Override
+    public XContentParser createParserForCompatibility(NamedXContentRegistry xContentRegistry, DeprecationHandler deprecationHandler,
+                                                       String content, RestApiVersion restApiVersion) throws IOException {
+        return new CborXContentParser(xContentRegistry, deprecationHandler, cborFactory.createParser(content), restApiVersion);
+    }
 }

libs/x-content/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContent.java

@@ -112,4 +112,9 @@ public XContentParser createParserForCompatibility(NamedXContentRegistry xConten
         return new JsonXContentParser(xContentRegistry, deprecationHandler, jsonFactory.createParser(data, offset, length), restApiVersion);
     }
 
+    @Override
+    public XContentParser createParserForCompatibility(NamedXContentRegistry xContentRegistry, DeprecationHandler deprecationHandler,
+                                                       String content, RestApiVersion restApiVersion) throws IOException {
+        return new JsonXContentParser(xContentRegistry, deprecationHandler, jsonFactory.createParser(content), restApiVersion);
+    }
 }

libs/x-content/src/main/java/org/elasticsearch/common/xcontent/smile/SmileXContent.java

@@ -113,4 +113,11 @@ public XContentParser createParserForCompatibility(NamedXContentRegistry xConten
         return new SmileXContentParser(xContentRegistry, deprecationHandler, smileFactory.createParser(data, offset, length),
             restApiVersion);
     }
+
+    @Override
+    public XContentParser createParserForCompatibility(NamedXContentRegistry xContentRegistry, DeprecationHandler deprecationHandler,
+                                                       String content, RestApiVersion restApiVersion) throws IOException {
+        return new SmileXContentParser(xContentRegistry, deprecationHandler, smileFactory.createParser(content),
+            restApiVersion);
+    }
 }

libs/x-content/src/main/java/org/elasticsearch/common/xcontent/yaml/YamlXContent.java

@@ -107,5 +107,10 @@ public XContentParser createParserForCompatibility(NamedXContentRegistry xConten
             restApiVersion);
     }
 
-
+    @Override
+    public XContentParser createParserForCompatibility(NamedXContentRegistry xContentRegistry, DeprecationHandler deprecationHandler,
+                                                       String content, RestApiVersion restApiVersion) throws IOException {
+        return new YamlXContentParser(xContentRegistry, deprecationHandler, yamlFactory.createParser(content),
+            restApiVersion);
+    }
 }

rest-api-spec/build.gradle

@@ -106,8 +106,6 @@ tasks.named("yamlRestCompatTest").configure {
     'mtermvectors/11_basic_with_types/Basic tests for multi termvector get',
     'mtermvectors/21_deprecated_with_types/Deprecated camel case and _ parameters should fail in Term Vectors query',
     'mtermvectors/30_mix_typeless_typeful/mtermvectors without types on an index that has types',
-    'search/150_rewrite_on_coordinator/Ensure that we fetch the document only once', //terms_lookup
-    'search/171_terms_query_with_types/Terms Query with No.of terms exceeding index.max_terms_count should FAIL', //bulk
     'search/260_parameter_validation/test size=-1 is deprecated', //size=-1 change
     'search/310_match_bool_prefix/multi_match multiple fields with cutoff_frequency throws exception', //cutoff_frequency
     'search/340_type_query/type query', // type_query - probably should behave like match_all

server/src/main/java/org/elasticsearch/indices/TermsLookup.java

@@ -17,13 +17,15 @@
 import org.elasticsearch.common.xcontent.ToXContentFragment;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.query.TermsQueryBuilder;
 
 import java.io.IOException;
 import java.util.Objects;
 
 import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
+import static org.elasticsearch.core.RestApiVersion.equalTo;
 
 /**
  * Encapsulates the parameters needed to fetch terms.
@@ -107,6 +109,8 @@ public TermsLookup routing(String routing) {
         PARSER.declareString(constructorArg(), new ParseField("id"));
         PARSER.declareString(constructorArg(), new ParseField("path"));
         PARSER.declareString(TermsLookup::routing, new ParseField("routing"));
+        PARSER.declareString((termLookup,type)-> {}, new ParseField("type")
+            .forRestApiVersion(equalTo(RestApiVersion.V_7)));
     }
 
     public static TermsLookup parseTermsLookup(XContentParser parser) throws IOException {

test/framework/src/main/java/org/elasticsearch/test/rest/RestActionTestCase.java

@@ -64,6 +64,10 @@ protected RestController controller() {
      */
     protected void dispatchRequest(RestRequest request) {
         FakeRestChannel channel = new FakeRestChannel(request, false, 1);
+        dispatchRequest(request, channel);
+    }
+
+    protected void dispatchRequest(RestRequest request, FakeRestChannel channel) {
         ThreadContext threadContext = verifyingClient.threadPool().getThreadContext();
         try(ThreadContext.StoredContext ignore = threadContext.stashContext()) {
             controller.dispatchRequest(request, channel, threadContext);

x-pack/plugin/build.gradle

@@ -149,7 +149,6 @@ tasks.named("yamlRestCompatTest").configure {
     'ml/set_upgrade_mode/Test setting upgrade_mode to false when it is already false',
     'ml/trained_model_cat_apis/Test cat trained models',
     'roles/11_idx_arrays/Test put role api using as array of index names',
-    'roles/30_prohibited_role_query/Test use prohibited query inside role query',
     'rollup/delete_job/Test basic delete_job',
     'rollup/delete_job/Test delete job twice',
     'rollup/delete_job/Test delete running job',

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/role/PutRoleRequestBuilder.java

@@ -9,9 +9,9 @@
 import org.elasticsearch.action.ActionRequestBuilder;
 import org.elasticsearch.action.support.WriteRequestBuilder;
 import org.elasticsearch.client.ElasticsearchClient;
-import org.elasticsearch.core.Nullable;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.core.Nullable;
 import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
 
 import java.io.IOException;
@@ -38,6 +38,7 @@ public PutRoleRequestBuilder source(String name, BytesReference source, XContent
         // we pass false as last parameter because we want to reject the request if field permissions
         // are given in 2.x syntax
         RoleDescriptor descriptor = RoleDescriptor.parse(name, source, false, xContentType);
+
         assert name.equals(descriptor.getName());
         request.name(name);
         request.cluster(descriptor.getClusterPrivileges());

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/DocumentPermissions.java

@@ -13,6 +13,7 @@
 import org.apache.lucene.search.join.ToChildBlockJoinQuery;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.lucene.search.Queries;
+import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryRewriteContext;
 import org.elasticsearch.index.query.SearchExecutionContext;
@@ -119,7 +120,7 @@ private static void buildRoleQuery(User user, ScriptService scriptService, Shard
         for (BytesReference bytesReference : queries) {
             SearchExecutionContext context = searchExecutionContextProvider.apply(shardId);
             QueryBuilder queryBuilder = DLSRoleQueryValidator.evaluateAndVerifyRoleQuery(bytesReference, scriptService,
-                context.getXContentRegistry(), user);
+                context.getXContentRegistry(), user, RestApiVersion.current());
             if (queryBuilder != null) {
                 failIfQueryUsesClient(queryBuilder, context);
                 Query roleQuery = context.toQuery(queryBuilder).query();

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/support/DLSRoleQueryValidator.java

@@ -17,6 +17,7 @@
 import org.elasticsearch.common.xcontent.XContentParseException;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.index.query.AbstractQueryBuilder;
 import org.elasticsearch.index.query.BoolQueryBuilder;
 import org.elasticsearch.index.query.BoostingQueryBuilder;
@@ -45,23 +46,23 @@ private DLSRoleQueryValidator() {
     /**
      * Validates the query field in the {@link RoleDescriptor.IndicesPrivileges} only if it is not a template query.<br>
      * It parses the query and builds the {@link QueryBuilder}, also checks if the query type is supported in DLS role query.
-     *
-     * @param indicesPrivileges {@link RoleDescriptor.IndicesPrivileges}
+     *  @param indicesPrivileges {@link RoleDescriptor.IndicesPrivileges}
      * @param xContentRegistry  {@link NamedXContentRegistry} for finding named queries
+     * @param restApiVersion
      */
     public static void validateQueryField(RoleDescriptor.IndicesPrivileges[] indicesPrivileges,
-                                          NamedXContentRegistry xContentRegistry) {
+                                          NamedXContentRegistry xContentRegistry, RestApiVersion restApiVersion) {
         if (indicesPrivileges != null) {
             for (int i = 0; i < indicesPrivileges.length; i++) {
                 BytesReference query = indicesPrivileges[i].getQuery();
                 try {
                     if (query != null) {
-                        if (isTemplateQuery(query, xContentRegistry)) {
+                        if (isTemplateQuery(query, xContentRegistry, restApiVersion)) {
                             // skip template query, this requires runtime information like 'User' information.
                             continue;
                         }
 
-                        evaluateAndVerifyRoleQuery(query.utf8ToString(), xContentRegistry);
+                        evaluateAndVerifyRoleQuery(query.utf8ToString(), xContentRegistry, restApiVersion);
                     }
                 } catch (ParsingException | IllegalArgumentException |  IOException e) {
                     throw new ElasticsearchParseException("failed to parse field 'query' for indices [" +
@@ -72,9 +73,10 @@ public static void validateQueryField(RoleDescriptor.IndicesPrivileges[] indices
         }
     }
 
-    private static boolean isTemplateQuery(BytesReference query, NamedXContentRegistry xContentRegistry) throws IOException {
-        try (XContentParser parser = XContentType.JSON.xContent().createParser(xContentRegistry,
-            LoggingDeprecationHandler.INSTANCE, query.utf8ToString())) {
+    private static boolean isTemplateQuery(BytesReference query, NamedXContentRegistry xContentRegistry, RestApiVersion restApiVersion)
+        throws IOException {
+        try (XContentParser parser = XContentType.JSON.xContent().createParserForCompatibility(xContentRegistry,
+            LoggingDeprecationHandler.INSTANCE, query.streamInput(), restApiVersion)) {
             XContentParser.Token token = parser.nextToken();
             if (token != XContentParser.Token.START_OBJECT) {
                 throw new XContentParseException(parser.getTokenLocation(), "expected [" + XContentParser.Token.START_OBJECT + "] but " +
@@ -103,17 +105,19 @@ private static boolean isTemplateQuery(BytesReference query, NamedXContentRegist
      * @param scriptService    {@link ScriptService} used for evaluation of a template query
      * @param xContentRegistry {@link NamedXContentRegistry} for finding named queries
      * @param user             {@link User} used when evaluation a template query
+     * @param restApiVersion
      * @return {@link QueryBuilder} if the query is valid and allowed, in case {@link RoleDescriptor.IndicesPrivileges}
      * * does not have a query field then it returns {@code null}.
      */
     @Nullable
     public static QueryBuilder evaluateAndVerifyRoleQuery(BytesReference query, ScriptService scriptService,
-                                                          NamedXContentRegistry xContentRegistry, User user) {
+                                                          NamedXContentRegistry xContentRegistry, User user,
+                                                          RestApiVersion restApiVersion) {
         if (query != null) {
             String templateResult = SecurityQueryTemplateEvaluator.evaluateTemplate(query.utf8ToString(), scriptService,
                 user);
             try {
-                return evaluateAndVerifyRoleQuery(templateResult, xContentRegistry);
+                return evaluateAndVerifyRoleQuery(templateResult, xContentRegistry, restApiVersion);
             } catch (ElasticsearchParseException | ParsingException | XContentParseException | IOException e) {
                 throw new ElasticsearchParseException("failed to parse field 'query' from the role descriptor", e);
             }
@@ -122,10 +126,11 @@ public static QueryBuilder evaluateAndVerifyRoleQuery(BytesReference query, Scri
     }
 
     @Nullable
-    private static QueryBuilder evaluateAndVerifyRoleQuery(String query, NamedXContentRegistry xContentRegistry) throws IOException {
+    private static QueryBuilder evaluateAndVerifyRoleQuery(String query, NamedXContentRegistry xContentRegistry,
+                                                           RestApiVersion restApiVersion) throws IOException {
         if (query != null) {
-            try (XContentParser parser = XContentFactory.xContent(query).createParser(xContentRegistry,
-                LoggingDeprecationHandler.INSTANCE, query)) {
+            try (XContentParser parser = XContentFactory.xContent(query).createParserForCompatibility(xContentRegistry,
+                LoggingDeprecationHandler.INSTANCE, query, restApiVersion)) {
                 QueryBuilder queryBuilder = AbstractQueryBuilder.parseInnerQueryBuilder(parser);
                 verifyRoleQuery(queryBuilder);
                 return queryBuilder;

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/TransportTermsEnumAction.java

@@ -36,6 +36,7 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.util.concurrent.EsThreadPoolExecutor;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.core.internal.io.IOUtils;
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.engine.Engine;
@@ -435,8 +436,8 @@ private boolean canAccess(
                             querySource,
                             scriptService,
                             queryShardContext.getXContentRegistry(),
-                            securityContext.getUser()
-                        );
+                            securityContext.getUser(),
+                            RestApiVersion.current());
                         QueryBuilder rewrittenQueryBuilder = Rewriteable.rewrite(queryBuilder, queryShardContext);
                         if (rewrittenQueryBuilder instanceof MatchAllQueryBuilder) {
                             // One of the roles assigned has "all" permissions - allow unfettered access to termsDict

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/role/TransportPutRoleAction.java

@@ -6,7 +6,6 @@
  */
 package org.elasticsearch.xpack.security.action.role;
 
-import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.HandledTransportAction;
@@ -18,7 +17,6 @@
 import org.elasticsearch.xpack.core.security.action.role.PutRoleRequest;
 import org.elasticsearch.xpack.core.security.action.role.PutRoleResponse;
 import org.elasticsearch.xpack.core.security.authz.store.ReservedRolesStore;
-import org.elasticsearch.xpack.core.security.authz.support.DLSRoleQueryValidator;
 import org.elasticsearch.xpack.security.authz.store.NativeRolesStore;
 
 public class TransportPutRoleAction extends HandledTransportAction<PutRoleRequest, PutRoleResponse> {
@@ -42,13 +40,6 @@ protected void doExecute(Task task, final PutRoleRequest request, final ActionLi
             return;
         }
 
-        try {
-            DLSRoleQueryValidator.validateQueryField(request.roleDescriptor().getIndicesPrivileges(), xContentRegistry);
-        } catch (ElasticsearchException | IllegalArgumentException e) {
-            listener.onFailure(e);
-            return;
-        }
-
         rolesStore.putRole(request, request.roleDescriptor(), listener.delegateFailure((l, created) -> {
             if (created) {
                 logger.info("added role [{}]", request.name());

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/ApiKeyGenerator.java

@@ -11,6 +11,7 @@
 import org.elasticsearch.ElasticsearchSecurityException;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.xpack.core.security.action.CreateApiKeyRequest;
 import org.elasticsearch.xpack.core.security.action.CreateApiKeyResponse;
 import org.elasticsearch.xpack.core.security.authc.Authentication;
@@ -44,7 +45,7 @@ public void generateApiKey(Authentication authentication, CreateApiKeyRequest re
             ActionListener.wrap(roleDescriptors -> {
                     for (RoleDescriptor rd : roleDescriptors) {
                         try {
-                            DLSRoleQueryValidator.validateQueryField(rd.getIndicesPrivileges(), xContentRegistry);
+                            DLSRoleQueryValidator.validateQueryField(rd.getIndicesPrivileges(), xContentRegistry, RestApiVersion.current());
                         } catch (ElasticsearchException | IllegalArgumentException e) {
                             listener.onFailure(e);
                             return;

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/store/FileRolesStore.java

@@ -21,6 +21,7 @@
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.yaml.YamlXContent;
+import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.license.XPackLicenseState.Feature;
@@ -318,7 +319,7 @@ private static RoleDescriptor checkDescriptor(RoleDescriptor descriptor, Path pa
                 return null;
             } else {
                 try {
-                    DLSRoleQueryValidator.validateQueryField(descriptor.getIndicesPrivileges(), xContentRegistry);
+                    DLSRoleQueryValidator.validateQueryField(descriptor.getIndicesPrivileges(), xContentRegistry, RestApiVersion.current());
                 } catch (ElasticsearchException | IllegalArgumentException e) {
                     logger.error((Supplier<?>) () -> new ParameterizedMessage(
                         "invalid role definition [{}] in roles file [{}]. failed to validate query field. skipping role...", roleName,