Merge branch '6.x' into ccr-6.x

* 6.x:
  Add get stored script and delete stored script to high level REST API
  Increasing skip version for failing test on 6.x
  Skip get_alias tests for 5.x (#31397)
  Fix defaults in GeoShapeFieldMapper output (#31302)
  Test: better error message on failure
  Mute DefaultShardsIT#testDefaultShards test
  Fix reference to XContentBuilder.string() (#31337)
  [DOCS] Adds monitoring breaking change (#31369)
  [DOCS] Adds security breaking change (#31375)
  [DOCS] Backports breaking change (#31373)
  RestAPI: Reject forcemerge requests with a body (#30792)
  Docs: Use the default distribution to test docs (#31251)
  Use system context for cluster state update tasks (#31241)
  [DOCS] Adds testing for security APIs (#31345)
  [DOCS] Removes ML item from release highlights
  [DOCS] Removes breaking change (#31376)
  REST high-level client: add validate query API (#31077)
  Move language analyzers from server to analysis-common module. (#31300)
  Expose lucene's RemoveDuplicatesTokenFilter (#31275)
  [Test] Fix :example-plugins:rest-handler on Windows
  Delete typos in SAML docs (#31199)
  Ensure we don't use a remote profile if cluster name matches (#31331)
  Test: Skip alias tests that failed all weekend
  [DOCS] Fix version in SQL JDBC Maven template
  [DOCS] Improve install and setup section for SQL JDBC
  Add ingest-attachment support for per document `indexed_chars` limit (#31352)
  SQL: Fix rest endpoint names in node stats (#31371)
  [DOCS] Fixes small issue in release notes
  Support for remote path in reindex api Closes #22913
  [ML] Put ML filter API response should contain the filter (#31362)
  Remove trial status info from start trial doc (#31365)
  [DOCS] Added links in breaking changes pages
  [DOCS] Adds links to release notes and highlights
  Docs: Document changes in rest client
  QA: Fix tribe tests to use node selector
  REST Client: NodeSelector for node attributes (#31296)
  LLClient: Fix assertion on windows
  LLClient: Support host selection (#30523)
  Add QA project and fixture based test for discovery-ec2 plugin (#31107)
  [ML] Hold ML filter items in sorted set (#31338)
  [ML] Add description to ML filters (#31330)

Author: dnhatn

buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/DocsTestPlugin.groovy

@@ -32,6 +32,8 @@ public class DocsTestPlugin extends RestTestPlugin {
     public void apply(Project project) {
         project.pluginManager.apply('elasticsearch.standalone-rest-test')
         super.apply(project)
+        // The distribution can be configured with -Dtests.distribution on the command line
+        project.integTestCluster.distribution = System.getProperty('tests.distribution', 'zip')
         // Docs are published separately so no need to assemble
         project.tasks.remove(project.assemble)
         project.build.dependsOn.remove('assemble')
@@ -43,6 +45,8 @@ public class DocsTestPlugin extends RestTestPlugin {
             '\\{version\\}':
                 VersionProperties.elasticsearch.toString().replace('-SNAPSHOT', ''),
             '\\{lucene_version\\}' : VersionProperties.lucene.replaceAll('-snapshot-\\w+$', ''),
+            '\\{build_flavor\\}' :
+                project.integTestCluster.distribution.startsWith('oss-') ? 'oss' : 'default',
         ]
         Task listSnippets = project.tasks.create('listSnippets', SnippetsTask)
         listSnippets.group 'Docs'

client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java

@@ -58,6 +58,8 @@
 import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse;
 import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest;
 import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateResponse;
+import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryRequest;
+import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryResponse;
 import org.elasticsearch.rest.RestStatus;
 
 import java.io.IOException;
@@ -1091,6 +1093,36 @@ public void putTemplateAsync(PutIndexTemplateRequest putIndexTemplateRequest, Re
             PutIndexTemplateResponse::fromXContent, listener, emptySet());
     }
 
+    /**
+     * Validate a potentially expensive query without executing it.
+     * <p>
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/search-validate.html"> Validate Query API
+     * on elastic.co</a>
+     * @param validateQueryRequest the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @return the response
+     * @throws IOException in case there is a problem sending the request or parsing back the response
+     */
+    public ValidateQueryResponse validateQuery(ValidateQueryRequest validateQueryRequest, RequestOptions options) throws IOException {
+        return restHighLevelClient.performRequestAndParseEntity(validateQueryRequest, RequestConverters::validateQuery, options,
+            ValidateQueryResponse::fromXContent, emptySet());
+    }
+
+    /**
+     * Asynchronously validate a potentially expensive query without executing it.
+     * <p>
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/search-validate.html"> Validate Query API
+     * on elastic.co</a>
+     * @param validateQueryRequest the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @param listener the listener to be notified upon request completion
+     */
+    public void validateQueryAsync(ValidateQueryRequest validateQueryRequest, RequestOptions options,
+                                   ActionListener<ValidateQueryResponse> listener) {
+        restHighLevelClient.performRequestAsyncAndParseEntity(validateQueryRequest, RequestConverters::validateQuery, options,
+            ValidateQueryResponse::fromXContent, listener, emptySet());
+    }
+
     /**
      * Gets index templates using the Index Templates API
      * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-templates.html"> Index Templates API

client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java

@@ -37,6 +37,8 @@
 import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryRequest;
 import org.elasticsearch.action.admin.cluster.repositories.verify.VerifyRepositoryRequest;
 import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest;
+import org.elasticsearch.action.admin.cluster.storedscripts.DeleteStoredScriptRequest;
+import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptRequest;
 import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
 import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
 import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheRequest;
@@ -58,6 +60,7 @@
 import org.elasticsearch.action.admin.indices.shrink.ResizeType;
 import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesRequest;
 import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest;
+import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryRequest;
 import org.elasticsearch.action.bulk.BulkRequest;
 import org.elasticsearch.action.delete.DeleteRequest;
 import org.elasticsearch.action.fieldcaps.FieldCapabilitiesRequest;
@@ -877,6 +880,20 @@ static Request getTemplates(GetIndexTemplatesRequest getIndexTemplatesRequest) t
         return request;
     }
 
+    static Request validateQuery(ValidateQueryRequest validateQueryRequest) throws IOException {
+        String[] indices = validateQueryRequest.indices() == null ? Strings.EMPTY_ARRAY : validateQueryRequest.indices();
+        String[] types = validateQueryRequest.types() == null || indices.length <= 0 ? Strings.EMPTY_ARRAY : validateQueryRequest.types();
+        String endpoint = endpoint(indices, types, "_validate/query");
+        Request request = new Request(HttpGet.METHOD_NAME, endpoint);
+        Params params = new Params(request);
+        params.withIndicesOptions(validateQueryRequest.indicesOptions());
+        params.putParam("explain", Boolean.toString(validateQueryRequest.explain()));
+        params.putParam("all_shards", Boolean.toString(validateQueryRequest.allShards()));
+        params.putParam("rewrite", Boolean.toString(validateQueryRequest.rewrite()));
+        request.setEntity(createEntity(validateQueryRequest, REQUEST_BODY_CONTENT_TYPE));
+        return request;
+    }
+
     static Request getAlias(GetAliasesRequest getAliasesRequest) {
         String[] indices = getAliasesRequest.indices() == null ? Strings.EMPTY_ARRAY : getAliasesRequest.indices();
         String[] aliases = getAliasesRequest.aliases() == null ? Strings.EMPTY_ARRAY : getAliasesRequest.aliases();
@@ -888,6 +905,23 @@ static Request getAlias(GetAliasesRequest getAliasesRequest) {
         return request;
     }
 
+    static Request getScript(GetStoredScriptRequest getStoredScriptRequest) {
+        String endpoint = new EndpointBuilder().addPathPartAsIs("_scripts").addPathPart(getStoredScriptRequest.id()).build();
+        Request request = new Request(HttpGet.METHOD_NAME, endpoint);
+        Params params = new Params(request);
+        params.withMasterTimeout(getStoredScriptRequest.masterNodeTimeout());
+        return request;
+    }
+
+    static Request deleteScript(DeleteStoredScriptRequest deleteStoredScriptRequest) {
+        String endpoint = new EndpointBuilder().addPathPartAsIs("_scripts").addPathPart(deleteStoredScriptRequest.id()).build();
+        Request request = new Request(HttpDelete.METHOD_NAME, endpoint);
+        Params params = new Params(request);
+        params.withTimeout(deleteStoredScriptRequest.timeout());
+        params.withMasterTimeout(deleteStoredScriptRequest.masterNodeTimeout());
+        return request;
+    }
+
     private static HttpEntity createEntity(ToXContent toXContent, XContentType xContentType) throws IOException {
         BytesRef source = XContentHelper.toXContent(toXContent, xContentType, false).toBytesRef();
         return new ByteArrayEntity(source.bytes, source.offset, source.length, createContentType(xContentType));

client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java

@@ -26,6 +26,10 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.action.admin.cluster.storedscripts.DeleteStoredScriptRequest;
+import org.elasticsearch.action.admin.cluster.storedscripts.DeleteStoredScriptResponse;
+import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptRequest;
+import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptResponse;
 import org.elasticsearch.action.bulk.BulkRequest;
 import org.elasticsearch.action.bulk.BulkResponse;
 import org.elasticsearch.action.delete.DeleteRequest;
@@ -963,6 +967,62 @@ public final FieldCapabilitiesResponse fieldCaps(FieldCapabilitiesRequest fieldC
             FieldCapabilitiesResponse::fromXContent, emptySet());
     }
 
+    /**
+     * Get stored script by id.
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-scripting-using.html">
+     *     How to use scripts on elastic.co</a>
+     * @param request the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @return the response
+     * @throws IOException in case there is a problem sending the request or parsing back the response
+     */
+    public GetStoredScriptResponse getScript(GetStoredScriptRequest request, RequestOptions options) throws IOException {
+        return performRequestAndParseEntity(request, RequestConverters::getScript, options,
+            GetStoredScriptResponse::fromXContent, emptySet());
+    }
+
+    /**
+     * Asynchronously get stored script by id.
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-scripting-using.html">
+     *     How to use scripts on elastic.co</a>
+     * @param request the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @param listener the listener to be notified upon request completion
+     */
+    public void getScriptAsync(GetStoredScriptRequest request, RequestOptions options,
+                               ActionListener<GetStoredScriptResponse> listener) {
+        performRequestAsyncAndParseEntity(request, RequestConverters::getScript, options,
+            GetStoredScriptResponse::fromXContent, listener, emptySet());
+    }
+
+    /**
+     * Delete stored script by id.
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-scripting-using.html">
+     *     How to use scripts on elastic.co</a>
+     * @param request the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @return the response
+     * @throws IOException in case there is a problem sending the request or parsing back the response
+     */
+    public DeleteStoredScriptResponse deleteScript(DeleteStoredScriptRequest request, RequestOptions options) throws IOException {
+        return performRequestAndParseEntity(request, RequestConverters::deleteScript, options,
+            DeleteStoredScriptResponse::fromXContent, emptySet());
+    }
+
+    /**
+     * Asynchronously delete stored script by id.
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-scripting-using.html">
+     *     How to use scripts on elastic.co</a>
+     * @param request the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @param listener the listener to be notified upon request completion
+     */
+    public void deleteScriptAsync(DeleteStoredScriptRequest request, RequestOptions options,
+                                  ActionListener<DeleteStoredScriptResponse> listener) {
+        performRequestAsyncAndParseEntity(request, RequestConverters::deleteScript, options,
+            DeleteStoredScriptResponse::fromXContent, listener, emptySet());
+    }
+
     /**
      * Asynchronously executes a request using the Field Capabilities API.
      * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/search-field-caps.html">Field Capabilities API

client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java

@@ -20,6 +20,7 @@
 package org.elasticsearch.client;
 
 import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.methods.HttpPost;
 import org.apache.http.client.methods.HttpPut;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.ElasticsearchStatusException;
@@ -64,6 +65,8 @@
 import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse;
 import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest;
 import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateResponse;
+import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryRequest;
+import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryResponse;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.action.support.WriteRequest;
@@ -81,6 +84,8 @@
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.rest.RestStatus;
 
 import java.io.IOException;
@@ -1197,6 +1202,40 @@ public void testPutTemplateBadRequests() throws Exception {
         assertThat(unknownSettingError.getDetailedMessage(), containsString("unknown setting [index.this-setting-does-not-exist]"));
     }
 
+    public void testValidateQuery() throws IOException{
+        String index = "some_index";
+        createIndex(index, Settings.EMPTY);
+        QueryBuilder builder = QueryBuilders
+            .boolQuery()
+            .must(QueryBuilders.queryStringQuery("*:*"))
+            .filter(QueryBuilders.termQuery("user", "kimchy"));
+        ValidateQueryRequest request = new ValidateQueryRequest(index).query(builder);
+        request.explain(randomBoolean());
+        ValidateQueryResponse response = execute(request, highLevelClient().indices()::validateQuery,
+            highLevelClient().indices()::validateQueryAsync);
+        assertTrue(response.isValid());
+    }
+
+    public void testInvalidValidateQuery() throws IOException{
+        String index = "shakespeare";
+
+        createIndex(index, Settings.EMPTY);
+        Request postDoc = new Request(HttpPost.METHOD_NAME, "/" + index + "/1");
+        postDoc.setJsonEntity(
+            "{\"type\":\"act\",\"line_id\":1,\"play_name\":\"Henry IV\", \"speech_number\":\"\"," +
+                "\"line_number\":\"\",\"speaker\":\"\",\"text_entry\":\"ACT I\"}");
+        assertOK(client().performRequest(postDoc));
+
+        QueryBuilder builder = QueryBuilders
+            .queryStringQuery("line_id:foo")
+            .lenient(false);
+        ValidateQueryRequest request = new ValidateQueryRequest(index).query(builder);
+        request.explain(true);
+        ValidateQueryResponse response = execute(request, highLevelClient().indices()::validateQuery,
+            highLevelClient().indices()::validateQueryAsync);
+        assertFalse(response.isValid());
+    }
+
     public void testGetIndexTemplate() throws Exception {
         RestHighLevelClient client = highLevelClient();