Merge remote-tracking branch 'upstream/master' into 45223-refactor-env-var-handling

Author: pugnascotia

CONTRIBUTING.md

@@ -115,6 +115,10 @@ We support development in the Eclipse and IntelliJ IDEs.
 For Eclipse, the minimum version that we support is [4.13][eclipse].
 For IntelliJ, the minimum version that we support is [IntelliJ 2017.2][intellij].
 
+[Docker](https://docs.docker.com/install/) is required for building some Elasticsearch artifacts and executing certain test suites. You can run Elasticsearch without building all the artifacts with:
+
+    ./gradlew :run
+
 ### Configuring IDEs And Running Tests
 
 Eclipse users can automatically configure their IDE: `./gradlew eclipse`
@@ -153,6 +157,10 @@ For IntelliJ, go to
 For Eclipse, go to `Preferences->Java->Installed JREs` and add `-ea` to
 `VM Arguments`.
 
+Some tests related to locale testing also require the flag 
+`-Djava.locale.providers` to be set. Set the VM options/VM arguments for
+IntelliJ or Eclipse like describe above to use 
+`-Djava.locale.providers=SPI,COMPAT`.
 
 ### Java Language Formatting Guidelines
 

TESTING.asciidoc

@@ -263,6 +263,10 @@ If you want to just run the precommit checks:
 
 Some of these checks will require `docker-compose` installed for bringing up
 test fixtures. If it's not present those checks will be skipped automatically.
+The host running Docker (or VM if you're using Docker Desktop) needs 4GB of
+memory or some of the containers will fail to start. You can tell that you
+are short of memory if containers are exiting quickly after starting with
+code 137 (128 + 9, where 9 means SIGKILL).
 
 == Testing the REST layer
 

client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsSource.java

@@ -26,6 +26,7 @@
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
 
 import java.io.IOException;
 import java.util.Arrays;
@@ -44,20 +45,27 @@ public static Builder builder() {
 
     private static final ParseField INDEX = new ParseField("index");
     private static final ParseField QUERY = new ParseField("query");
+    public static final ParseField _SOURCE = new ParseField("_source");
 
     private static ObjectParser<Builder, Void> PARSER = new ObjectParser<>("data_frame_analytics_source", true, Builder::new);
 
     static {
         PARSER.declareStringArray(Builder::setIndex, INDEX);
         PARSER.declareObject(Builder::setQueryConfig, (p, c) -> QueryConfig.fromXContent(p), QUERY);
+        PARSER.declareField(Builder::setSourceFiltering,
+            (p, c) -> FetchSourceContext.fromXContent(p),
+            _SOURCE,
+            ObjectParser.ValueType.OBJECT_ARRAY_BOOLEAN_OR_STRING);
     }
 
     private final String[] index;
     private final QueryConfig queryConfig;
+    private final FetchSourceContext sourceFiltering;
 
-    private DataFrameAnalyticsSource(String[] index, @Nullable QueryConfig queryConfig) {
+    private DataFrameAnalyticsSource(String[] index, @Nullable QueryConfig queryConfig, @Nullable FetchSourceContext sourceFiltering) {
         this.index = Objects.requireNonNull(index);
         this.queryConfig = queryConfig;
+        this.sourceFiltering = sourceFiltering;
     }
 
     public String[] getIndex() {
@@ -68,13 +76,20 @@ public QueryConfig getQueryConfig() {
         return queryConfig;
     }
 
+    public FetchSourceContext getSourceFiltering() {
+        return sourceFiltering;
+    }
+
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
         builder.field(INDEX.getPreferredName(), index);
         if (queryConfig != null) {
             builder.field(QUERY.getPreferredName(), queryConfig.getQuery());
         }
+        if (sourceFiltering != null) {
+            builder.field(_SOURCE.getPreferredName(), sourceFiltering);
+        }
         builder.endObject();
         return builder;
     }
@@ -86,12 +101,13 @@ public boolean equals(Object o) {
 
         DataFrameAnalyticsSource other = (DataFrameAnalyticsSource) o;
         return Arrays.equals(index, other.index)
-            && Objects.equals(queryConfig, other.queryConfig);
+            && Objects.equals(queryConfig, other.queryConfig)
+            && Objects.equals(sourceFiltering, other.sourceFiltering);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(Arrays.asList(index), queryConfig);
+        return Objects.hash(Arrays.asList(index), queryConfig, sourceFiltering);
     }
 
     @Override
@@ -103,6 +119,7 @@ public static class Builder {
 
         private String[] index;
         private QueryConfig queryConfig;
+        private FetchSourceContext sourceFiltering;
 
         private Builder() {}
 
@@ -121,8 +138,13 @@ public Builder setQueryConfig(QueryConfig queryConfig) {
             return this;
         }
 
+        public Builder setSourceFiltering(FetchSourceContext sourceFiltering) {
+            this.sourceFiltering = sourceFiltering;
+            return this;
+        }
+
         public DataFrameAnalyticsSource build() {
-            return new DataFrameAnalyticsSource(index, queryConfig);
+            return new DataFrameAnalyticsSource(index, queryConfig, sourceFiltering);
         }
     }
 }

client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/CRUDDocumentationIT.java

@@ -83,7 +83,6 @@
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
-import org.elasticsearch.search.sort.SortOrder;
 import org.elasticsearch.tasks.TaskId;
 
 import java.util.Collections;
@@ -833,10 +832,6 @@ public void testReindex() throws Exception {
             // tag::reindex-request-pipeline
             request.setDestPipeline("my_pipeline"); // <1>
             // end::reindex-request-pipeline
-            // tag::reindex-request-sort
-            request.addSortField("field1", SortOrder.DESC); // <1>
-            request.addSortField("field2", SortOrder.ASC); // <2>
-            // end::reindex-request-sort
             // tag::reindex-request-script
             request.setScript(
                 new Script(

client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java

@@ -2939,6 +2939,9 @@ public void testPutDataFrameAnalytics() throws Exception {
             DataFrameAnalyticsSource sourceConfig = DataFrameAnalyticsSource.builder() // <1>
                 .setIndex("put-test-source-index") // <2>
                 .setQueryConfig(queryConfig) // <3>
+                .setSourceFiltering(new FetchSourceContext(true,
+                    new String[] { "included_field_1", "included_field_2" },
+                    new String[] { "excluded_field" })) // <4>
                 .build();
             // end::put-data-frame-analytics-source-config
 

client/rest-high-level/src/test/java/org/elasticsearch/client/ml/dataframe/DataFrameAnalyticsSourceTests.java

@@ -23,6 +23,7 @@
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.search.SearchModule;
+import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
 import org.elasticsearch.test.AbstractXContentTestCase;
 
 import java.io.IOException;
@@ -35,9 +36,17 @@
 public class DataFrameAnalyticsSourceTests extends AbstractXContentTestCase<DataFrameAnalyticsSource> {
 
     public static DataFrameAnalyticsSource randomSourceConfig() {
+        FetchSourceContext sourceFiltering = null;
+        if (randomBoolean()) {
+            sourceFiltering = new FetchSourceContext(true,
+                generateRandomStringArray(10, 10, false, false),
+                generateRandomStringArray(10, 10, false, false));
+        }
+
         return DataFrameAnalyticsSource.builder()
             .setIndex(generateRandomStringArray(10, 10, false, false))
             .setQueryConfig(randomBoolean() ? null : randomQueryConfig())
+            .setSourceFiltering(sourceFiltering)
             .build();
     }
 

distribution/docker/docker-compose.yml

@@ -1,12 +1,12 @@
 # Only used for testing the docker images
-version: '3'
+version: '3.7'
 services:
   elasticsearch-default-1:
     image: elasticsearch:test
-    environment:  
-       - node.name=elasticsearch-default-1 
+    environment:
+       - node.name=elasticsearch-default-1
        - cluster.initial_master_nodes=elasticsearch-default-1,elasticsearch-default-2
-       - discovery.seed_hosts=elasticsearch-default-2:9300 
+       - discovery.seed_hosts=elasticsearch-default-2:9300
        - cluster.name=elasticsearch-default
        - bootstrap.memory_lock=true
        - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
@@ -16,20 +16,20 @@ services:
        - cluster.routing.allocation.disk.watermark.high=1b
        - cluster.routing.allocation.disk.watermark.flood_stage=1b
        - script.max_compilations_rate=2048/1m
-       - node.store.allow_mmap=false  
+       - node.store.allow_mmap=false
        - xpack.security.enabled=true
        - xpack.security.transport.ssl.enabled=true
        - xpack.security.http.ssl.enabled=true
        - xpack.security.authc.token.enabled=true
        - xpack.security.audit.enabled=true
-       - xpack.security.authc.realms.file.file1.order=0   
+       - xpack.security.authc.realms.file.file1.order=0
        - xpack.security.authc.realms.native.native1.order=1
        - xpack.security.transport.ssl.keystore.path=/usr/share/elasticsearch/config/testnode.jks
        - xpack.security.http.ssl.keystore.path=/usr/share/elasticsearch/config/testnode.jks
-       - xpack.http.ssl.verification_mode=certificate  
-       - xpack.security.transport.ssl.verification_mode=certificate  
+       - xpack.http.ssl.verification_mode=certificate
+       - xpack.security.transport.ssl.verification_mode=certificate
        - xpack.license.self_generated.type=trial
-    volumes: 
+    volumes:
        - ./build/repo:/tmp/es-repo
        - ./build/certs/testnode.jks:/usr/share/elasticsearch/config/testnode.jks
        - ./build/logs/default-1:/usr/share/elasticsearch/logs
@@ -42,14 +42,20 @@ services:
         hard: -1
       nofile:
         soft: 65536
-        hard: 65536      
+        hard: 65536
     entrypoint: /docker-test-entrypoint.sh
+    healthcheck:
+      start_period: 15s
+      test: ["CMD", "curl", "-f", "-u", "x_pack_rest_user:x-pack-test-password", "-k", "https://localhost:9200"]
+      interval: 10s
+      timeout: 2s
+      retries: 5
   elasticsearch-default-2:
     image: elasticsearch:test
-    environment:  
+    environment:
        - node.name=elasticsearch-default-2
        - cluster.initial_master_nodes=elasticsearch-default-1,elasticsearch-default-2
-       - discovery.seed_hosts=elasticsearch-default-1:9300 
+       - discovery.seed_hosts=elasticsearch-default-1:9300
        - cluster.name=elasticsearch-default
        - bootstrap.memory_lock=true
        - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
@@ -59,20 +65,20 @@ services:
        - cluster.routing.allocation.disk.watermark.high=1b
        - cluster.routing.allocation.disk.watermark.flood_stage=1b
        - script.max_compilations_rate=2048/1m
-       - node.store.allow_mmap=false  
+       - node.store.allow_mmap=false
        - xpack.security.enabled=true
        - xpack.security.transport.ssl.enabled=true
        - xpack.security.http.ssl.enabled=true
        - xpack.security.authc.token.enabled=true
        - xpack.security.audit.enabled=true
-       - xpack.security.authc.realms.file.file1.order=0   
-       - xpack.security.authc.realms.native.native1.order=1  
+       - xpack.security.authc.realms.file.file1.order=0
+       - xpack.security.authc.realms.native.native1.order=1
        - xpack.security.transport.ssl.keystore.path=/usr/share/elasticsearch/config/testnode.jks
        - xpack.security.http.ssl.keystore.path=/usr/share/elasticsearch/config/testnode.jks
-       - xpack.http.ssl.verification_mode=certificate  
-       - xpack.security.transport.ssl.verification_mode=certificate  
+       - xpack.http.ssl.verification_mode=certificate
+       - xpack.security.transport.ssl.verification_mode=certificate
        - xpack.license.self_generated.type=trial
-    volumes: 
+    volumes:
        - ./build/repo:/tmp/es-repo
        - ./build/certs/testnode.jks:/usr/share/elasticsearch/config/testnode.jks
        - ./build/logs/default-2:/usr/share/elasticsearch/logs
@@ -85,14 +91,20 @@ services:
         hard: -1
       nofile:
         soft: 65536
-        hard: 65536            
+        hard: 65536
     entrypoint: /docker-test-entrypoint.sh
+    healthcheck:
+      start_period: 15s
+      test: ["CMD", "curl", "-f", "-u", "x_pack_rest_user:x-pack-test-password", "-k", "https://localhost:9200"]
+      interval: 10s
+      timeout: 2s
+      retries: 5
   elasticsearch-oss-1:
     image: elasticsearch:test
-    environment:  
-       - node.name=elasticsearch-oss-1 
+    environment:
+       - node.name=elasticsearch-oss-1
        - cluster.initial_master_nodes=elasticsearch-oss-1,elasticsearch-oss-2
-       - discovery.seed_hosts=elasticsearch-oss-2:9300 
+       - discovery.seed_hosts=elasticsearch-oss-2:9300
        - cluster.name=elasticsearch-oss
        - bootstrap.memory_lock=true
        - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
@@ -102,8 +114,8 @@ services:
        - cluster.routing.allocation.disk.watermark.high=1b
        - cluster.routing.allocation.disk.watermark.flood_stage=1b
        - script.max_compilations_rate=2048/1m
-       - node.store.allow_mmap=false  
-    volumes: 
+       - node.store.allow_mmap=false
+    volumes:
        - ./build/oss-repo:/tmp/es-repo
        - ./build/logs/oss-1:/usr/share/elasticsearch/logs
     ports:
@@ -114,13 +126,19 @@ services:
         hard: -1
       nofile:
         soft: 65536
-        hard: 65536    
+        hard: 65536
+    healthcheck:
+      start_period: 15s
+      test: ["CMD", "curl", "-f", "http://localhost:9200"]
+      interval: 10s
+      timeout: 2s
+      retries: 5
   elasticsearch-oss-2:
     image: elasticsearch:test
-    environment:  
+    environment:
        - node.name=elasticsearch-oss-2
        - cluster.initial_master_nodes=elasticsearch-oss-1,elasticsearch-oss-2
-       - discovery.seed_hosts=elasticsearch-oss-1:9300 
+       - discovery.seed_hosts=elasticsearch-oss-1:9300
        - cluster.name=elasticsearch-oss
        - bootstrap.memory_lock=true
        - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
@@ -130,8 +148,8 @@ services:
        - cluster.routing.allocation.disk.watermark.high=1b
        - cluster.routing.allocation.disk.watermark.flood_stage=1b
        - script.max_compilations_rate=2048/1m
-       - node.store.allow_mmap=false  
-    volumes: 
+       - node.store.allow_mmap=false
+    volumes:
        - ./build/oss-repo:/tmp/es-repo
        - ./build/logs/oss-2:/usr/share/elasticsearch/logs
     ports:
@@ -140,3 +158,9 @@ services:
       memlock:
         soft: -1
         hard: -1
+    healthcheck:
+      start_period: 15s
+      test: ["CMD", "curl", "-f", "http://localhost:9200"]
+      interval: 10s
+      timeout: 2s
+      retries: 5

distribution/docker/src/docker/Dockerfile

@@ -33,6 +33,7 @@ RUN grep ES_DISTRIBUTION_TYPE=tar /usr/share/elasticsearch/bin/elasticsearch-env
 RUN mkdir -p config data logs
 RUN chmod 0775 config data logs
 COPY config/elasticsearch.yml config/log4j2.properties config/
+RUN chmod 0660 config/elasticsearch.yml config/log4j2.properties
 
 ################################################################################
 # Build stage 1 (the actual elasticsearch image):

docs/java-rest/high-level/document/reindex.asciidoc

@@ -89,16 +89,6 @@ include-tagged::{doc-tests-file}[{api}-request-pipeline]
 --------------------------------------------------
 <1> set pipeline to `my_pipeline`
 
-If you want a particular set of documents from the source index you’ll need to use sort. If possible, prefer a more
-selective query to maxDocs and sort.
-
-["source","java",subs="attributes,callouts,macros"]
---------------------------------------------------
-include-tagged::{doc-tests-file}[{api}-request-sort]
---------------------------------------------------
-<1> add descending sort to`field1`
-<2> add ascending sort to `field2`
-
 +{request}+ also supports a `script` that modifies the document. It allows you to
 also change the document's metadata. The following example illustrates that.
 

docs/java-rest/high-level/ml/put-data-frame-analytics.asciidoc

@@ -52,6 +52,7 @@ include-tagged::{doc-tests-file}[{api}-source-config]
 <1> Constructing a new DataFrameAnalyticsSource
 <2> The source index
 <3> The query from which to gather the data. If query is not set, a `match_all` query is used by default.
+<4> Source filtering to select which fields will exist in the destination index.
 
 ===== QueryConfig