Merge branch 'upstream/master' into tsdb-deal-timestreamp

* upstream/master: (209 commits)
  Enforce license expiration (elastic#79671)
  TSDB: Automatically add timestamp mapper (elastic#79136)
  [DOCS]  `_id` is required for bulk API's `update` action (elastic#79774)
  EQL: Add optional fields and limit joining keys on non-null values only (elastic#79677)
  [DOCS] Document range enrich policy (elastic#79607)
  [DOCS] Fix typos in 8.0 security migration (elastic#79802)
  Allow listing older repositories (elastic#78244)
  [ML] track inference model feature usage per node (elastic#79752)
  Remove IncrementalClusterStateWriter & related code (elastic#79738)
  Reuse previous indices lookup when possible (elastic#79004)
  Reduce merging in PersistedClusterStateService (elastic#79793)
  SQL: Adjust JDBC docs to use milliseconds for timeouts (elastic#79628)
  Remove endpoint for freezing indices (elastic#78918)
  [ML] add timeout parameter for DELETE trained_models API (elastic#79739)
  [ML] wait for .ml-state-write alias to be readable (elastic#79731)
  [Docs] Update edgengram-tokenizer.asciidoc (elastic#79577)
  [Test][Transform] fix UpdateTransformActionRequestTests failure (elastic#79787)
  Limit CS Update Task Description Size (elastic#79443)
  Apply the reader wrapper on can_match source (elastic#78988)
  [DOCS] Adds new transform limitation item and a note to the tutorial (elastic#79479)
  ...

# Conflicts:
#	server/src/main/java/org/elasticsearch/index/IndexMode.java
#	server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java

Author: weizijun

.backportrc.json

@@ -2,15 +2,14 @@
   "upstream": "elastic/elasticsearch",
   "targetBranchChoices": [
     { "name": "master", "checked": true },
-    { "name": "7.x", "checked": true },
+    "7.16",
     "7.15",
     "7.14",
     "6.8"
   ],
   "targetPRLabels": ["backport"],
   "branchLabelMapping": {
     "^v8.0.0$": "master",
-    "^v7.16.0$": "7.x",
     "^v(\\d+).(\\d+).\\d+$": "$1.$2"
   }
 }

.ci/jobs.t/elastic+elasticsearch+pull-request+packaging-tests-windows-sample.yml

@@ -21,8 +21,10 @@
           github-hooks: true
           status-context: elasticsearch-ci/packaging-tests-windows-sample
           cancel-builds-on-update: true
-          white-list-target-branches:
-            - master
+          black-list-target-branches:
+            - 7.16
+            - 7.15
+            - 6.8
           excluded-regions:
             - ^docs/.*
           black-list-labels:

.ci/jobs.t/elastic+elasticsearch+pull-request+packaging-tests-windows.yml

@@ -21,8 +21,10 @@
           github-hooks: true
           status-context: elasticsearch-ci/packaging-tests-windows
           cancel-builds-on-update: true
-          white-list-target-branches:
-            - master
+          black-list-target-branches:
+            - 7.16
+            - 7.15
+            - 6.8
           excluded-regions:
             - ^docs/.*
           white-list-labels:

.ci/jobs.t/elastic+elasticsearch+pull-request+release-tests.yml

@@ -0,0 +1,49 @@
+---
+- job:
+    name: "elastic+elasticsearch+pull-request+release-tests"
+    display-name: "elastic / elasticsearch - pull request release-tests"
+    description: "Testing of Elasticsearch pull requests - release-tests"
+    workspace: "/dev/shm/elastic+elasticsearch+pull-request+release-tests"
+    scm:
+      - git:
+          refspec: "+refs/pull/${ghprbPullId}/*:refs/remotes/origin/pr/${ghprbPullId}/*"
+          branches:
+            - "${ghprbActualCommit}"
+    triggers:
+      - github-pull-request:
+          org-list:
+            - elastic
+          allow-whitelist-orgs-as-admins: true
+          trigger-phrase: '.*run\W+elasticsearch-ci/release-tests.*'
+          github-hooks: true
+          status-context: elasticsearch-ci/release-tests
+          cancel-builds-on-update: true
+          excluded-regions:
+            - ^docs/.*
+          white-list-labels:
+            - 'test-release'
+          black-list-target-branches:
+            - 7.15
+            - 6.8
+    builders:
+      - inject:
+          properties-file: '.ci/java-versions.properties'
+          properties-content: |
+            JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA
+            RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA
+            JAVA8_HOME=$HOME/.java/java8
+            JAVA11_HOME=$HOME/.java/java11
+            JAVA15_HOME=$HOME/.java/openjdk15
+      - shell: |
+          #!/usr/local/bin/runbld --redirect-stderr
+
+          # Fetch beats artifacts
+          export ES_VERSION=$(grep 'elasticsearch' build-tools-internal/version.properties | awk '{print $3}')
+          export BEATS_DIR=$(pwd)/distribution/docker/build/artifacts/beats
+
+          mkdir -p ${BEATS_DIR}
+          curl -o "${BEATS_DIR}/metricbeat-${ES_VERSION}-linux-x86_64.tar.gz" https://snapshots-no-kpi.elastic.co/downloads/beats/metricbeat/metricbeat-${ES_VERSION}-SNAPSHOT-linux-x86_64.tar.gz
+          curl -o "${BEATS_DIR}/filebeat-${ES_VERSION}-linux-x86_64.tar.gz" https://snapshots-no-kpi.elastic.co/downloads/beats/filebeat/filebeat-${ES_VERSION}-SNAPSHOT-linux-x86_64.tar.gz
+
+          $WORKSPACE/.ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dbuild.snapshot=false \
+            -Dtests.jvm.argline=-Dbuild.snapshot=false -Dlicense.key=${WORKSPACE}/x-pack/license-tools/src/test/resources/public.key -Dbuild.id=deadbeef build

.ci/jobs.t/elastic+elasticsearch+pull-request+rest-compatibility.yml

@@ -18,8 +18,10 @@
           github-hooks: true
           status-context: elasticsearch-ci/rest-compatibility
           cancel-builds-on-update: true
-          white-list-target-branches:
-            - master
+          black-list-target-branches:
+            - 7.16
+            - 7.15
+            - 6.8
           excluded-regions:
             - ^docs/.*
           black-list-labels:

build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/precommit/FormattingPrecommitPlugin.java

@@ -89,7 +89,6 @@ private Object[] getTargets(String projectPath) {
             return new String[] {
                 "src/*/java/org/elasticsearch/action/admin/cluster/repositories/**/*.java",
                 "src/*/java/org/elasticsearch/action/admin/cluster/snapshots/**/*.java",
-                "src/test/java/org/elasticsearch/common/xcontent/support/AbstractFilteringTestCase.java",
                 "src/test/java/org/elasticsearch/common/xcontent/support/XContentMapValuesTests.java",
                 "src/*/java/org/elasticsearch/index/IndexMode.java",
                 "src/*/java/org/elasticsearch/index/IndexRouting.java",
@@ -98,6 +97,10 @@ private Object[] getTargets(String projectPath) {
                 "src/*/java/org/elasticsearch/repositories/**/*.java",
                 "src/*/java/org/elasticsearch/search/aggregations/**/*.java",
                 "src/*/java/org/elasticsearch/snapshots/**/*.java" };
+        } else if (projectPath.equals(":test:framework")) {
+            return new String[] {
+                "src/test/java/org/elasticsearch/common/xcontent/support/AbstractFilteringTestCase.java",
+            };
         } else {
             // Normally this isn"t necessary, but we have Java sources in
             // non-standard places
@@ -115,6 +118,10 @@ private Object[] getTargets(String projectPath) {
         ":client:sniffer",
         ":client:test",
         ":distribution:archives:integ-test-zip",
+        ":distribution:bwc:bugfix",
+        ":distribution:bwc:maintenance",
+        ":distribution:bwc:minor",
+        ":distribution:bwc:staged",
         ":distribution:docker",
         ":docs",
         ":example-plugins:custom-settings",
@@ -203,7 +210,6 @@ private Object[] getTargets(String projectPath) {
         ":test:fixtures:geoip-fixture",
         ":test:fixtures:krb5kdc-fixture",
         ":test:fixtures:old-elasticsearch",
-        ":test:framework",
         ":test:logger-usage",
         ":x-pack:docs",
         ":x-pack:license-tools",

build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/precommit/LicenseHeadersTask.java

@@ -36,7 +36,7 @@
 import org.w3c.dom.Element;
 import org.w3c.dom.NodeList;
 import org.xml.sax.SAXException;
-
+import org.gradle.api.model.ObjectFactory;
 import javax.xml.parsers.DocumentBuilderFactory;
 import javax.xml.parsers.ParserConfigurationException;
 import java.io.BufferedWriter;
@@ -51,6 +51,8 @@
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
+import javax.inject.Inject;
+import java.io.Serializable;
 
 /**
  * Checks files for license headers..
@@ -95,17 +97,18 @@ public List<String> getExcludes() {
         return excludes;
     }
 
-    public Map<String, String> getAdditionalLicenses() {
-        return additionalLicenses;
-    }
-
     public void setExcludes(List<String> excludes) {
         this.excludes = excludes;
     }
 
     @OutputFile
     private File reportFile = new File(getProject().getBuildDir(), "reports/licenseHeaders/rat.xml");
 
+    private static List<License> conventionalLicenses = Arrays.asList(
+            // Dual SSPLv1 and Elastic
+            new License("DUAL", "SSPL+Elastic License", "the Elastic License 2.0 or the Server")
+    );
+
     /**
      * Allowed license families for this project.
      */
@@ -118,13 +121,17 @@ public void setExcludes(List<String> excludes) {
      */
     @Input
     private List<String> excludes = new ArrayList<String>();
+
+    private ListProperty<License> additionalLicenses;
+
     /**
      * Additional license families that may be found. The key is the license category name (5 characters),
      * followed by the family name and the value list of patterns to search for.
      */
     @Input
-    protected Map<String, String> additionalLicenses = new HashMap<String, String>();
-
+    public ListProperty<License> getAdditionalLicenses() {
+        return additionalLicenses;
+    }
     /**
      * Add a new license type.
      * <p>
@@ -139,7 +146,12 @@ public void additionalLicense(final String categoryName, String familyName, Stri
             throw new IllegalArgumentException("License category name must be exactly 5 characters, got " + categoryName);
         }
 
-        additionalLicenses.put(categoryName + familyName, pattern);
+        additionalLicenses.add(new License(categoryName, familyName, pattern));
+    }
+
+    @Inject
+    public LicenseHeadersTask(ObjectFactory objectFactory) {
+        additionalLicenses = objectFactory.listProperty(License.class).convention(conventionalLicenses);
     }
 
     @TaskAction
@@ -160,14 +172,10 @@ public void runRat() {
         matchers.add(subStringMatcher("GEN  ", "Generated", "ANTLR GENERATED CODE"));
         // Vendored Code
         matchers.add(subStringMatcher("VEN  ", "Vendored", "@notice"));
-        // Dual SSPLv1 and Elastic
-        matchers.add(subStringMatcher("DUAL", "SSPL+Elastic License", "the Elastic License 2.0 or the Server"));
 
-        for (Map.Entry<String, String> additional : additionalLicenses.entrySet()) {
-            String category = additional.getKey().substring(0, 5);
-            String family = additional.getKey().substring(5);
-            matchers.add(subStringMatcher(category, family, additional.getValue()));
-        }
+        additionalLicenses.get().forEach(l ->
+            matchers.add(subStringMatcher(l.licenseFamilyCategory, l.licenseFamilyName, l.substringPattern))
+        );
 
         reportConfiguration.setHeaderMatcher(new HeaderMatcherMultiplexer(matchers.toArray(IHeaderMatcher[]::new)));
         reportConfiguration.setApprovedLicenseNames(approvedLicenses.stream().map(license -> {
@@ -190,7 +198,6 @@ private IHeaderMatcher subStringMatcher(String licenseFamilyCategory, String lic
         SubstringLicenseMatcher substringLicenseMatcher = new SubstringLicenseMatcher();
         substringLicenseMatcher.setLicenseFamilyCategory(licenseFamilyCategory);
         substringLicenseMatcher.setLicenseFamilyName(licenseFamilyName);
-
         SubstringLicenseMatcher.Pattern pattern = new SubstringLicenseMatcher.Pattern();
         pattern.setSubstring(substringPattern);
         substringLicenseMatcher.addConfiguredPattern(pattern);
@@ -249,4 +256,16 @@ private static List<Element> elementList(NodeList resourcesNodes) {
         }
         return nodeList;
     }
+
+    static class License implements Serializable {
+        private String licenseFamilyCategory;
+        private String licenseFamilyName;
+        private String substringPattern;
+
+        public License(String licenseFamilyCategory, String licenseFamilyName, String substringPattern) {
+            this.licenseFamilyCategory = licenseFamilyCategory;
+            this.licenseFamilyName = licenseFamilyName;
+            this.substringPattern = substringPattern;
+        }
+    }
 }

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalBwcGitPluginFuncTest.groovy

@@ -21,11 +21,11 @@ class InternalBwcGitPluginFuncTest extends AbstractGitAwareGradleFuncTest {
 
             bwcGitConfig {
                  bwcVersion = project.provider { Version.fromString("7.9.1") }
-                 bwcBranch = project.provider { "7.x" }
+                 bwcBranch = project.provider { "7.9" }
                  checkoutDir = project.provider{file("build/checkout")}
             }
         """
-        execute("git branch origin/7.x", file("cloned"))
+        execute("git branch origin/7.9", file("cloned"))
     }
 
     def "current repository can be cloned"() {