Merge branch 'master' into batch-settings-update

Author: joegallo

build-tools-internal/build.gradle

@@ -202,27 +202,35 @@ configurations {
   integTestRuntimeOnly.extendsFrom(testRuntimeOnly)
 }
 dependencies {
+  constraints {
+    api("com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:${versions.getProperty('jackson')}") {
+      version {
+        strictly "${versions.getProperty('jackson')}"
+      }
+      because 'We want to use the exact same jackson version we use in production'
+    }
+  }
   api localGroovy()
   api gradleApi()
 
   api "org.elasticsearch:build-conventions:$version"
   api "org.elasticsearch.gradle:build-tools:$version"
 
-  api 'commons-codec:commons-codec:1.12'
+  // same version as http client transitive dep
+  api 'commons-codec:commons-codec:1.11'
   api 'org.apache.commons:commons-compress:1.19'
   api 'org.apache.ant:ant:1.10.8'
-  api 'com.netflix.nebula:gradle-extra-configurations-plugin:5.0.1'
-  api 'com.netflix.nebula:gradle-info-plugin:9.2.0'
+  api 'com.netflix.nebula:gradle-info-plugin:11.1.0'
   api 'org.apache.rat:apache-rat:0.11'
   api "net.java.dev.jna:jna:${versions.getProperty('jna')}"
-  api 'gradle.plugin.com.github.jengelman.gradle.plugins:shadow:7.0.0'
+  api 'gradle.plugin.com.github.johnrengelman:shadow:7.1.2'
   // for our ide tweaking
-  api 'gradle.plugin.org.jetbrains.gradle.plugin.idea-ext:gradle-idea-ext:0.7'
+  api 'gradle.plugin.org.jetbrains.gradle.plugin.idea-ext:gradle-idea-ext:1.1.1'
   // When upgrading forbidden apis, ensure dependency version is bumped in ThirdPartyPrecommitPlugin as well
   api 'de.thetaphi:forbiddenapis:3.2'
-  api 'com.avast.gradle:gradle-docker-compose-plugin:0.14.0'
+  api 'com.avast.gradle:gradle-docker-compose-plugin:0.14.13'
   api 'org.apache.maven:maven-model:3.6.2'
-  api 'com.networknt:json-schema-validator:1.0.36'
+  api 'com.networknt:json-schema-validator:1.0.65'
   api "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:${versions.getProperty('jackson')}"
   api 'org.ow2.asm:asm:9.2'
   api 'org.ow2.asm:asm-tree:9.2'
@@ -231,7 +239,6 @@ dependencies {
   compileOnly "com.puppycrawl.tools:checkstyle:${versions.getProperty('checkstyle')}"
   runtimeOnly "org.elasticsearch.gradle:reaper:$version"
   testImplementation "com.puppycrawl.tools:checkstyle:${versions.getProperty('checkstyle')}"
-//  testImplementation "junit:junit:${versions.getProperty('junit')}"
   testImplementation 'com.github.tomakehurst:wiremock-jre8-standalone:2.23.2'
   testImplementation 'org.mockito:mockito-core:1.9.5'
   testImplementation "org.hamcrest:hamcrest:${versions.getProperty('hamcrest')}"
@@ -290,4 +297,4 @@ tasks.register("bootstrapPerformanceTests", Copy) {
   into('build/performanceTests')
   def root = file('..')
   filter(ReplaceTokens, tokens: [testGitCommit:GitInfo.gitInfo(root).revision])
-}
+}

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcVersions.java

@@ -10,7 +10,6 @@
 import org.elasticsearch.gradle.Architecture;
 import org.elasticsearch.gradle.Version;
 import org.elasticsearch.gradle.VersionProperties;
-import org.jetbrains.annotations.NotNull;
 
 import java.util.ArrayList;
 import java.util.Collections;
@@ -284,7 +283,7 @@ public record UnreleasedVersionInfo(Version version, String branch, String gradl
     public record VersionPair(Version elasticsearch, Version lucene) implements Comparable<VersionPair> {
 
         @Override
-        public int compareTo(@NotNull VersionPair o) {
+        public int compareTo(VersionPair o) {
             // For ordering purposes, sort by Elasticsearch version
             return this.elasticsearch.compareTo(o.elasticsearch);
         }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java

@@ -187,7 +187,7 @@ public void execute(Task t) {
                     SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class);
                     FileCollection mainRuntime = sourceSets.getByName(SourceSet.MAIN_SOURCE_SET_NAME).getRuntimeClasspath();
                     // Add any "shadow" dependencies. These are dependencies that are *not* bundled into the shadow JAR
-                    Configuration shadowConfig = project.getConfigurations().getByName(ShadowBasePlugin.getCONFIGURATION_NAME());
+                    Configuration shadowConfig = project.getConfigurations().getByName(ShadowBasePlugin.CONFIGURATION_NAME);
                     // Add the shadow JAR artifact itself
                     FileCollection shadowJar = project.files(project.getTasks().named("shadowJar"));
                     FileCollection testRuntime = sourceSets.getByName(SourceSet.TEST_SOURCE_SET_NAME).getRuntimeClasspath();

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/RestTestBasePlugin.java

@@ -21,7 +21,6 @@
 import org.gradle.api.Project;
 import org.gradle.api.plugins.JavaBasePlugin;
 import org.gradle.api.provider.ProviderFactory;
-import org.jetbrains.annotations.Nullable;
 
 import javax.inject.Inject;
 
@@ -78,7 +77,6 @@ public void apply(Project project) {
             .configureEach(t -> t.finalizedBy(project.getTasks().withType(FixtureStop.class)));
     }
 
-    @Nullable
     private String systemProperty(String propName) {
         return providerFactory.systemProperty(propName).forUseAtConfigurationTime().getOrNull();
     }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/skip/Skip.java

@@ -18,7 +18,6 @@
 import org.elasticsearch.gradle.internal.test.rest.transform.RestTestTransformByParentObject;
 import org.elasticsearch.gradle.internal.test.rest.transform.RestTestTransformGlobalSetup;
 import org.gradle.api.tasks.Input;
-import org.jetbrains.annotations.Nullable;
 
 import java.util.Iterator;
 
@@ -43,7 +42,7 @@ public Skip(String skipReason) {
     }
 
     @Override
-    public ObjectNode transformSetup(@Nullable ObjectNode setupNodeParent) {
+    public ObjectNode transformSetup(ObjectNode setupNodeParent) {
         // only transform the global setup if there is no named test
         if (testName.isBlank()) {
             ArrayNode setupNode;

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/testfixtures/TestFixturesPlugin.java

@@ -108,11 +108,10 @@ public void execute(Task task) {
             maybeSkipTask(dockerSupport, buildFixture);
 
             ComposeExtension composeExtension = project.getExtensions().getByType(ComposeExtension.class);
-            composeExtension.setUseComposeFiles(Collections.singletonList(DOCKER_COMPOSE_YML));
-            composeExtension.setRemoveContainers(true);
-            composeExtension.setExecutable(
-                project.file("/usr/local/bin/docker-compose").exists() ? "/usr/local/bin/docker-compose" : "/usr/bin/docker-compose"
-            );
+            composeExtension.getUseComposeFiles().addAll(Collections.singletonList(DOCKER_COMPOSE_YML));
+            composeExtension.getRemoveContainers().set(true);
+            composeExtension.getExecutable()
+                .set(project.file("/usr/local/bin/docker-compose").exists() ? "/usr/local/bin/docker-compose" : "/usr/bin/docker-compose");
 
             tasks.named("composeUp").configure(t -> {
                 // Avoid running docker-compose tasks in parallel in CI due to some issues on certain Linux distributions

docs/changelog/81856.yaml

@@ -0,0 +1,5 @@
+pr: 81856
+summary: Allow scaling executors to reject tasks after shutdown
+area: Infra/Core
+type: enhancement
+issues: []

docs/changelog/82833.yaml

@@ -0,0 +1,5 @@
+pr: 82833
+summary: Adjust ILM policy for deprecation logs
+area: "Infra/Logging"
+type: enhancement
+issues: []

docs/reference/analysis/tokenfilters/hunspell-tokenfilter.asciidoc

@@ -24,13 +24,12 @@ See <<dictionary-stemmers>>.
 [[analysis-hunspell-tokenfilter-dictionary-config]]
 ==== Configure Hunspell dictionaries
 
-By default, Hunspell dictionaries are stored and detected on a dedicated
-hunspell directory on the filesystem: `<path.config>/hunspell`. Each dictionary
+Hunspell dictionaries are stored and detected on a dedicated
+`hunspell` directory on the filesystem: `<$ES_PATH_CONF>/hunspell`. Each dictionary
 is expected to have its own directory, named after its associated language and
 locale (e.g., `pt_BR`, `en_GB`). This dictionary directory is expected to hold a
 single `.aff` and one or more `.dic` files, all of which will automatically be
-picked up. For example, assuming the default `<path.config>/hunspell` path
-is used, the following directory layout will define the `en_US` dictionary:
+picked up. For example, the following directory layout will define the `en_US` dictionary:
 
 [source,txt]
 --------------------------------------------------
@@ -47,16 +46,16 @@ Each dictionary can be configured with one setting:
 `ignore_case`::
 (Static, Boolean)
 If true, dictionary matching will be case insensitive. Defaults to `false`.
-
++
 This setting can be configured globally in `elasticsearch.yml` using
 `indices.analysis.hunspell.dictionary.ignore_case`.
-
++
 To configure the setting for a specific locale, use the
 `indices.analysis.hunspell.dictionary.<locale>.ignore_case` setting (e.g., for
 the `en_US` (American English) locale, the setting is
 `indices.analysis.hunspell.dictionary.en_US.ignore_case`).
-
-It is also possible to add `settings.yml` file under the dictionary
++
+You can also add a `settings.yml` file under the dictionary
 directory which holds these settings. This overrides any other `ignore_case`
 settings defined in `elasticsearch.yml`.
 
@@ -67,7 +66,7 @@ The following analyze API request uses the `hunspell` filter to stem
 `the foxes jumping quickly` to `the fox jump quick`.
 
 The request specifies the `en_US` locale, meaning that the
-`.aff` and `.dic` files in the `<path.config>/hunspell/en_US` directory are used
+`.aff` and `.dic` files in the `<$ES_PATH_CONF>/hunspell/en_US` directory are used
 for the Hunspell dictionary.
 
 [source,console]
@@ -140,11 +139,8 @@ One or more `.dic` files (e.g, `en_US.dic, my_custom.dic`) to use for the
 Hunspell dictionary.
 +
 By default, the `hunspell` filter uses all `.dic` files in the
-`<path.config>/hunspell/<locale>` directory specified using the
-`lang`, `language`, or `locale` parameter. To use another directory, the
-directory's path must be registered using the
-<<indices-analysis-hunspell-dictionary-location,
-`indices.analysis.hunspell.dictionary.location`>> setting.
+`<$ES_PATH_CONF>/hunspell/<locale>` directory specified using the
+`lang`, `language`, or `locale` parameter.
 
 `dedup`::
 (Optional, Boolean)
@@ -194,7 +190,7 @@ uses a custom `hunspell` filter, `my_en_US_dict_stemmer`, to configure a new
 <<analysis-custom-analyzer,custom analyzer>>.
 
 The `my_en_US_dict_stemmer` filter uses a `locale` of `en_US`, meaning that the
-`.aff` and `.dic` files in the `<path.config>/hunspell/en_US` directory are
+`.aff` and `.dic` files in the `<$ES_PATH_CONF>/hunspell/en_US` directory are
 used. The filter also includes a `dedup` argument of `false`, meaning that
 duplicate tokens added from the dictionary are not removed from the filter's
 output.
@@ -236,12 +232,3 @@ If `true`, the loading of Hunspell dictionaries is deferred until a dictionary
 is used. If `false`, the dictionary directory is checked for dictionaries when
 the node starts, and any dictionaries are automatically loaded. Defaults to
 `false`.
-
-[[indices-analysis-hunspell-dictionary-location]]
-`indices.analysis.hunspell.dictionary.location`::
-(Static, string)
-Path to a Hunspell dictionary directory. This path must be absolute or
-relative to the `config` location.
-+
-By default, the `<path.config>/hunspell` directory is used, as described in
-<<analysis-hunspell-tokenfilter-dictionary-config>>.

docs/reference/cluster/state.asciidoc

@@ -4,7 +4,8 @@
 <titleabbrev>Cluster state</titleabbrev>
 ++++
 
-Returns metadata about the state of the cluster.
+Returns an internal representation of the cluster state for debugging or
+diagnostic purposes.
 
 [[cluster-state-api-request]]
 ==== {api-request-title}
@@ -20,44 +21,46 @@ Returns metadata about the state of the cluster.
 [[cluster-state-api-desc]]
 ==== {api-description-title}
 
-The cluster state API allows access to metadata representing the state of the
-whole cluster. This includes information such as
+The _cluster state_ is an internal data structure which keeps track of a
+variety of information needed by every node, including:
 
-* the set of nodes in the cluster
+* The identity and attributes of the other nodes in the cluster
 
-* all cluster-level settings
+* Cluster-wide settings
 
-* information about the indices in the cluster, including their mappings and
-  settings
+* Index metadata, including the mapping and settings for each index
 
-* the locations of all the shards in the cluster.
+* The location and status of every shard copy in the cluster
 
-NOTE: The response is an internal representation of the cluster state and its
-format may change from version to version. If possible, you should obtain any
-information from the cluster state using the other, more stable,
-<<cluster,cluster APIs>>.
+The elected master node ensures that every node in the cluster has a copy of
+the same cluster state. The cluster state API lets you retrieve a
+representation of this internal state for debugging or diagnostic purposes. You
+may need to consult the {es} source code to determine the precise meaning of
+the response.
 
-The response provides the cluster state itself, which can be filtered to only
-retrieve the parts of interest as described below.
+By default the cluster state API will route requests to the elected master node
+since this node is the authoritative source of cluster states. You can also
+retrieve the cluster state held on the node handling the API request by adding
+the query parameter `?local=true`.
 
-The cluster's `cluster_uuid` is also returned as part of the top-level response,
-in addition to the `metadata` section.
+{es} may need to expend significant effort to compute a response to this API in
+larger clusters, and the response may comprise a very large quantity of data.
+If you use this API repeatedly, your cluster may become unstable.
 
-NOTE: While the cluster is still forming, it is possible for the `cluster_uuid`
-      to be `_na_` as well as the cluster state's version to be `-1`.
-
-By default, the cluster state request is routed to the master node, to ensure
-that the latest cluster state is returned. For debugging purposes, you can
-retrieve the cluster state local to a particular node by adding `local=true` to
-the query string.
+[[cluster-state-api-unstable-warning]]
+WARNING: The response is a representation of an internal data structure. Its
+format is not subject to the same compatibility guarantees as other more stable
+APIs and may change from version to version. **Do not query this API using
+external monitoring tools.** Instead, obtain the information you require using
+other more stable <<cluster,cluster APIs>>.
 
 [[cluster-state-api-path-params]]
 ==== {api-path-parms-title}
 
-The cluster state contains information about all the indices in the cluster,
-including their mappings, as well as templates and other metadata. This means it
-can sometimes be quite large. To avoid the need to process all this information
-you can request only the part of the cluster state that you need:
+The cluster state can sometimes be very large, and {es} may consume significant
+resources while computing a response to this API. To reduce the size of the
+response, you can request only the part of the cluster state in which you are
+interested:
 
 `<metrics>`::
     (Optional, string) A comma-separated list of the following options:
@@ -70,7 +73,7 @@ you can request only the part of the cluster state that you need:
       Shows the `blocks` part of the response.
 
   `master_node`::
-      Shows the elected `master_node` part of the response.
+      Shows the `master_node` part of the response.
 
   `metadata`::
       Shows the `metadata` part of the response. If you supply a comma separated