Merge branch 'master' into add_memory_override

Author: David Roberts

BUILDING.md

@@ -135,3 +135,99 @@ dependencies {
   }
 }
 ```
+
+## FAQ
+
+### How do I test a development version of a third party dependency?
+
+To test an unreleased development version of a third party dependency you have several options.
+
+#### How to use a maven based third party dependency via mavenlocal?
+
+1. Clone the third party repository locally
+2. Run `mvn install` to install copy into your `~/.m2/repository` folder.
+3. Add this to the root build script:
+
+```
+   allprojects {
+     repositories {
+         mavenLocal()
+     }
+   }
+   ```
+4. Update the version in your dependency declaration accordingly (likely a snapshot version)
+5. Run the gradle build as needed
+
+#### How to use a maven built based third party dependency with jitpack repository?
+
+https://jitpack.io is an adhoc repository that supports building maven projects transparently in the background when 
+resolving unreleased snapshots from a github repository. This approach also works as temporally solution
+and is compliant with our CI builds. 
+
+1. Add the JitPack repository to the root build file:
+
+```
+   allprojects {
+     repositories {
+        maven { url "https://jitpack.io" }
+     }
+   }
+```
+2. Add the dependency in the following format
+```
+dependencies {
+  implementation 'com.github.User:Repo:Tag'
+}
+```
+
+As version you could also use a certain short commit hash or `master-SNAPSHOT`.
+In addition to snapshot builds JitPack supports building Pull Requests. Simply use PR<NR>-SNAPSHOT as the version.
+
+3. Run the gradle build as needed. Keep in mind the initial resolution might take a bit longer as this needs to be built 
+by JitPack in the background before we can resolve the adhoc built dependency. 
+
+---
+
+**NOTE**
+
+You should only use that approach locally or on a developer branch for for production dependencies as we do
+not want to ship unreleased libraries into our releases.
+---
+
+#### How to use a custom third party artifact?
+
+For third party libraries that are not built with maven (e.g. ant) or provided as a plain jar artifact we can leverage
+a flat directory repository that resolves artifacts from a flat directory on your filesystem.
+
+1. Put the jar artifact with the format `artifactName-version.jar` into a directory named `localRepo` (you have to create this manually)
+2. Declare a flatDir repository in your root build.gradle file
+
+```
+allprojects { 
+  repositories {
+      flatDir {
+          dirs 'localRepo'
+      }
+  }
+}
+```
+
+3. Update the dependency declaration of the artifact in question to match the custom build version. For a file named e.g. `jmxri-1.2.1.jar` the 
+  dependency definition would be `:jmxri:1.2.1` as it comes with no group information:
+  
+  ```
+  dependencies {
+      implementation ':jmxri:1.2.1'
+  }
+  ```
+4. Run the gradle build as needed.
+
+---
+**NOTE**
+
+As Gradle prefers to use modules whose descriptor has been created from real meta-data rather than being generated,
+flat directory repositories cannot be used to override artifacts with real meta-data from other repositories declared in the build.
+For example, if Gradle finds only `jmxri-1.2.1.jar` in a flat directory repository, but `jmxri-1.2.1.pom` in another repository
+that supports meta-data, it will use the second repository to provide the module.
+Therefore it is recommended to declare a version that is not resolveable from public repositories we use (e.g. maven central)
+---

benchmarks/src/main/java/org/elasticsearch/benchmark/time/DateFormatterBenchmark.java

@@ -22,7 +22,6 @@
 import java.io.File;
 import java.io.IOException;
 import java.util.Properties;
-import java.util.function.Function;
 
 abstract class VersionPropertiesBuildService implements BuildService<VersionPropertiesBuildService.Params>, AutoCloseable {
 
@@ -42,7 +41,7 @@ public VersionPropertiesBuildService(ProviderFactory providerFactory) {
 
     private JavaVersion resolveMinimumJavaVersion(File infoPath) {
         final JavaVersion minimumJavaVersion;
-        File minimumJavaInfoSource = new File(infoPath, "src/main/resources/minimumCompilerVersion");
+        File minimumJavaInfoSource = new File(infoPath, "src/main/resources/minimumRuntimeVersion");
         try {
             String versionString = FileUtils.readFileToString(minimumJavaInfoSource);
             minimumJavaVersion = JavaVersion.toVersion(versionString);

build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/VersionPropertiesBuildService.java

@@ -82,7 +82,8 @@ class YamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTest {
         String wrongTest = "wrong_version.yml"
         String additionalTest = "additional_test.yml"
         setupRestResources([wrongApi], [wrongTest]) //setups up resources for current version, which should not be used for this test
-        addRestTestsToProject([additionalTest], "yamlRestCompatTest")
+        String sourceSetName = "yamlRestTestV" + compatibleVersion + "Compat"
+        addRestTestsToProject([additionalTest], sourceSetName)
         //intentionally adding to yamlRestTest source set since the .classes are copied from there
         file("src/yamlRestTest/java/MockIT.java") << "import org.junit.Test;class MockIT { @Test public void doNothing() { }}"
 
@@ -107,17 +108,17 @@ class YamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTest {
         file("/build/${testIntermediateDir}/original/rest-api-spec/test/" + test).exists()
         file("/build/${testIntermediateDir}/transformed/rest-api-spec/test/" + test).exists()
         file("/build/${testIntermediateDir}/transformed/rest-api-spec/test/" + test).text.contains("headers") //transformation adds this
-        file("/build/resources/yamlRestCompatTest/rest-api-spec/test/" + additionalTest).exists()
+        file("/build/resources/${sourceSetName}/rest-api-spec/test/" + additionalTest).exists()
 
         //additionalTest is not copied from the prior version, and thus not in the intermediate directory, nor transformed
-        file("/build/resources/yamlRestCompatTest/" + testIntermediateDir + "/rest-api-spec/test/" + additionalTest).exists() == false
-        file("/build/resources/yamlRestCompatTest/rest-api-spec/test/" + additionalTest).text.contains("headers") == false
+        file("/build/resources/${sourceSetName}/" + testIntermediateDir + "/rest-api-spec/test/" + additionalTest).exists() == false
+        file("/build/resources/${sourceSetName}/rest-api-spec/test/" + additionalTest).text.contains("headers") == false
 
         file("/build/classes/java/yamlRestTest/MockIT.class").exists() //The "standard" runner is used to execute the compat test
 
-        file("/build/resources/yamlRestCompatTest/rest-api-spec/api/" + wrongApi).exists() == false
-        file("/build/resources/yamlRestCompatTest/" + testIntermediateDir + "/rest-api-spec/test/" + wrongTest).exists() == false
-        file("/build/resources/yamlRestCompatTest/rest-api-spec/test/" + wrongTest).exists() == false
+        file("/build/resources/${sourceSetName}/rest-api-spec/api/" + wrongApi).exists() == false
+        file("/build/resources/${sourceSetName}/" + testIntermediateDir + "/rest-api-spec/test/" + wrongTest).exists() == false
+        file("/build/resources/${sourceSetName}/rest-api-spec/test/" + wrongTest).exists() == false
 
         result.task(':copyRestApiSpecsTask').outcome == TaskOutcome.NO_SOURCE
         result.task(':copyYamlTestsTask').outcome == TaskOutcome.NO_SOURCE

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/YamlRestCompatTestPluginFuncTest.groovy

@@ -8,17 +8,17 @@
 
 package org.elasticsearch.gradle.internal.rest.compat;
 
-import org.elasticsearch.gradle.internal.ElasticsearchJavaBasePlugin;
 import org.elasticsearch.gradle.Version;
 import org.elasticsearch.gradle.VersionProperties;
+import org.elasticsearch.gradle.internal.ElasticsearchJavaBasePlugin;
 import org.elasticsearch.gradle.internal.test.RestIntegTestTask;
 import org.elasticsearch.gradle.internal.test.RestTestBasePlugin;
 import org.elasticsearch.gradle.internal.test.rest.CopyRestApiTask;
 import org.elasticsearch.gradle.internal.test.rest.CopyRestTestsTask;
+import org.elasticsearch.gradle.internal.test.rest.InternalYamlRestTestPlugin;
 import org.elasticsearch.gradle.internal.test.rest.RestResourcesExtension;
 import org.elasticsearch.gradle.internal.test.rest.RestResourcesPlugin;
 import org.elasticsearch.gradle.internal.test.rest.RestTestUtil;
-import org.elasticsearch.gradle.internal.test.rest.InternalYamlRestTestPlugin;
 import org.elasticsearch.gradle.testclusters.TestClustersPlugin;
 import org.elasticsearch.gradle.util.GradleUtils;
 import org.gradle.api.Plugin;
@@ -46,21 +46,22 @@
  * Apply this plugin to run the YAML based REST tests from a prior major version against this version's cluster.
  */
 public class YamlRestCompatTestPlugin implements Plugin<Project> {
+    public static final String BWC_MINOR_CONFIG_NAME = "bwcMinor";
     private static final String REST_COMPAT_CHECK_TASK_NAME = "checkRestCompat";
     private static final String COMPATIBILITY_APIS_CONFIGURATION = "restCompatSpecs";
     private static final String COMPATIBILITY_TESTS_CONFIGURATION = "restCompatTests";
-    private static final String SOURCE_SET_NAME = "yamlRestCompatTest";
     private static final Path RELATIVE_API_PATH = Path.of("rest-api-spec/api");
     private static final Path RELATIVE_TEST_PATH = Path.of("rest-api-spec/test");
     private static final Path RELATIVE_REST_API_RESOURCES = Path.of("rest-api-spec/src/main/resources");
     private static final Path RELATIVE_REST_XPACK_RESOURCES = Path.of("x-pack/plugin/src/test/resources");
     private static final Path RELATIVE_REST_PROJECT_RESOURCES = Path.of("src/yamlRestTest/resources");
-    public static final String BWC_MINOR_CONFIG_NAME = "bwcMinor";
+    private static final int COMPATIBLE_VERSION = Version.fromString(VersionProperties.getVersions().get("elasticsearch")).getMajor() - 1;
+    private static final String SOURCE_SET_NAME = "yamlRestTestV" + COMPATIBLE_VERSION + "Compat";
 
     @Override
     public void apply(Project project) {
-        final int compatibleVersion = Version.fromString(VersionProperties.getVersions().get("elasticsearch")).getMajor() - 1;
-        final Path compatRestResourcesDir = Path.of("restResources").resolve("v" + compatibleVersion);
+
+        final Path compatRestResourcesDir = Path.of("restResources").resolve("v" + COMPATIBLE_VERSION);
         final Path compatSpecsDir = compatRestResourcesDir.resolve("yamlSpecs");
         final Path compatTestsDir = compatRestResourcesDir.resolve("yamlTests");
 
@@ -160,7 +161,7 @@ public void apply(Project project) {
 
         // transform the copied tests task
         TaskProvider<RestCompatTestTransformTask> transformCompatTestTask = project.getTasks()
-            .register("yamlRestTestV" + compatibleVersion + "CompatTransform", RestCompatTestTransformTask.class, task -> {
+            .register("yamlRestTestV" + COMPATIBLE_VERSION + "CompatTransform", RestCompatTestTransformTask.class, task -> {
                 task.getSourceDirectory().set(copyCompatYamlTestTask.flatMap(CopyRestTestsTask::getOutputResourceDir));
                 task.getOutputDirectory()
                     .set(project.getLayout().getBuildDirectory().dir(compatTestsDir.resolve("transformed").toString()));
@@ -191,7 +192,7 @@ public void apply(Project project) {
             .named(RestResourcesPlugin.COPY_YAML_TESTS_TASK)
             .flatMap(CopyRestTestsTask::getOutputResourceDir);
 
-        String testTaskName = "yamlRestTestV" + compatibleVersion + "CompatTest";
+        String testTaskName = "yamlRestTestV" + COMPATIBLE_VERSION + "CompatTest";
 
         // setup the test task
         Provider<RestIntegTestTask> yamlRestCompatTestTask = RestTestUtil.registerTestTask(project, yamlCompatTestSourceSet, testTaskName);
@@ -207,11 +208,13 @@ public void apply(Project project) {
                     .minus(project.files(originalYamlSpecsDir))
                     .minus(project.files(originalYamlTestsDir))
             );
+
             // run compatibility tests after "normal" tests
             testTask.mustRunAfter(project.getTasks().named(InternalYamlRestTestPlugin.SOURCE_SET_NAME));
             testTask.onlyIf(t -> isEnabled(project));
         });
 
+
         setupTestDependenciesDefaults(project, yamlCompatTestSourceSet);
 
         // setup IDE

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/rest/compat/YamlRestCompatTestPlugin.java

@@ -321,7 +321,12 @@ private void commonNodeConfig() {
         for (ElasticsearchNode node : nodes) {
             if (node.getTestDistribution().equals(TestDistribution.INTEG_TEST)) {
                 node.defaultConfig.put("xpack.security.enabled", "false");
+            } else {
+                if (node.getVersion().onOrAfter("8.0.0")) {
+                    node.defaultConfig.put("cluster.deprecation_indexing.enabled", "false");
+                }
             }
+
             // Can only configure master nodes if we have node names defined
             if (nodeNames != null) {
                 if (node.getVersion().onOrAfter("7.0.0")) {

build-tools/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchCluster.java

@@ -103,7 +103,8 @@ public class ElasticsearchNode implements TestClusterConfiguration {
     private static final TimeUnit ADDITIONAL_CONFIG_TIMEOUT_UNIT = TimeUnit.SECONDS;
     private static final List<String> OVERRIDABLE_SETTINGS = Arrays.asList(
         "path.repo",
-        "discovery.seed_providers"
+        "discovery.seed_providers",
+        "cluster.deprecation_indexing.enabled"
 
     );
 

build-tools/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java

@@ -134,9 +134,9 @@ tasks.register("verifyVersions") {
  * after the backport of the backcompat code is complete.
  */
 
-boolean bwc_tests_enabled = false
+boolean bwc_tests_enabled = true
 // place a PR link here when committing bwc changes:
-String bwc_tests_disabled_issue = "https://github.com/elastic/elasticsearch/pull/77552"
+String bwc_tests_disabled_issue = ""
 /*
  * FIPS 140-2 behavior was fixed in 7.11.0. Before that there is no way to run elasticsearch in a
  * JVM that is properly configured to be in fips mode with BCFIPS. For now we need to disable

build.gradle

@@ -0,0 +1,51 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.client.transform;
+
+import org.elasticsearch.client.Validatable;
+
+import java.util.Objects;
+
+public class UpgradeTransformsRequest implements Validatable {
+
+    private Boolean dryRun;
+
+    public UpgradeTransformsRequest() {}
+
+    public Boolean isDryRun() {
+        return dryRun;
+    }
+
+    /**
+     * Whether to only check for an upgrade without taking action
+     *
+     * @param dryRun {@code true} will only check for upgrades
+     */
+    public void setDryRun(boolean dryRun) {
+        this.dryRun = dryRun;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(dryRun);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == null) {
+            return false;
+        }
+        if (getClass() != obj.getClass()) {
+            return false;
+        }
+        UpgradeTransformsRequest other = (UpgradeTransformsRequest) obj;
+        return Objects.equals(dryRun, other.dryRun);
+    }
+
+}