Merge branch 'master' into machine-dependent-heap

Author: elasticmachine

.editorconfig

@@ -13,9 +13,11 @@ indent_size = 2
 
 [*.groovy]
 indent_size = 4
+max_line_length = 140
 
 [*.java]
 indent_size = 4
+max_line_length = 140
 
 [*.json]
 indent_size = 2

CONTRIBUTING.md

@@ -164,6 +164,47 @@ You can import the Elasticsearch project into IntelliJ IDEA via:
  - In the subsequent dialog navigate to the root `build.gradle` file
  - In the subsequent dialog select **Open as Project**
 
+#### Checkstyle
+
+If you have the [Checkstyle] plugin installed, you can configure IntelliJ to
+check the Elasticsearch code. However, the Checkstyle configuration file does
+not work by default with the IntelliJ plugin, so instead an IDE-specific config
+file is generated automatically after IntelliJ finishes syncing.
+
+   1. Open **Preferences > Tools > Checkstyle**
+   2. Change the "Scan Scope" to "Only Java sources (including tests)"
+   3. Check the "+" under "Configuration file"
+   4. Set "Description" to "Elasticsearch" (or whatever you want)
+   5. Select "Use a local Checkstyle file"
+   6. For the "File", navigate to `build/checkstyle_ide.xml`
+   7. Tick "Store relative to project location"
+   8. Click "Next"
+   9. The Checkstyle config file contains the variable `config_loc`, and
+      IntelliJ will ask for a value. Fill in `buildSrc/src/main/resources`.
+      This allows the config file to reference the exclusions file in that directory.
+   10. Click "Next", then "Finish".
+   11. Click the box next to the new configuration to make it "Active". Without doing this,
+       you'll have to explicitly choose the "Elasticsearch" configuration in the Checkstyle
+       tool window and run the check manually. You can still do this with an active config.
+   12. Click "OK" to apply the new preferences
+
+#### Formatting
+
+We are in the process of migrating towards automatic formatting Java file
+using [spotless], backed by the Eclipse formatter. If you have the [Eclipse
+Code Formatter] installed, you can apply formatting directly in IntelliJ.
+
+   1. Open **Preferences > Other Settings > Eclipse Code Formatter**
+   2. Click "Use the Eclipse Code Formatter"
+   3. Under "Eclipse formatter config", select "Eclipse workspace/project
+      folder or config file"
+   4. Click "Browse", and navigate to the file `buildSrc/formatterConfig.xml`
+   5. Click "OK"
+
+Note that only some sub-projects in the Elasticsearch project are currently
+fully-formatted. You can see a list of project that **are not**
+automatically formatted in [gradle/formatting.gradle](gradle/formatting.gradle).
+
 ### Importing the project into Eclipse
 
 Elasticsearch builds using Gradle and Java 14. When importing into Eclipse you
@@ -699,3 +740,6 @@ non-documentation contribution. This is mentioned above, but it is worth
 repeating in this section because it has come up in this context.
 
 [intellij]: https://blog.jetbrains.com/idea/2017/07/intellij-idea-2017-2-is-here-smart-sleek-and-snappy/
+[Checkstyle]: https://plugins.jetbrains.com/plugin/1065-checkstyle-idea
+[spotless]: https://github.com/diffplug/spotless
+[Eclipse Code Formatter]: https://plugins.jetbrains.com/plugin/6546-eclipse-code-formatter

build.gradle

@@ -287,23 +287,17 @@ gradle.projectsEvaluated {
       // :test:framework:test cannot run before and after :server:test
       return
     }
-    tasks.matching {it.name.equals('integTest')}.configureEach {integTestTask ->
+    tasks.matching { it.name.equals('integTest')}.configureEach {integTestTask ->
       integTestTask.mustRunAfter tasks.matching { it.name.equals("test") }
     }
 
     configurations.matching { it.canBeResolved }.all { Configuration configuration ->
       dependencies.matching { it instanceof ProjectDependency }.all { ProjectDependency dep ->
         Project upstreamProject = dep.dependencyProject
-        if (upstreamProject != null) {
-          if (project.path == upstreamProject.path) {
-            // TODO: distribution integ tests depend on themselves (!), fix that
-            return
-          }
+        if (project.path != upstreamProject?.path) {
           for (String taskName : ['test', 'integTest']) {
-            Task task = project.tasks.findByName(taskName)
-            Task upstreamTask = upstreamProject.tasks.findByName(taskName)
-            if (task != null && upstreamTask != null) {
-              task.shouldRunAfter(upstreamTask)
+            project.tasks.matching { it.name == taskName }.configureEach {task ->
+              task.shouldRunAfter(upstreamProject.tasks.matching { upStreamTask -> upStreamTask.name == taskName })
             }
           }
         }
@@ -487,7 +481,7 @@ allprojects {
 subprojects {
   project.ext.disableTasks = { String... tasknames ->
     for (String taskname : tasknames) {
-      project.tasks.named(taskname).configure { onlyIf { false } }
+      project.tasks.named(taskname).configure { enabled = false }
     }
   }
 }

buildSrc/reaper/build.gradle

@@ -1,4 +1,4 @@
-jar {
+tasks.named("jar").configure {
   archiveFileName = "${project.name}.jar"
   manifest {
     attributes 'Main-Class': 'org.elasticsearch.gradle.reaper.Reaper'

buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/DocsTestPlugin.groovy

@@ -23,7 +23,9 @@ import org.elasticsearch.gradle.Version
 import org.elasticsearch.gradle.VersionProperties
 import org.gradle.api.Plugin
 import org.gradle.api.Project
-import org.gradle.api.Task
+import org.gradle.api.file.Directory
+import org.gradle.api.file.DirectoryProperty
+import org.gradle.api.provider.Provider
 import org.gradle.api.tasks.TaskProvider
 
 /**
@@ -72,8 +74,13 @@ class DocsTestPlugin implements Plugin<Project> {
             }
         }
 
-        project.tasks.register('buildRestTests', RestTestsFromSnippetsTask) {
+        Provider<Directory> restRootDir = project.getLayout().buildDirectory.dir("rest")
+        TaskProvider<RestTestsFromSnippetsTask> buildRestTests = project.tasks.register('buildRestTests', RestTestsFromSnippetsTask) {
             defaultSubstitutions = commonDefaultSubstitutions
+            testRoot.convention(restRootDir)
         }
+
+        // TODO: This effectively makes testRoot not customizable, which we don't do anyway atm
+        project.sourceSets.test.output.dir(restRootDir, builtBy: buildRestTests)
     }
 }

buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTask.groovy

@@ -22,10 +22,13 @@ package org.elasticsearch.gradle.doc
 import groovy.transform.PackageScope
 import org.elasticsearch.gradle.doc.SnippetsTask.Snippet
 import org.gradle.api.InvalidUserDataException
+import org.gradle.api.file.DirectoryProperty
 import org.gradle.api.tasks.Input
 import org.gradle.api.tasks.Internal
 import org.gradle.api.tasks.OutputDirectory
+import org.gradle.api.model.ObjectFactory
 
+import javax.inject.Inject;
 import java.nio.file.Files
 import java.nio.file.Path
 
@@ -54,19 +57,16 @@ class RestTestsFromSnippetsTask extends SnippetsTask {
 
     /**
      * Root directory of the tests being generated. To make rest tests happy
-     * we generate them in a testRoot() which is contained in this directory.
+     * we generate them in a testRoot which is contained in this directory.
      */
-    @OutputDirectory
-    File testRoot = project.file('build/rest')
+    private DirectoryProperty testRoot
 
     @Internal
     Set<String> names = new HashSet<>()
 
-    RestTestsFromSnippetsTask() {
-        project.afterEvaluate {
-            // Wait to set this so testRoot can be customized
-            project.sourceSets.test.output.dir(testRoot, builtBy: this)
-        }
+    @Inject
+    RestTestsFromSnippetsTask(ObjectFactory objectFactory) {
+        testRoot = objectFactory.directoryProperty()
         TestBuilder builder = new TestBuilder()
         doFirst { outputRoot().delete() }
         perSnippet builder.&handleSnippet
@@ -79,10 +79,14 @@ class RestTestsFromSnippetsTask extends SnippetsTask {
      * contained within testRoot.
      */
     File outputRoot() {
-        return new File(testRoot, '/rest-api-spec/test')
+        return new File(testRoot.get().asFile, '/rest-api-spec/test')
     }
 
-    /**
+    @OutputDirectory
+    DirectoryProperty getTestRoot() {
+        return testRoot
+    }
+/**
      * Is this snippet a candidate for conversion to `// CONSOLE`?
      */
     static isConsoleCandidate(Snippet snippet) {

buildSrc/src/main/java/org/elasticsearch/gradle/util/GradleUtils.java

@@ -32,6 +32,7 @@
 import org.gradle.api.services.BuildService;
 import org.gradle.api.services.BuildServiceRegistration;
 import org.gradle.api.services.BuildServiceRegistry;
+import org.gradle.api.specs.Spec;
 import org.gradle.api.tasks.SourceSet;
 import org.gradle.api.tasks.SourceSetContainer;
 import org.gradle.api.tasks.TaskContainer;
@@ -82,11 +83,7 @@ public static <T extends Task> void maybeConfigure(
         Class<? extends T> type,
         Action<? super T> config
     ) {
-        tasks.withType(type).configureEach(task -> {
-            if (task.getName().equals(name)) {
-                config.execute(task);
-            }
-        });
+        tasks.withType(type).matching((Spec<T>) t -> t.getName().equals(name)).configureEach(task -> { config.execute(task); });
     }
 
     public static TaskProvider<?> findByName(TaskContainer tasks, String name) {

client/rest-high-level/build.gradle

@@ -79,7 +79,7 @@ tasks.named("integTest").configure {
 }
 
 // Requires https://github.com/elastic/elasticsearch/pull/64403 to have this moved to task avoidance api.
-RestIntegTestTask asyncIntegTest = tasks.create("asyncIntegTest", RestIntegTestTask) {
+TaskProvider<RestIntegTestTask> asyncIntegTest = tasks.register("asyncIntegTest", RestIntegTestTask) {
     systemProperty 'tests.rest.async', 'true'
     systemProperty 'tests.rest.cluster.username', System.getProperty('tests.rest.cluster.username', 'test_user')
     systemProperty 'tests.rest.cluster.password', System.getProperty('tests.rest.cluster.password', 'test-password')

client/rest-high-level/src/main/java/org/elasticsearch/client/ml/job/config/CategorizationAnalyzerConfig.java

@@ -18,6 +18,7 @@
  */
 package org.elasticsearch.client.ml.job.config;
 
+import org.elasticsearch.action.admin.indices.analyze.AnalyzeAction;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.ToXContentFragment;
@@ -59,9 +60,9 @@
 public class CategorizationAnalyzerConfig implements ToXContentFragment {
 
     public static final ParseField CATEGORIZATION_ANALYZER = new ParseField("categorization_analyzer");
-    private static final ParseField TOKENIZER = RestAnalyzeAction.Fields.TOKENIZER;
-    private static final ParseField TOKEN_FILTERS = RestAnalyzeAction.Fields.TOKEN_FILTERS;
-    private static final ParseField CHAR_FILTERS = RestAnalyzeAction.Fields.CHAR_FILTERS;
+    private static final ParseField TOKENIZER = AnalyzeAction.Fields.TOKENIZER;
+    private static final ParseField TOKEN_FILTERS = AnalyzeAction.Fields.TOKEN_FILTERS;
+    private static final ParseField CHAR_FILTERS = AnalyzeAction.Fields.CHAR_FILTERS;
 
     /**
      * This method is only used in the unit tests - in production code this config is always parsed as a fragment.

client/test/build.gradle

@@ -45,7 +45,7 @@ tasks.named('forbiddenApisTest').configure {
 
 // JarHell is part of es server, which we don't want to pull in
 // TODO: Not anymore. Now in :libs:elasticsearch-core
-jarHell.enabled = false
+tasks.named("jarHell").configure { enabled = false }
 
 // TODO: should we have licenses for our test deps?
 tasks.named("dependencyLicenses").configure { enabled = false }