Merge branch 'master' into add-setting-to-delete-searchable-snapshot-on-index-deletion

Author: tlrx

.ci/bwcVersions

@@ -37,6 +37,7 @@ BWC_VERSION:
   - "7.13.1"
   - "7.13.2"
   - "7.13.3"
+  - "7.13.4"
   - "7.14.0"
   - "7.15.0"
   - "8.0.0"

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/test/rest/YamlRestCompatTestPluginFuncTest.groovy

@@ -250,7 +250,12 @@ class YamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTest {
           - is_false: "value_to_replace"
           - is_true: "value_not_to_replace"
           - is_false: "value_not_to_replace"
-
+        ---
+        "use cat with no header":
+          - do:
+              cat.indices:
+                {}
+          - match: {}
         """.stripIndent()
         when:
         def result = gradleRunner("yamlRestCompatTest").build()
@@ -337,6 +342,16 @@ class YamlRestCompatTestPluginFuncTest extends AbstractRestResourcesFuncTest {
         - is_false: "replaced_value"
         - is_true: "value_not_to_replace"
         - is_false: "value_not_to_replace"
+        ---
+        "use cat with no header":
+          - do:
+              cat.indices:
+                {}
+              allowed_warnings:
+                - "added allowed warning"
+              allowed_warnings_regex:
+                - "added allowed warning regex .* [0-9]"
+          - match: {}
         """.stripIndent()).readAll()
 
         expectedAll.eachWithIndex{ ObjectNode expected, int i ->

build-tools-internal/src/main/groovy/elasticsearch.fips.gradle

@@ -76,6 +76,7 @@ if (BuildParams.inFipsJvm) {
         }
       }
       project.tasks.withType(Test).configureEach { Test task ->
+        dependsOn 'fipsResources'
         task.systemProperty('javax.net.ssl.trustStorePassword', 'password')
         task.systemProperty('javax.net.ssl.keyStorePassword', 'password')
         task.systemProperty('javax.net.ssl.trustStoreType', 'BCFKS')

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/rest/compat/RestCompatTestTransformTask.java

@@ -53,6 +53,7 @@
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Iterator;
 import java.util.LinkedHashMap;
 import java.util.LinkedList;
 import java.util.List;
@@ -94,7 +95,18 @@ public RestCompatTestTransformTask(
         // always inject compat headers
         headers.put("Content-Type", "application/vnd.elasticsearch+json;compatible-with=" + compatibleVersion);
         headers.put("Accept", "application/vnd.elasticsearch+json;compatible-with=" + compatibleVersion);
-        transformations.add(new InjectHeaders(headers));
+        transformations.add(new InjectHeaders(headers, Set.of(RestCompatTestTransformTask::doesNotHaveCatOperation)));
+    }
+
+    private static boolean doesNotHaveCatOperation(ObjectNode doNodeValue) {
+        final Iterator<String> fieldNamesIterator = doNodeValue.fieldNames();
+        while (fieldNamesIterator.hasNext()) {
+            final String fieldName = fieldNamesIterator.next();
+            if (fieldName.startsWith("cat.")) {
+                return false;
+            }
+        }
+        return true;
     }
 
     /**

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/transform/headers/InjectHeaders.java

@@ -18,6 +18,9 @@
 import org.gradle.api.tasks.Internal;
 
 import java.util.Map;
+import java.util.Set;
+import java.util.function.BiConsumer;
+import java.util.function.Function;
 
 /**
  * A {@link RestTestTransform} that injects HTTP headers into a REST test. This includes adding the necessary values to the "do" section
@@ -28,25 +31,37 @@ public class InjectHeaders extends FeatureInjector implements RestTestTransformB
     private static JsonNodeFactory jsonNodeFactory = JsonNodeFactory.withExactBigDecimals(false);
 
     private final Map<String, String> headers;
+    private final Set<Function<ObjectNode, Boolean>> applyConditions;
 
     /**
      * @param headers The headers to inject
+     * @param applyConditions a set of conditions that has to be satisfied in order to apply headers
+     *                        If the Set is empty then headers are always applied.
      */
-    public InjectHeaders(Map<String, String> headers) {
+    public InjectHeaders(Map<String, String> headers, Set<Function<ObjectNode, Boolean>> applyConditions) {
         this.headers = headers;
+        this.applyConditions = applyConditions;
     }
 
     @Override
     public void transformTest(ObjectNode doNodeParent) {
         ObjectNode doNodeValue = (ObjectNode) doNodeParent.get(getKeyToFind());
-        ObjectNode headersNode = (ObjectNode) doNodeValue.get("headers");
-        if (headersNode == null) {
-            headersNode = new ObjectNode(jsonNodeFactory);
-        }
-        for (Map.Entry<String, String> entry : headers.entrySet()) {
-            headersNode.set(entry.getKey(), TextNode.valueOf(entry.getValue()));
+
+        if (shouldApplyHeaders(doNodeValue)) {
+            ObjectNode headersNode = (ObjectNode) doNodeValue.get("headers");
+            if (headersNode == null) {
+                headersNode = new ObjectNode(jsonNodeFactory);
+            }
+
+            for (Map.Entry<String, String> entry : headers.entrySet()) {
+                headersNode.set(entry.getKey(), TextNode.valueOf(entry.getValue()));
+            }
+            doNodeValue.set("headers", headersNode);
         }
-        doNodeValue.set("headers", headersNode);
+    }
+
+    private boolean shouldApplyHeaders(ObjectNode doNodeValue) {
+        return applyConditions.stream().allMatch(f -> f.apply(doNodeValue));
     }
 
     @Override

build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/TransformTests.java

@@ -34,6 +34,7 @@
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.LongAdder;
+import java.util.function.Consumer;
 import java.util.stream.Collectors;
 
 public abstract class TransformTests extends GradleUnitTestCase {
@@ -125,8 +126,8 @@ protected List<String> getKnownFeatures() {
 
     protected List<RestTestTransform<?>> getTransformations() {
         List<RestTestTransform<?>> transformations = new ArrayList<>();
-        transformations.add(new InjectHeaders(headers1));
-        transformations.add(new InjectHeaders(headers2));
+        transformations.add(new InjectHeaders(headers1, Collections.emptySet()));
+        transformations.add(new InjectHeaders(headers2, Collections.emptySet()));
         return transformations;
     }
 

build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/test/rest/transform/header/InjectHeaderTests.java

@@ -14,9 +14,12 @@
 import org.elasticsearch.gradle.internal.test.rest.transform.headers.InjectHeaders;
 import org.junit.Test;
 
+import java.util.ArrayList;
 import java.util.Collections;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
 public class InjectHeaderTests extends InjectFeatureTests {
 
@@ -57,14 +60,42 @@ public void testInjectHeadersWithPreExisting() throws Exception {
         validateBodyHasHeaders(transformedTests, headers);
     }
 
+
+    @Test
+    public void testNotInjectingHeaders() throws Exception {
+        String testName = "/rest/transform/header/with_operation_to_skip_adding_headers.yml";
+        List<ObjectNode> tests = getTests(testName);
+        validateSetupExist(tests);
+        validateBodyHasHeaders(tests, Map.of("foo", "bar"));
+
+        List<RestTestTransform<?>> transformations =
+            Collections.singletonList(new InjectHeaders(headers, Set.of(InjectHeaderTests::applyCondition)));
+        List<ObjectNode> transformedTests = transformTests(tests, transformations);
+        printTest(testName, transformedTests);
+        validateSetupAndTearDown(transformedTests);
+        validateBodyHasHeaders(tests, Map.of("foo", "bar"));
+        validateBodyHasHeaders(transformedTests, Map.of("foo", "bar"));
+    }
+
+    private static boolean applyCondition(ObjectNode doNodeValue) {
+        final Iterator<String> fieldNamesIterator = doNodeValue.fieldNames();
+        while (fieldNamesIterator.hasNext()) {
+            final String fieldName = fieldNamesIterator.next();
+            if (fieldName.startsWith("something_to_skip")) {
+                return false;
+            }
+        }
+        return true;
+    }
+
     @Override
     protected List<String> getKnownFeatures() {
         return Collections.singletonList("headers");
     }
 
     @Override
     protected List<RestTestTransform<?>> getTransformations() {
-        return Collections.singletonList(new InjectHeaders(headers));
+        return Collections.singletonList(new InjectHeaders(headers, Collections.emptySet()));
     }
 
     @Override

build-tools-internal/src/test/resources/rest/transform/header/with_operation_to_skip_adding_headers.yml

@@ -0,0 +1,13 @@
+---
+setup:
+  - skip:
+      features: headers
+---
+"Test without a setup":
+  - do:
+      headers:
+        foo: "bar"
+      something_to_skip:
+        id: "something"
+  - match: { acknowledged: true }
+

distribution/packages/src/common/scripts/postinst

@@ -23,7 +23,7 @@ case "$1" in
     configure)
 
         # If $1=configure and $2 is set, this is an upgrade
-        if [ -n $2 ]; then
+        if [ -n "$2" ]; then
             IS_UPGRADE=true
         fi
         PACKAGE=deb

distribution/tools/keystore-cli/src/test/java/org/elasticsearch/common/settings/KeyStoreWrapperTests.java

@@ -9,9 +9,9 @@
 package org.elasticsearch.common.settings;
 
 import org.apache.lucene.codecs.CodecUtil;
+import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.IOContext;
 import org.apache.lucene.store.IndexOutput;
-import org.apache.lucene.store.SimpleFSDirectory;
 import org.elasticsearch.common.Randomness;
 import org.elasticsearch.core.internal.io.IOUtils;
 import org.elasticsearch.env.Environment;
@@ -198,8 +198,10 @@ public void testUpgradeNoop() throws Exception {
     public void testFailWhenCannotConsumeSecretStream() throws Exception {
         assumeFalse("Cannot open unprotected keystore on FIPS JVM", inFipsJvm());
         Path configDir = env.configFile();
-        SimpleFSDirectory directory = new SimpleFSDirectory(configDir);
-        try (IndexOutput indexOutput = directory.createOutput("elasticsearch.keystore", IOContext.DEFAULT)) {
+        try (
+            Directory directory = newFSDirectory(configDir);
+            IndexOutput indexOutput = directory.createOutput("elasticsearch.keystore", IOContext.DEFAULT)
+        ) {
             CodecUtil.writeHeader(indexOutput, "elasticsearch.keystore", 3);
             indexOutput.writeByte((byte) 0); // No password
             SecureRandom random = Randomness.createSecure();
@@ -227,8 +229,10 @@ public void testFailWhenCannotConsumeSecretStream() throws Exception {
     public void testFailWhenCannotConsumeEncryptedBytesStream() throws Exception {
         assumeFalse("Cannot open unprotected keystore on FIPS JVM", inFipsJvm());
         Path configDir = env.configFile();
-        SimpleFSDirectory directory = new SimpleFSDirectory(configDir);
-        try (IndexOutput indexOutput = directory.createOutput("elasticsearch.keystore", IOContext.DEFAULT)) {
+        try (
+            Directory directory = newFSDirectory(configDir);
+            IndexOutput indexOutput = directory.createOutput("elasticsearch.keystore", IOContext.DEFAULT)
+        ) {
             CodecUtil.writeHeader(indexOutput, "elasticsearch.keystore", 3);
             indexOutput.writeByte((byte) 0); // No password
             SecureRandom random = Randomness.createSecure();
@@ -257,8 +261,10 @@ public void testFailWhenCannotConsumeEncryptedBytesStream() throws Exception {
     public void testFailWhenSecretStreamNotConsumed() throws Exception {
         assumeFalse("Cannot open unprotected keystore on FIPS JVM", inFipsJvm());
         Path configDir = env.configFile();
-        SimpleFSDirectory directory = new SimpleFSDirectory(configDir);
-        try (IndexOutput indexOutput = directory.createOutput("elasticsearch.keystore", IOContext.DEFAULT)) {
+        try (
+            Directory directory = newFSDirectory(configDir);
+            IndexOutput indexOutput = directory.createOutput("elasticsearch.keystore", IOContext.DEFAULT)
+        ) {
             CodecUtil.writeHeader(indexOutput, "elasticsearch.keystore", 3);
             indexOutput.writeByte((byte) 0); // No password
             SecureRandom random = Randomness.createSecure();
@@ -285,8 +291,10 @@ public void testFailWhenSecretStreamNotConsumed() throws Exception {
     public void testFailWhenEncryptedBytesStreamIsNotConsumed() throws Exception {
         assumeFalse("Cannot open unprotected keystore on FIPS JVM", inFipsJvm());
         Path configDir = env.configFile();
-        SimpleFSDirectory directory = new SimpleFSDirectory(configDir);
-        try (IndexOutput indexOutput = directory.createOutput("elasticsearch.keystore", IOContext.DEFAULT)) {
+        try (
+            Directory directory = newFSDirectory(configDir);
+            IndexOutput indexOutput = directory.createOutput("elasticsearch.keystore", IOContext.DEFAULT)
+        ) {
             CodecUtil.writeHeader(indexOutput, "elasticsearch.keystore", 3);
             indexOutput.writeByte((byte) 0); // No password
             SecureRandom random = Randomness.createSecure();
@@ -372,8 +380,10 @@ public void testIllegalSettingName() throws Exception {
     public void testBackcompatV1() throws Exception {
         assumeFalse("Can't run in a FIPS JVM as PBE is not available", inFipsJvm());
         Path configDir = env.configFile();
-        SimpleFSDirectory directory = new SimpleFSDirectory(configDir);
-        try (IndexOutput output = directory.createOutput("elasticsearch.keystore", IOContext.DEFAULT)) {
+        try (
+            Directory directory = newFSDirectory(configDir);
+            IndexOutput output = directory.createOutput("elasticsearch.keystore", IOContext.DEFAULT)
+        ) {
             CodecUtil.writeHeader(output, "elasticsearch.keystore", 1);
             output.writeByte((byte) 0); // hasPassword = false
             output.writeString("PKCS12");
@@ -403,10 +413,12 @@ public void testBackcompatV1() throws Exception {
     public void testBackcompatV2() throws Exception {
         assumeFalse("Can't run in a FIPS JVM as PBE is not available", inFipsJvm());
         Path configDir = env.configFile();
-        SimpleFSDirectory directory = new SimpleFSDirectory(configDir);
         byte[] fileBytes = new byte[20];
         random().nextBytes(fileBytes);
-        try (IndexOutput output = directory.createOutput("elasticsearch.keystore", IOContext.DEFAULT)) {
+        try (
+            Directory directory = newFSDirectory(configDir);
+            IndexOutput output = directory.createOutput("elasticsearch.keystore", IOContext.DEFAULT)
+        ) {
 
             CodecUtil.writeHeader(output, "elasticsearch.keystore", 2);
             output.writeByte((byte) 0); // hasPassword = false