Flip node shutdown feature flag to default to true on snapshot builds (elastic#75962)

* Flip node shutdown feature flag to default to true on snapshot builds

It previously defaulted to false. The setting can still only be set to 'true' on a
non-release (snapshot) build of Elasticsearch.

Relates to elastic#70338

* Handle case where operator privileges are enabled

Author: dakrone

build-tools-internal/src/main/groovy/elasticsearch.run.gradle

@@ -27,9 +27,6 @@ testClusters {
         throw new IllegalArgumentException("Unsupported self-generated license type: [" + licenseType + "[basic] or [trial].")
       }
       setting 'xpack.security.enabled', 'true'
-      if (VersionProperties.elasticsearch.toString().endsWith('-SNAPSHOT')) {
-        setting 'es.shutdown_feature_flag_enabled', 'true'
-      }
       keystore 'bootstrap.password', 'password'
       user username: 'elastic-admin', password: 'elastic-password', role: 'superuser'
     }

docs/build.gradle

@@ -62,9 +62,6 @@ testClusters.matching { it.name == "integTest"}.configureEach {
     setting 'xpack.license.self_generated.type', 'trial'
     setting 'indices.lifecycle.history_index_enabled', 'false'
     setting 'ingest.geoip.downloader.enabled', 'false'
-    if (VersionProperties.elasticsearch.toString().endsWith('-SNAPSHOT')) {
-      setting 'es.shutdown_feature_flag_enabled', 'true'
-    }
     systemProperty 'es.geoip_v2_feature_flag_enabled', 'true'
     keystorePassword 'keystore-password'
   }

test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java

@@ -762,7 +762,7 @@ private void wipeCluster() throws Exception {
      * If any nodes are registered for shutdown, removes their metadata.
      */
     @SuppressWarnings("unchecked")
-    private static void deleteAllNodeShutdownMetadata() throws IOException {
+    protected void deleteAllNodeShutdownMetadata() throws IOException {
         Request getShutdownStatus = new Request("GET", "_nodes/shutdown");
         Map<String, Object> statusResponse = responseAsMap(adminClient().performRequest(getShutdownStatus));
         if (statusResponse.containsKey("_nodes") && statusResponse.containsKey("cluster_name")) {

x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/OperatorPrivilegesIT.java

@@ -27,6 +27,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.stream.Collectors;
 
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
@@ -42,6 +43,29 @@ protected Settings restClientSettings() {
         return Settings.builder().put(ThreadContext.PREFIX + ".Authorization", token).build();
     }
 
+    @SuppressWarnings("unchecked")
+    @Override
+    protected void deleteAllNodeShutdownMetadata() throws IOException {
+        Request getShutdownStatus = new Request("GET", "_nodes/shutdown");
+        getShutdownStatus.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("Authorization", OPERATOR_AUTH_HEADER));
+        Map<String, Object> statusResponse = responseAsMap(adminClient().performRequest(getShutdownStatus));
+        if (statusResponse.containsKey("_nodes") && statusResponse.containsKey("cluster_name")) {
+            // If the response contains these two keys, the feature flag isn't enabled on this cluster, so skip out now.
+            // We can't check the system property directly because it only gets set for the cluster under test's JVM, not for the test
+            // runner's JVM.
+            return;
+        }
+        List<Map<String, Object>> nodesArray = (List<Map<String, Object>>) statusResponse.get("nodes");
+        List<String> nodeIds = nodesArray.stream()
+            .map(nodeShutdownMetadata -> (String) nodeShutdownMetadata.get("node_id"))
+            .collect(Collectors.toUnmodifiableList());
+        for (String nodeId : nodeIds) {
+            Request deleteRequest = new Request("DELETE", "_nodes/" + nodeId + "/shutdown");
+            deleteRequest.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("Authorization", OPERATOR_AUTH_HEADER));
+            assertOK(adminClient().performRequest(deleteRequest));
+        }
+    }
+
     public void testNonOperatorSuperuserWillFailToCallOperatorOnlyApiWhenOperatorPrivilegesIsEnabled() throws IOException {
         final Request postVotingConfigExclusionsRequest = new Request("POST", "_cluster/voting_config_exclusions?node_names=foo");
         final ResponseException responseException = expectThrows(

x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/resources/roles.yml

@@ -4,3 +4,4 @@ limited_operator:
     - "monitor"
     - "cluster:admin/settings/update"
     - "cluster:admin/snapshot/restore"
+    - "cluster:admin/shutdown/*"

x-pack/plugin/shutdown/build.gradle

@@ -22,9 +22,6 @@ testClusters.all {
   testDistribution = 'default'
   setting 'xpack.security.enabled', 'true'
   setting 'xpack.license.self_generated.type', 'trial'
-  if (VersionProperties.elasticsearch.toString().endsWith('-SNAPSHOT')) {
-    setting 'es.shutdown_feature_flag_enabled', 'true'
-  }
   keystore 'bootstrap.password', 'x-pack-test-password'
   user username: "x_pack_rest_user", password: "x-pack-test-password"
 }

x-pack/plugin/shutdown/qa/multi-node/build.gradle

@@ -18,9 +18,6 @@ testClusters.all {
   testDistribution = 'DEFAULT'
   numberOfNodes = 4
 
-  if (VersionProperties.elasticsearch.toString().endsWith('-SNAPSHOT')) {
-    setting 'es.shutdown_feature_flag_enabled', 'true'
-  }
   setting 'xpack.security.enabled', 'true'
   user username: clusterCredentials.username, password: clusterCredentials.password, role: 'superuser'
 }

x-pack/plugin/shutdown/src/main/java/org/elasticsearch/xpack/shutdown/ShutdownPlugin.java

@@ -32,10 +32,21 @@ public class ShutdownPlugin extends Plugin implements ActionPlugin {
     public static final String SHUTDOWN_FEATURE_ENABLED_FLAG = "es.shutdown_feature_flag_enabled";
     public static final Setting<Boolean> SHUTDOWN_FEATURE_ENABLED_FLAG_SETTING = Setting.boolSetting(
         SHUTDOWN_FEATURE_ENABLED_FLAG,
-        false,
-        enabled -> {
-            if (enabled != null && enabled && Build.CURRENT.isSnapshot() == false) {
-                throw new IllegalArgumentException("shutdown plugin may not be enabled on a non-snapshot build");
+        (settings) -> {
+            final String enabled = settings.get(SHUTDOWN_FEATURE_ENABLED_FLAG);
+            // Enabled by default on snapshot builds, disabled on release builds
+            if (Build.CURRENT.isSnapshot()) {
+                if (enabled != null && enabled.equalsIgnoreCase("false")) {
+                    return "false";
+                } else {
+                    return "true";
+                }
+            } else {
+                if (enabled != null && enabled.equalsIgnoreCase("true")) {
+                    throw new IllegalArgumentException("shutdown plugin may not be enabled on a non-snapshot build");
+                } else {
+                    return "false";
+                }
             }
         },
         Setting.Property.NodeScope