Change Monitoring plugin cluster alerts to not install by default (elastic#79657)

Changes the default monitoring watches to not be installed by default any 
more. Tests have been updated to continue installing watches to make sure
the installation process continues to work.

Author: jbaiera

x-pack/plugin/monitoring/src/internalClusterTest/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporterIT.java

@@ -143,6 +143,7 @@ private Settings.Builder baseSettings() {
             .put("xpack.monitoring.exporters._http.ssl.truststore.password", "foobar") // ensure that ssl can be used by settings
             .put("xpack.monitoring.exporters._http.headers.ignored", "value") // ensure that headers can be used by settings
             .put("xpack.monitoring.exporters._http.host", getFormattedAddress(webServer))
+            .put("xpack.monitoring.exporters._http.cluster_alerts.management.enabled", true)
             .putList("xpack.monitoring.exporters._http.cluster_alerts.management.blacklist", clusterAlertBlacklist)
             .put("xpack.monitoring.exporters._http.auth.username", userName);
     }

x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/Exporter.java

@@ -87,7 +87,7 @@ public Iterator<Setting<?>> settings() {
      */
     public static final Setting.AffixSetting<Boolean> CLUSTER_ALERTS_MANAGEMENT_SETTING =
             Setting.affixKeySetting("xpack.monitoring.exporters.", "cluster_alerts.management.enabled",
-                    key -> Setting.boolSetting(key, true, Property.Dynamic, Property.NodeScope, Property.Deprecated), TYPE_DEPENDENCY);
+                    key -> Setting.boolSetting(key, false, Property.Dynamic, Property.NodeScope, Property.Deprecated), TYPE_DEPENDENCY);
     /**
      * Every {@code Exporter} allows users to explicitly disable specific cluster alerts.
      * <p>

x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporter.java

@@ -400,7 +400,7 @@ private void setupClusterAlertsTasks(ClusterState clusterState, boolean clusterS
                 logger.trace("watches shouldn't be setup, because state=[{}] and clusterStateChange=[{}]", state.get(), clusterStateChange);
             }
         } else {
-            logger.trace("watches can't be used, because xpack.watcher.enabled=[{}] and " +
+            logger.trace("watches will not be installed because xpack.watcher.enabled=[{}] and " +
                     "xpack.monitoring.exporters._local.cluster_alerts.management.enabled=[{}]",
                 XPackSettings.WATCHER_ENABLED.get(config.settings()),
                 CLUSTER_ALERTS_MANAGEMENT_SETTING.getConcreteSettingForNamespace(config.name()).get(config.settings()));

x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporterResourceTests.java

@@ -76,7 +76,9 @@ public class HttpExporterResourceTests extends AbstractPublishableHttpResourceTe
     private final List<String> templateNames = new ArrayList<>(EXPECTED_TEMPLATES);
     private final List<String> watchNames = new ArrayList<>(EXPECTED_WATCHES);
 
-    private final Settings exporterSettings = Settings.builder().build();
+    private final Settings exporterSettings = Settings.builder()
+        .put("xpack.monitoring.exporters._http.cluster_alerts.management.enabled", true)
+        .build();
 
     private final MultiHttpResource resources =
             HttpExporter.createResources(
@@ -363,8 +365,12 @@ public void testDeployClusterAlerts() {
         verifyDeleteWatches(EXPECTED_WATCHES);
         verifyNoMoreInteractions(client);
 
-        assertWarnings("[xpack.monitoring.migration.decommission_alerts] setting was deprecated in Elasticsearch and will be " +
-            "removed in a future release! See the breaking changes documentation for the next major version.");
+        assertWarnings(
+            "[xpack.monitoring.migration.decommission_alerts] setting was deprecated in Elasticsearch and will be " +
+                "removed in a future release! See the breaking changes documentation for the next major version.",
+            "[xpack.monitoring.exporters._http.cluster_alerts.management.enabled] setting was deprecated in Elasticsearch and " +
+                "will be removed in a future release! See the breaking changes documentation for the next major version."
+        );
     }
 
     public void testSuccessfulChecksOnElectedMasterNode() {
@@ -431,6 +437,9 @@ public void testSuccessfulChecksIfNotElectedMasterNode() {
         verifyVersionCheck();
         verifyGetTemplates(EXPECTED_TEMPLATES);
         verifyNoMoreInteractions(client);
+
+        assertWarnings("[xpack.monitoring.exporters._http.cluster_alerts.management.enabled] setting was deprecated in Elasticsearch " +
+            "and will be removed in a future release! See the breaking changes documentation for the next major version.");
     }
 
     private Exception failureGetException() {

x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporterTests.java

@@ -301,6 +301,7 @@ public void testExporterWithUnknownBlacklistedClusterAlerts() {
         final Settings.Builder builder = Settings.builder()
                 .put("xpack.monitoring.exporters._http.type", HttpExporter.TYPE)
                 .put("xpack.monitoring.exporters._http.host", "http://localhost:9200")
+                .put("xpack.monitoring.exporters._http.cluster_alerts.management.enabled", true)
                 .putList("xpack.monitoring.exporters._http.cluster_alerts.management.blacklist", blacklist);
 
         final Config config = createConfig(builder.build());
@@ -312,8 +313,12 @@ public void testExporterWithUnknownBlacklistedClusterAlerts() {
         assertThat(exception.getMessage(),
                    equalTo("[xpack.monitoring.exporters._http.cluster_alerts.management.blacklist] contains unrecognized Cluster " +
                            "Alert IDs [does_not_exist]"));
-        assertWarnings("[xpack.monitoring.exporters._http.cluster_alerts.management.blacklist] setting was deprecated in Elasticsearch" +
-            " and will be removed in a future release! See the breaking changes documentation for the next major version.");
+        assertWarnings(
+            "[xpack.monitoring.exporters._http.cluster_alerts.management.blacklist] setting was deprecated in Elasticsearch" +
+                " and will be removed in a future release! See the breaking changes documentation for the next major version.",
+            "[xpack.monitoring.exporters._http.cluster_alerts.management.enabled] setting was deprecated in Elasticsearch" +
+                " and will be removed in a future release! See the breaking changes documentation for the next major version."
+        );
     }
 
     public void testExporterWithHostOnly() throws Exception {
@@ -471,8 +476,8 @@ public void testCreateResources() {
                 .put("xpack.monitoring.exporters._http.type", "http");
         List<String> warningsExpected = new ArrayList<>();
 
-        if (clusterAlertManagement == false) {
-            builder.put("xpack.monitoring.exporters._http.cluster_alerts.management.enabled", false);
+        if (clusterAlertManagement) {
+            builder.put("xpack.monitoring.exporters._http.cluster_alerts.management.enabled", true);
             warningsExpected.add("[xpack.monitoring.exporters._http.cluster_alerts.management.enabled] setting was deprecated in " +
                 "Elasticsearch and will be removed in a future release! See the breaking changes documentation for the next major " +
                 "version.");

x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterResourceIntegTests.java

@@ -88,7 +88,9 @@ public void testRemoveWhenResourcesShouldBeRemoved() throws Exception {
         waitNoPendingTasksOnAll();
 
         Settings exporterSettings = Settings.builder().put(localExporterSettings())
-            .put("xpack.monitoring.migration.decommission_alerts", true).build();
+            .put("xpack.monitoring.migration.decommission_alerts", true)
+            .put("xpack.monitoring.exporters.decommission_local.cluster_alerts.management.enabled", true)
+            .build();
 
         createResources("decommission_local", exporterSettings);
         waitNoPendingTasksOnAll();