Remove more usages of .watches system index. (#65871)

Backporting #65835 to 7.x branch.

Make use of the new query watches api to simply list watches or
to delete all watches using the delete watch api.

Relates to #62501

Author: martijnvg

x-pack/docs/en/watcher/managing-watches.asciidoc

@@ -28,9 +28,8 @@ For example, the following returns the first 100 watches:
 
 [source,console]
 --------------------------------------------------
-GET .watches/_search
+GET /_watcher/_query/watches
 {
   "size" : 100
 }
 --------------------------------------------------
-// TEST[skip:deprecation warning]

x-pack/plugin/watcher/qa/common/src/main/java/org/elasticsearch/xpack/watcher/WatcherRestTestCase.java

@@ -15,6 +15,9 @@
 import org.junit.Assert;
 import org.junit.Before;
 
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
 import java.util.concurrent.TimeUnit;
 
 /**
@@ -70,16 +73,21 @@ public final void stopWatcher() throws Exception {
                     throw new AssertionError("unknown state[" + state + "]");
             }
         }, 60, TimeUnit.SECONDS);
+        deleteAllWatcherData();
+    }
+
+    public static void deleteAllWatcherData() throws IOException {
+        Request queryWatchesRequest = new Request("GET", "/_watcher/_query/watches");
+        ObjectPath response = ObjectPath.createFromResponse(ESRestTestCase.adminClient().performRequest(queryWatchesRequest));
 
-        Request deleteWatchesIndexRequest = new Request("DELETE", ".watches");
-        deleteWatchesIndexRequest.addParameter("ignore_unavailable", "true");
-        deleteWatchesIndexRequest.setOptions(
-            expectWarnings(
-                "this request accesses system indices: [.watches], but in a future major "
-                    + "version, direct access to system indices will be prevented by default"
-            )
-        );
-        ESRestTestCase.adminClient().performRequest(deleteWatchesIndexRequest);
+        int totalCount = response.evaluate("count");
+        List<Map<?, ?>> watches = response.evaluate("watches");
+        assert watches.size() == totalCount : "number of watches returned is unequal to the total number of watches";
+        for (Map<?, ?> watch : watches) {
+            String id = (String) watch.get("_id");
+            Request deleteWatchRequest = new Request("DELETE", "/_watcher/watch/" + id);
+            assertOK(ESRestTestCase.adminClient().performRequest(deleteWatchRequest));
+        }
 
         Request deleteWatchHistoryRequest = new Request("DELETE", ".watcher-history-*");
         deleteWatchHistoryRequest.addParameter("ignore_unavailable", "true");

x-pack/plugin/watcher/qa/common/src/main/java/org/elasticsearch/xpack/watcher/WatcherYamlSuiteTestCase.java

@@ -7,10 +7,7 @@
 
 import com.carrotsearch.randomizedtesting.annotations.Name;
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
-import org.elasticsearch.client.Request;
-import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.test.ESTestCase;
-import org.elasticsearch.test.rest.ESRestTestCase;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestResponse;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
@@ -19,11 +16,11 @@
 import org.junit.Assert;
 import org.junit.Before;
 
-import java.io.IOException;
 import java.util.concurrent.TimeUnit;
 
 import static java.util.Collections.emptyList;
 import static java.util.Collections.emptyMap;
+import static org.elasticsearch.xpack.watcher.WatcherRestTestCase.deleteAllWatcherData;
 
 /**
  * Parent test class for Watcher YAML based REST tests
@@ -63,7 +60,7 @@ public final void startWatcher() throws Exception {
                     int watcherCount = (int) response.evaluate("stats.0.watch_count");
                     if (watcherCount > 0) {
                         logger.info("expected 0 active watches, but got [{}], deleting watcher indices again", watcherCount);
-                        deleteWatcherIndices();
+                        deleteAllWatcherData();
                     }
                     // all good here, we are done
                     break;
@@ -100,24 +97,6 @@ public final void stopWatcher() throws Exception {
                     throw new AssertionError("unknown state[" + state + "]");
             }
         }, 60, TimeUnit.SECONDS);
-        deleteWatcherIndices();
-    }
-
-    private static void deleteWatcherIndices() throws IOException {
-        Request deleteWatchesIndexRequest = new Request("DELETE", ".watches");
-        deleteWatchesIndexRequest.addParameter("ignore_unavailable", "true");
-        final RequestOptions.Builder requestOptions = RequestOptions.DEFAULT.toBuilder();
-        requestOptions.setWarningsHandler(warnings -> {
-            final String expectedWaring = "this request accesses system indices: [.watches], but in a future major version, direct "
-                + "access to system indices will be prevented by default";
-            // There might not be a warning if the .watches index doesn't exist
-            return (warnings.isEmpty() || warnings.get(0).equals(expectedWaring)) == false;
-        });
-        deleteWatchesIndexRequest.setOptions(requestOptions);
-        ESRestTestCase.adminClient().performRequest(deleteWatchesIndexRequest);
-
-        Request deleteWatchHistoryRequest = new Request("DELETE", ".watcher-history-*");
-        deleteWatchHistoryRequest.addParameter("ignore_unavailable", "true");
-        ESRestTestCase.adminClient().performRequest(deleteWatchHistoryRequest);
+        deleteAllWatcherData();
     }
 }

x-pack/plugin/watcher/qa/with-monitoring/build.gradle

@@ -2,6 +2,7 @@ apply plugin: 'elasticsearch.java-rest-test'
 
 dependencies {
   javaRestTestImplementation project(':x-pack:qa')
+  javaRestTestImplementation project(path: ':x-pack:plugin:watcher:qa:common')
 }
 
 testClusters.all {

x-pack/plugin/watcher/qa/with-monitoring/src/javaRestTest/java/org/elasticsearch/smoketest/MonitoringWithWatcherRestIT.java

@@ -6,7 +6,6 @@
 package org.elasticsearch.smoketest;
 
 import org.elasticsearch.client.Request;
-import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.test.rest.ESRestTestCase;
@@ -16,6 +15,7 @@
 import java.io.IOException;
 
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
+import static org.elasticsearch.xpack.watcher.WatcherRestTestCase.deleteAllWatcherData;
 import static org.hamcrest.Matchers.is;
 
 public class MonitoringWithWatcherRestIT extends ESRestTestCase {
@@ -40,20 +40,7 @@ public void cleanExporters() throws Exception {
                     .nullField("xpack.monitoring.exporters.*")
                 .endObject().endObject()));
         adminClient().performRequest(cleanupSettingsRequest);
-        final Request deleteRequest = new Request("DELETE", "/.watch*");
-        final RequestOptions.Builder requestOptions = RequestOptions.DEFAULT.toBuilder();
-        requestOptions.setWarningsHandler(warnings -> {
-            if (warnings.size() != 1) {
-                return true;
-            }
-            // We don't know exactly which indices we're cleaning up in advance, so just accept all system index access warnings.
-            final String warning = warnings.get(0);
-            final boolean isSystemIndexWarning = warning.contains("this request accesses system indices")
-                && warning.contains("but in a future major version, direct access to system indices will be prevented by default");
-            return isSystemIndexWarning == false;
-        });
-        deleteRequest.setOptions(requestOptions);
-        adminClient().performRequest(deleteRequest);
+        deleteAllWatcherData();
     }
 
     @AwaitsFix( bugUrl = "https://github.com/elastic/elasticsearch/issues/59132" )