[DOCS] Omit shard failures assertion for incompatible responses (#31430)

Filter out the assertion for _cat and _xpack/ml/datafeed APIs

Author: davidkyle

buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTask.groovy

@@ -27,7 +27,6 @@ import org.gradle.api.tasks.OutputDirectory
 
 import java.nio.file.Files
 import java.nio.file.Path
-import java.util.regex.Matcher
 
 /**
  * Generates REST tests for each snippet marked // TEST.
@@ -100,6 +99,14 @@ public class RestTestsFromSnippetsTask extends SnippetsTask {
         return snippet.language == 'js' || snippet.curl
     }
 
+    /**
+     * Certain requests should not have the shard failure check because the
+     * format of the response is incompatible i.e. it is not a JSON object.
+     */
+    static shouldAddShardFailureCheck(String path) {
+        return path.startsWith('_cat') == false &&  path.startsWith('_xpack/ml/datafeeds/') == false
+    }
+
     /**
      * Converts Kibana's block quoted strings into standard JSON. These
      * {@code """} delimited strings can be embedded in CONSOLE and can
@@ -306,13 +313,11 @@ public class RestTestsFromSnippetsTask extends SnippetsTask {
              * no shard succeeds. But we need to fail the tests on all of these
              * because they mean invalid syntax or broken queries or something
              * else that we don't want to teach people to do. The REST test
-             * framework doesn't allow us to has assertions in the setup
-             * section so we have to skip it there. We also have to skip _cat
-             * actions because they don't return json so we can't is_false
-             * them. That is ok because they don't have this
-             * partial-success-is-success thing.
+             * framework doesn't allow us to have assertions in the setup
+             * section so we have to skip it there. We also omit the assertion
+             * from APIs that don't return a JSON object
              */
-            if (false == inSetup && false == path.startsWith('_cat')) {
+            if (false == inSetup && shouldAddShardFailureCheck(path)) {
                 current.println("  - is_false: _shards.failures")
             }
         }

buildSrc/src/test/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTaskTest.groovy

@@ -19,9 +19,7 @@
 
 package org.elasticsearch.gradle.doc
 
-import org.elasticsearch.gradle.doc.SnippetsTask.Snippet
-import org.gradle.api.InvalidUserDataException
-
+import static org.elasticsearch.gradle.doc.RestTestsFromSnippetsTask.shouldAddShardFailureCheck
 import static org.elasticsearch.gradle.doc.RestTestsFromSnippetsTask.replaceBlockQuote
 
 class RestTestFromSnippetsTaskTest extends GroovyTestCase {
@@ -47,4 +45,10 @@ class RestTestFromSnippetsTaskTest extends GroovyTestCase {
         assertEquals("\"foo\": \"bort\\n baz\"",
             replaceBlockQuote("\"foo\": \"\"\"bort\n baz\"\"\""));
     }
+
+    void testIsDocWriteRequest() {
+        assertTrue(shouldAddShardFailureCheck("doc-index/_search"));
+        assertFalse(shouldAddShardFailureCheck("_cat"))
+        assertFalse(shouldAddShardFailureCheck("_xpack/ml/datafeeds/datafeed-id/_preview"));
+    }
 }