[7.x] Enhance the ingest node simulate verbose output (#60433) (#60678)

This commit enhances the verbose output for the
`_ingest/pipeline/_simulate?verbose` api. Specifically
this adds the following:
* the pipeline processor is now included in the output
* the conditional (if) and result is now included in the output iff it was defined
* a status field is always displayed. the possible values of status are
  * `success` - if the processor ran with out errors
  * `error` - if the processor ran but threw an error that was not ingored
  * `error_ignored` - if the processor ran but threw an error that was ingored
  * `skipped` - if the process did not run (currently only possible if the if condition evaluates to false)
  * `dropped` - if the the `drop` processor ran and dropped the document
* a `processor_type` field for the type of processor (e.g. set, rename, etc.)
* throw a better error if trying to simulate with a pipeline that does not exist

closes #56004

Author: jakelandis

docs/reference/ingest/apis/simulate-pipeline.asciidoc

@@ -342,84 +342,93 @@ The API returns the following response:
 [source,console-result]
 ----
 {
-   "docs": [
-      {
-         "processor_results": [
-            {
-               "doc": {
-                  "_id": "id",
-                  "_index": "index",
-                  "_type": "_doc",
-                  "_source": {
-                     "field2": "_value2",
-                     "foo": "bar"
-                  },
-                  "_ingest": {
-                     "timestamp": "2017-05-04T22:46:09.674Z",
-                     "pipeline": "_simulate_pipeline"
-                  }
-               }
+  "docs": [
+    {
+      "processor_results": [
+        {
+          "processor_type": "set",
+          "status": "success",
+          "doc": {
+            "_index": "index",
+            "_type": "_doc",
+            "_id": "id",
+            "_source": {
+              "field2": "_value2",
+              "foo": "bar"
             },
-            {
-               "doc": {
-                  "_id": "id",
-                  "_index": "index",
-                  "_type": "_doc",
-                  "_source": {
-                     "field3": "_value3",
-                     "field2": "_value2",
-                     "foo": "bar"
-                  },
-                  "_ingest": {
-                     "timestamp": "2017-05-04T22:46:09.675Z",
-                     "pipeline": "_simulate_pipeline"
-                  }
-               }
+            "_ingest": {
+              "pipeline": "_simulate_pipeline",
+              "timestamp": "2020-07-30T01:21:24.251836Z"
             }
-         ]
-      },
-      {
-         "processor_results": [
-            {
-               "doc": {
-                  "_id": "id",
-                  "_index": "index",
-                  "_type": "_doc",
-                  "_source": {
-                     "field2": "_value2",
-                     "foo": "rab"
-                  },
-                  "_ingest": {
-                     "timestamp": "2017-05-04T22:46:09.676Z",
-                     "pipeline": "_simulate_pipeline"
-                  }
-               }
+          }
+        },
+        {
+          "processor_type": "set",
+          "status": "success",
+          "doc": {
+            "_index": "index",
+            "_type": "_doc",
+            "_id": "id",
+            "_source": {
+              "field3": "_value3",
+              "field2": "_value2",
+              "foo": "bar"
             },
-            {
-               "doc": {
-                  "_id": "id",
-                  "_index": "index",
-                  "_type": "_doc",
-                  "_source": {
-                     "field3": "_value3",
-                     "field2": "_value2",
-                     "foo": "rab"
-                  },
-                  "_ingest": {
-                     "timestamp": "2017-05-04T22:46:09.677Z",
-                     "pipeline": "_simulate_pipeline"
-                  }
-               }
+            "_ingest": {
+              "pipeline": "_simulate_pipeline",
+              "timestamp": "2020-07-30T01:21:24.251836Z"
             }
-         ]
-      }
-   ]
+          }
+        }
+      ]
+    },
+    {
+      "processor_results": [
+        {
+          "processor_type": "set",
+          "status": "success",
+          "doc": {
+            "_index": "index",
+            "_type": "_doc",
+            "_id": "id",
+            "_source": {
+              "field2": "_value2",
+              "foo": "rab"
+            },
+            "_ingest": {
+              "pipeline": "_simulate_pipeline",
+              "timestamp": "2020-07-30T01:21:24.251863Z"
+            }
+          }
+        },
+        {
+          "processor_type": "set",
+          "status": "success",
+          "doc": {
+            "_index": "index",
+            "_type": "_doc",
+            "_id": "id",
+            "_source": {
+              "field3": "_value3",
+              "field2": "_value2",
+              "foo": "rab"
+            },
+            "_ingest": {
+              "pipeline": "_simulate_pipeline",
+              "timestamp": "2020-07-30T01:21:24.251863Z"
+            }
+          }
+        }
+      ]
+    }
+  ]
 }
+
 ----
-// TESTRESPONSE[s/"2017-05-04T22:46:09.674Z"/$body.docs.0.processor_results.0.doc._ingest.timestamp/]
-// TESTRESPONSE[s/"2017-05-04T22:46:09.675Z"/$body.docs.0.processor_results.1.doc._ingest.timestamp/]
-// TESTRESPONSE[s/"2017-05-04T22:46:09.676Z"/$body.docs.1.processor_results.0.doc._ingest.timestamp/]
-// TESTRESPONSE[s/"2017-05-04T22:46:09.677Z"/$body.docs.1.processor_results.1.doc._ingest.timestamp/]
+// TESTRESPONSE[s/"2020-07-30T01:21:24.251836Z"/$body.docs.0.processor_results.0.doc._ingest.timestamp/]
+// TESTRESPONSE[s/"2020-07-30T01:21:24.251836Z"/$body.docs.0.processor_results.1.doc._ingest.timestamp/]
+// TESTRESPONSE[s/"2020-07-30T01:21:24.251863Z"/$body.docs.1.processor_results.0.doc._ingest.timestamp/]
+// TESTRESPONSE[s/"2020-07-30T01:21:24.251863Z"/$body.docs.1.processor_results.1.doc._ingest.timestamp/]
 
 ////
 [source,console]

modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/90_simulate.yml

@@ -559,11 +559,15 @@ teardown:
   - match: { docs.0.processor_results.0.tag: "setstatus-1" }
   - match: { docs.0.processor_results.0.doc._source.field1: "123.42 400 <foo>" }
   - match: { docs.0.processor_results.0.doc._source.status: 200 }
+  - match: { docs.0.processor_results.0.status: "success" }
+  - match: { docs.0.processor_results.0.processor_type: "set" }
   - match: { docs.0.processor_results.1.tag: "rename-1" }
   - match: { docs.0.processor_results.1.ignored_error.error.type: "illegal_argument_exception" }
   - match: { docs.0.processor_results.1.ignored_error.error.reason: "field [foofield] doesn't exist" }
   - match: { docs.0.processor_results.1.doc._source.field1: "123.42 400 <foo>" }
   - match: { docs.0.processor_results.1.doc._source.status: 200 }
+  - match: { docs.0.processor_results.1.status: "error_ignored" }
+  - match: { docs.0.processor_results.1.processor_type: "rename" }
 
 ---
 "Test verbose simulate with ignore_failure and no exception thrown":
@@ -605,11 +609,16 @@ teardown:
           }
   - length: { docs: 1 }
   - length: { docs.0.processor_results: 2 }
+  - length: { docs.0.processor_results.0: 4 }
   - match: { docs.0.processor_results.0.tag: "setstatus-1" }
+  - match: { docs.0.processor_results.0.status: "success" }
+  - match: { docs.0.processor_results.0.processor_type: "set" }
   - match: { docs.0.processor_results.0.doc._source.field1: "123.42 400 <foo>" }
   - match: { docs.0.processor_results.0.doc._source.status: 200 }
-  - length: { docs.0.processor_results.1: 2 }
+  - length: { docs.0.processor_results.1: 4 }
   - match: { docs.0.processor_results.1.tag: "rename-1" }
+  - match: { docs.0.processor_results.1.status: "success" }
+  - match: { docs.0.processor_results.1.processor_type: "rename" }
   - match: { docs.0.processor_results.1.doc._source.new_status: 200 }
 
 ---
@@ -725,7 +734,8 @@ teardown:
             {
               "set": {
                 "field": "pipeline0",
-                "value": true
+                "value": true,
+                "description" : "first_set"
               }
             },
             {
@@ -747,16 +757,25 @@ teardown:
           ]
         }
 - length: { docs: 1 }
-- length: { docs.0.processor_results: 3 }
+- length: { docs.0.processor_results: 5 }
 - match: { docs.0.processor_results.0.doc._source.pipeline0: true }
+- match: { docs.0.processor_results.0.status: "success" }
+- match: { docs.0.processor_results.0.processor_type: "set" }
+- match: { docs.0.processor_results.0.description: "first_set" }
 - is_false: docs.0.processor_results.0.doc._source.pipeline1
 - is_false: docs.0.processor_results.0.doc._source.pipeline2
-- match: { docs.0.processor_results.1.doc._source.pipeline0: true }
-- match: { docs.0.processor_results.1.doc._source.pipeline1: true }
-- is_false: docs.0.processor_results.1.doc._source.pipeline2
+- match: { docs.0.processor_results.1.doc: null }
+- match: { docs.0.processor_results.1.status: "success" }
+- match: { docs.0.processor_results.1.processor_type: "pipeline" }
 - match: { docs.0.processor_results.2.doc._source.pipeline0: true }
 - match: { docs.0.processor_results.2.doc._source.pipeline1: true }
-- match: { docs.0.processor_results.2.doc._source.pipeline2: true }
+- is_false: docs.0.processor_results.2.doc._source.pipeline2
+- match: { docs.0.processor_results.3.doc: null }
+- match: { docs.0.processor_results.3.status: "success" }
+- match: { docs.0.processor_results.3.processor_type: "pipeline" }
+- match: { docs.0.processor_results.4.doc._source.pipeline0: true }
+- match: { docs.0.processor_results.4.doc._source.pipeline1: true }
+- match: { docs.0.processor_results.4.doc._source.pipeline2: true }
 
 ---
 "Test verbose simulate with true conditional and on failure":
@@ -817,19 +836,27 @@ teardown:
 - length: { docs.0.processor_results: 4 }
 - match: { docs.0.processor_results.0.tag: "gunna_fail" }
 - match: { docs.0.processor_results.0.error.reason: "field [foo1] doesn't exist" }
+- match: { docs.0.processor_results.0.status: "error" }
+- match: { docs.0.processor_results.0.processor_type: "rename" }
 - match: { docs.0.processor_results.1.tag: "failed1" }
 - match: { docs.0.processor_results.1.doc._source.failed1: "failed1" }
 - match: { docs.0.processor_results.1.doc._ingest.on_failure_processor_tag: "gunna_fail" }
+- match: { docs.0.processor_results.1.status: "success" }
+- match: { docs.0.processor_results.1.processor_type: "set" }
 - match: { docs.0.processor_results.2.tag: "gunna_fail_again" }
 - match: { docs.0.processor_results.2.error.reason: "field [foo2] doesn't exist" }
+- match: { docs.0.processor_results.2.status: "error" }
+- match: { docs.0.processor_results.2.processor_type: "rename" }
 - match: { docs.0.processor_results.3.tag: "failed2" }
 - match: { docs.0.processor_results.3.doc._source.failed1: "failed1" }
 - match: { docs.0.processor_results.3.doc._source.failed2: "failed2" }
 - match: { docs.0.processor_results.3.doc._ingest.on_failure_processor_tag: "gunna_fail_again" }
+- match: { docs.0.processor_results.3.status: "success" }
+- match: { docs.0.processor_results.3.processor_type: "set" }
 
 
 ---
-"Test simulate with provided pipeline definition with tag and description in processors":
+"Test simulate with pipeline with conditional and skipped and dropped":
   - do:
       ingest.simulate:
         verbose: true
@@ -845,6 +872,16 @@ teardown:
                     "field" : "field2",
                     "value" : "_value"
                   }
+                },
+                {
+                  "drop" : {
+                    "if": "false"
+                  }
+                },
+                {
+                  "drop" : {
+                     "if": "true"
+                   }
                 }
               ]
             },
@@ -859,7 +896,43 @@ teardown:
             ]
           }
   - length: { docs: 1 }
-  - length: { docs.0.processor_results: 1 }
+  - length: { docs.0.processor_results: 3 }
   - match: { docs.0.processor_results.0.doc._source.field2: "_value" }
   - match: { docs.0.processor_results.0.description: "processor_description" }
   - match: { docs.0.processor_results.0.tag: "processor_tag" }
+  - match: { docs.0.processor_results.0.status: "success" }
+  - match: { docs.0.processor_results.0.processor_type: "set" }
+  - match: { docs.0.processor_results.1.status: "skipped" }
+  - match: { docs.0.processor_results.1.processor_type: "drop" }
+  - match: { docs.0.processor_results.1.if.condition: "false" }
+  - match: { docs.0.processor_results.1.if.result: false }
+  - match: { docs.0.processor_results.2.status: "dropped" }
+  - match: { docs.0.processor_results.2.processor_type: "drop" }
+  - match: { docs.0.processor_results.2.if.condition: "true" }
+  - match: { docs.0.processor_results.2.if.result: true }
+---
+"Test simulate with provided pipeline that does not exist":
+  - do:
+      catch: bad_request
+      ingest.simulate:
+        verbose: true
+        body: >
+          {
+            "pipeline": {
+              "description": "_description",
+              "processors": [
+                {
+                  "pipeline": {
+                     "name": "____pipeline_doesnot_exist___"
+                  }
+                }
+              ]
+            },
+            "docs": [
+              {
+                "_source": {}
+              }
+            ]
+          }
+  - match: { error.root_cause.0.type: "illegal_argument_exception" }
+  - match: { error.root_cause.0.reason: "Pipeline processor configured for non-existent pipeline [____pipeline_doesnot_exist___]" }