[7.x] [DOCS] Document ingest processor description (#70899)

docs/reference/ingest.asciidoc

@@ -60,16 +60,18 @@ in the order specified.
 ----
 PUT _ingest/pipeline/my-pipeline
 {
-  "description": "My pipeline description",
+  "description": "My optional pipeline description",
   "processors": [
     {
       "set": {
+        "description": "My optional processor description",
         "field": "my-long-field",
         "value": 10
       }
     },
     {
       "set": {
+        "description": "Set 'my-boolean-field' to true",
         "field": "my-boolean-field",
         "value": true
       }
@@ -321,11 +323,13 @@ PUT _ingest/pipeline/my-pipeline
   "processors": [
     {
       "dot_expander": {
+        "description": "Expand 'my-object-field.my-property'",
         "field": "my-object-field.my-property"
       }
     },
     {
       "set": {
+        "description": "Set 'my-object-field.my-property' to 10",
         "field": "my-object-field.my-property",
         "value": 10
       }
@@ -337,8 +341,7 @@ PUT _ingest/pipeline/my-pipeline
 [[template-snippets]]
 To access field values, enclose the field name in double curly brackets `{{ }}`
 to create a https://mustache.github.io[Mustache] template snippet. You can use
-template snippets to dynamically set field names. The following processor sets a
-field name as the `service` field value.
+template snippets to dynamically set field names.
 
 [source,console]
 ----
@@ -347,6 +350,7 @@ PUT _ingest/pipeline/my-pipeline
   "processors": [
     {
       "set": {
+        "description": "Set dynamic '<service>' field to 'code' value",
         "field": "{{service}}",
         "value": "{{code}}"
       }
@@ -365,16 +369,14 @@ Processors can access the following metadata fields by name:
 * `_id`
 * `_routing`
 
-For example, the following `set` processor sets the document's routing value as
-the `geoip.country_iso_code` field value.
-
 [source,console]
 ----
 PUT _ingest/pipeline/my-pipeline
 {
   "processors" : [
     {
       "set" : {
+        "description": "Set '_routing' to 'geoip.country_iso_code' value",
         "field": "_routing",
         "value": "{{geoip.country_iso_code}}"
       }
@@ -412,7 +414,8 @@ PUT _ingest/pipeline/my-pipeline
   "processors": [
     {
       "set": {
-        "field": "received",
+        "description": "Index the ingest timestamp as 'event.ingested'",
+        "field": "event.ingested",
         "value": "{{_ingest.timestamp}}"
       }
     }
@@ -437,8 +440,9 @@ PUT _ingest/pipeline/my-pipeline
   "processors": [
     {
       "rename": {
-        "field": "foo",
-        "target_field": "bar",
+        "description": "Rename 'provider' to 'cloud.provider'",
+        "field": "provider",
+        "target_field": "cloud.provider",
         "ignore_failure": true
       }
     }
@@ -458,13 +462,15 @@ PUT _ingest/pipeline/my-pipeline
   "processors": [
     {
       "rename": {
-        "field": "foo",
-        "target_field": "bar",
+        "description": "Rename 'provider' to 'cloud.provider'",
+        "field": "provider",
+        "target_field": "cloud.provider",
         "on_failure": [
           {
             "set": {
+              "description": "Set 'error.message'",
               "field": "error.message",
-              "value": "field \"foo\" does not exist, cannot rename to \"bar\"",
+              "value": "Field 'provider' does not exist. Cannot rename to 'cloud.provider'",
               "override": false
             }
           }
@@ -484,17 +490,20 @@ PUT _ingest/pipeline/my-pipeline
   "processors": [
     {
       "rename": {
-        "field": "foo",
-        "target_field": "bar",
+        "description": "Rename 'provider' to 'cloud.provider'",
+        "field": "provider",
+        "target_field": "cloud.provider",
         "on_failure": [
           {
             "set": {
+              "description": "Set 'error.message'",
               "field": "error.message",
-              "value": "field \"foo\" does not exist, cannot rename to \"bar\"",
+              "value": "Field 'provider' does not exist. Cannot rename to 'cloud.provider'",
               "override": false,
               "on_failure": [
                 {
                   "set": {
+                    "description": "Set 'error.message.multi'",
                     "field": "error.message.multi",
                     "value": "Document encountered multiple ingest errors",
                     "override": true
@@ -522,6 +531,7 @@ PUT _ingest/pipeline/my-pipeline
   "on_failure": [
     {
       "set": {
+        "description": "Index document to 'failed-<index>'",
         "field": "_index",
         "value": "failed-{{ _index }}"
       }
@@ -543,17 +553,15 @@ IMPORTANT: `if` condition scripts run in Painless's
 {painless}/painless-ingest-processor-context.html[ingest processor context]. In
 `if` conditions, `ctx` values are read-only.
 
-The following <<drop-processor,`drop`>> processor uses an `if` condition to drop
-documents with a `network_name` of `Guest`.
-
 [source,console]
 ----
 PUT _ingest/pipeline/my-pipeline
 {
   "processors": [
     {
       "drop": {
-        "if": "ctx?.network_name == 'Guest'"
+        "description": "Drop documents with 'network.name' of 'Guest'",
+        "if": "ctx?.network?.name == 'Guest'"
       }
     }
   ]
@@ -575,8 +583,9 @@ PUT _ingest/pipeline/my-pipeline
   "processors": [
     {
       "set": {
-        "if": "ctx.href?.url =~ /^http[^s]/",
-        "field": "href.insecure",
+        "description": "If 'url.scheme' is 'http', set 'url.insecure' to true",
+        "if": "ctx.url?.scheme =~ /^http[^s]/",
+        "field": "url.insecure",
         "value": true
       }
     }
@@ -598,6 +607,7 @@ PUT _ingest/pipeline/my-pipeline
   "processors": [
     {
       "drop": {
+        "description": "Drop documents that don't contain 'prod' tag",
         "if": """
             Collection tags = ctx.tags;
             if(tags != null){
@@ -643,6 +653,7 @@ PUT _ingest/pipeline/my-pipeline
   "processors": [
     {
       "drop": {
+        "description": "If 'url.scheme' is 'http', set 'url.insecure' to true",
         "if": { "id": "my-stored-script" }
       }
     }
@@ -670,6 +681,7 @@ PUT _ingest/pipeline/my-pipeline
   "processors": [
     {
       "drop": {
+        "description": "Drop documents that contain 'network.name' of 'Guest'",
         "if": "ctx.network?.name != null && ctx.network.name.contains('Guest')"
       }
     }
@@ -687,28 +699,28 @@ pipeline as the <<set-default-pipeline,default pipeline>> in an
 <<index-templates,index template>> used to configure multiple data streams or
 indices.
 
-The following pipeline applies different pipelines to incoming documents based
-on the `service.name` field value.
-
 [source,console]
 ----
 PUT _ingest/pipeline/one-pipeline-to-rule-them-all
 {
   "processors": [
     {
       "pipeline": {
+        "description": "If 'service.name' is 'apache_httpd', use 'httpd_pipeline'",
         "if": "ctx.service?.name == 'apache_httpd'",
         "name": "httpd_pipeline"
       }
     },
     {
       "pipeline": {
+        "description": "If 'service.name' is 'syslog', use 'syslog_pipeline'",
         "if": "ctx.service?.name == 'syslog'",
         "name": "syslog_pipeline"
       }
     },
     {
       "fail": {
+        "description": "If 'service.name' is not 'apache_httpd' or 'syslog', return a failure message",
         "if": "ctx.service?.name != 'apache_httpd' && ctx.service?.name != 'syslog'",
         "message": "This pipeline requires service.name to be either `syslog` or `apache_httpd`"
       }

docs/reference/ingest/apis/put-pipeline.asciidoc

@@ -11,12 +11,13 @@ take effect immediately.
 ----
 PUT _ingest/pipeline/my-pipeline-id
 {
-  "description" : "describe pipeline",
+  "description" : "My optional pipeline description",
   "processors" : [
     {
       "set" : {
-        "field": "foo",
-        "value": "bar"
+        "description" : "My optional processor description",
+        "field": "my-keyword-field",
+        "value": "foo"
       }
     }
   ]

docs/reference/ingest/common-log-format-example.asciidoc

@@ -19,7 +19,7 @@ AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36\"
 ----
 // NOTCONSOLE
 
-These logs contain an IP address, timestamp, and user agent. You want to give
+These logs contain a timestamp, IP address, and user agent. You want to give
 these three items their own field in {es} for faster searches and
 visualizations. You also want to know where the request is coming from.
 
@@ -34,8 +34,8 @@ image::images/ingest/ingest-pipeline-list.png[Kibana's Ingest Node Pipelines lis
 . Add a <<grok-processor,grok processor>> to parse the log message:
 
 .. Click **Add a processor** and select the **Grok** processor type.
-.. Set the field input to `message` and enter the following <<grok-basics,grok
-pattern>>:
+.. Set **Field** to `message` and **Patterns** to the following
+<<grok-basics,grok pattern>>:
 +
 [source,grok]
 ----
@@ -44,19 +44,32 @@ pattern>>:
 // NOTCONSOLE
 +
 .. Click **Add** to save the processor.
+.. Set the processor description to `Extract fields from 'message'`.
 
-. Add processors to map the date, IP, and user agent fields. Map the appropriate
-field to each processor type:
+. Add processors for the timestamp, IP address, and user agent fields. Configure
+the processors as follows:
 +
 --
-* <<date-processor,**Date**>>: `@timestamp`
-* <<geoip-processor,**GeoIP**>>: `source.ip`
-* <<user-agent-processor,**User agent**>>: `user_agent`
 
-In the **Date** processor, specify the date format you want to use:
-`dd/MMM/yyyy:HH:mm:ss Z`.
+[options="header"]
+|====
+| Processor type | Field  | Additional options | Description  
 
-In the **GeoIP** processor, specify the target field as `source.geo`.
+| <<date-processor,**Date**>>
+| `@timestamp`
+| **Formats**: `dd/MMM/yyyy:HH:mm:ss Z`
+| `Format '@timestamp' as 'dd/MMM/yyyy:HH:mm:ss Z'`
+
+| <<geoip-processor,**GeoIP**>>
+| `source.ip`
+| **Target field**: `source.geo`
+| `Add 'source.geo' geoIP data for 'source.ip'`
+
+| <<user-agent-processor,**User agent**>>
+| `user_agent`
+|
+| `Extract fields from 'user_agent'`
+|====
 
 Your form should look similar to this:
 
@@ -87,24 +100,28 @@ PUT /_ingest/pipeline/my-pipeline
   "processors": [
     {
       "grok": {
+        "description": "Extract fields from 'message'",
         "field": "message",
         "patterns": ["%{IPORHOST:source.ip} %{USER:user.id} %{USER:user.name} \\[%{HTTPDATE:@timestamp}\\] \"%{WORD:http.request.method} %{DATA:url.original} HTTP/%{NUMBER:http.version}\" %{NUMBER:http.response.status_code:int} (?:-|%{NUMBER:http.response.body.bytes:int}) %{QS:http.request.referrer} %{QS:user_agent}"]
       }
     },
     {
       "date": {
+        "description": "Format '@timestamp' as 'dd/MMM/yyyy:HH:mm:ss Z'",
         "field": "@timestamp",
         "formats": [ "dd/MMM/yyyy:HH:mm:ss Z" ]
       }
     },
     {
       "geoip": {
+        "description": "Add 'source.geo' geoIP data for 'source.ip'",
         "field": "source.ip",
         "target_field": "source.geo"
       }
     },
     {
       "user_agent": {
+        "description": "Extract fields from 'user_agent'",
         "field": "user_agent"
       }
     }

docs/reference/ingest/enrich.asciidoc

@@ -315,10 +315,10 @@ includes:
 ----
 PUT /_ingest/pipeline/postal_lookup
 {
-  "description": "Enrich postal codes",
   "processors": [
     {
       "enrich": {
+        "description": "Add 'geo_data' based on 'geo_location'",
         "policy_name": "postal_policy",
         "field": "geo_location",
         "target_field": "geo_data",
@@ -471,10 +471,10 @@ includes:
 ----
 PUT /_ingest/pipeline/user_lookup
 {
-  "description" : "Enriching user details to messages",
   "processors" : [
     {
       "enrich" : {
+        "description": "Add 'user' data based on 'email'",
         "policy_name": "users-policy",
         "field" : "email",
         "target_field": "user",

docs/reference/ingest/processors/common-options.asciidoc

@@ -1,5 +1,5 @@
-| `if`             | no       | -        | Conditionally execute this processor.
-| `on_failure`     | no       | -        | Handle failures for this processor. See <<handling-failure-in-pipelines>>.
-| `ignore_failure` | no       | `false`  | Ignore failures for this processor. See <<handling-failure-in-pipelines>>.
-| `tag`            | no       | -        | An identifier for this processor. Useful for debugging and metrics.
-// TODO: See <<ingest-conditionals>>. <-- for the if description once PR 35044 is merged
+| `description`     | no      | -        | Description of the processor. Useful for describing the purpose of the processor or its configuration.
+| `if`             | no       | -        | Conditionally execute the processor. See <<handling-pipeline-failures>>.
+| `ignore_failure` | no       | `false`  | Ignore failures for the processor. See <<handling-failure-in-pipelines>>.
+| `on_failure`     | no       | -        | Handle failures for the processor. See <<handling-failure-in-pipelines>>.
+| `tag`            | no       | -        | Identifier for the processor. Useful for debugging and metrics.