[DOCS] Refactor EQL docs (#60700) (#60745)

Changes:

* Moves sample data to reusable rest test
* Combines EQL index, requirements, and run a search pages
* Combines EQL syntax and limitations pages
* Adds related redirects

Author: jrodewig

docs/build.gradle

@@ -186,15 +186,42 @@ buildRestTests.setups['messages'] = '''
           refresh: true
           body: |
             {"index":{"_id": "0"}}
-            {"message": "trying out Elasticsearch" }
+            {"message": "trying out Elasticsearch"}
             {"index":{"_id": "1"}}
-            {"message": "some message with the number 1" }
+            {"message": "some message with the number 1"}
             {"index":{"_id": "2"}}
-            {"message": "some message with the number 2" }
+            {"message": "some message with the number 2"}
             {"index":{"_id": "3"}}
-            {"message": "some message with the number 3" }
+            {"message": "some message with the number 3"}
             {"index":{"_id": "4"}}
-            {"message": "some message with the number 4" }'''
+            {"message": "some message with the number 4"}'''
+
+// Used for EQL
+buildRestTests.setups['sec_logs'] = '''
+  - do:
+        indices.create:
+          index: my-index-000001
+          body:
+            settings:
+              number_of_shards: 1
+              number_of_replicas: 1
+  - do:
+        bulk:
+          index: my-index-000001
+          refresh: true
+          body: |
+            {"index":{}}
+            {"@timestamp": "2020-12-06T11:04:05.000Z", "event": { "category": "process", "id": "edwCRnyD", "sequence": 1 }, "process": { "pid": 2012, "name": "cmd.exe", "executable": "C:\\\\Windows\\\\System32\\\\cmd.exe" }}
+            {"index":{}}
+            {"@timestamp": "2020-12-06T11:04:07.000Z", "event": { "category": "file", "id": "dGCHwoeS", "sequence": 2 }, "file": { "accessed": "2020-12-07T11:07:08.000Z", "name": "cmd.exe", "path": "C:\\\\Windows\\\\System32\\\\cmd.exe", "type": "file", "size": 16384 }, "process": { "pid": 2012, "name": "cmd.exe", "executable": "C:\\\\Windows\\\\System32\\\\cmd.exe" }}
+            {"index":{}}
+            {"@timestamp": "2020-12-07T11:06:07.000Z", "event": { "category": "process", "id": "cMyt5SZ2", "sequence": 3 }, "process": { "pid": 2012, "name": "cmd.exe", "executable": "C:\\\\Windows\\\\System32\\\\cmd.exe" } }
+            {"index":{}}
+            {"@timestamp": "2020-12-07T11:07:08.000Z", "event": { "category": "file", "id": "bYA7gPay", "sequence": 4 }, "file": { "accessed": "2020-12-07T11:07:08.000Z", "name": "cmd.exe", "path": "C:\\\\Windows\\\\System32\\\\cmd.exe", "type": "file", "size": 16384 }, "process": { "pid": 2012, "name": "cmd.exe", "executable": "C:\\\\Windows\\\\System32\\\\cmd.exe" } }
+            {"index":{}}
+            {"@timestamp": "2020-12-07T11:07:09.000Z", "event": { "category": "process", "id": "aR3NWVOs", "sequence": 5 }, "process": { "pid": 2012, "name": "regsvr32.exe", "executable": "C:\\\\Windows\\\\System32\\\\regsvr32.exe" }}
+            {"index":{}}
+            {"@timestamp": "2020-12-07T11:07:10.000Z", "event": { "category": "process", "id": "GTSmSqgz0U", "sequence": 6, "type": "termination" }, "process": { "pid": 2012, "name": "regsvr32.exe", "executable": "C:\\\\Windows\\\\System32\\\\regsvr32.exe" }}'''
 
 buildRestTests.setups['host'] = '''
   # Fetch the http host. We use the host of the master because we know there will always be a master.

docs/reference/eql/delete-async-eql-search-api.asciidoc

@@ -27,12 +27,12 @@ DELETE /_eql/search/FkpMRkJGS1gzVDRlM3g4ZzMyRGlLbkEaTXlJZHdNT09TU2VTZVBoNDM3cFZM
 [[delete-async-eql-search-api-prereqs]]
 ==== {api-prereq-title}
 
-See <<eql-requirements,EQL requirements>>.
+See <<eql-required-fields>>.
 
 [[delete-async-eql-search-api-limitations]]
 ===== Limitations
 
-See <<eql-limitations,EQL limitations>>.
+See <<eql-syntax-limitations,EQL limitations>>.
 
 [[delete-async-eql-search-api-path-params]]
 ==== {api-path-parms-title}

docs/reference/eql/eql-search-api.asciidoc

@@ -14,35 +14,16 @@ Returns search results for an <<eql,Event Query Language (EQL)>> query.
 In {es}, EQL assumes each document in a data stream or index corresponds to an
 event.
 
-////
 [source,console]
 ----
-PUT /my-index-00001/_bulk?refresh
-{"index":{ }}
-{ "@timestamp": "2020-12-06T11:04:05.000Z", "agent": { "id": "8a4f500d" }, "event": { "category": "process", "id": "edwCRnyD", "sequence": 1 }, "process": { "name": "cmd.exe", "executable": "C:\\Windows\\System32\\cmd.exe" } }
-{"index":{ }}
-{ "@timestamp": "2020-12-06T11:04:07.000Z", "agent": { "id": "8a4f500d" }, "event": { "category": "file", "id": "dGCHwoeS", "sequence": 2 }, "file": { "accessed": "2020-12-07T11:07:08.000Z", "name": "cmd.exe", "path": "C:\\Windows\\System32\\cmd.exe", "type": "file", "size": 16384 }, "process": { "name": "cmd.exe", "executable": "C:\\Windows\\System32\\cmd.exe" } }
-{"index":{ }}
-{ "@timestamp": "2020-12-07T11:06:07.000Z", "agent": { "id": "8a4f500d" }, "event": { "category": "process", "id": "cMyt5SZ2", "sequence": 3 }, "process": { "name": "cmd.exe", "executable": "C:\\Windows\\System32\\cmd.exe" } }
-{"index":{ }}
-{ "@timestamp": "2020-12-07T11:07:08.000Z", "agent": { "id": "8a4f500d" }, "event": { "category": "file", "id": "bYA7gPay", "sequence": 4 }, "file": { "accessed": "2020-12-07T11:07:08.000Z", "name": "cmd.exe", "path": "C:\\Windows\\System32\\cmd.exe", "type": "file", "size": 16384 }, "process": { "name": "cmd.exe", "executable": "C:\\Windows\\System32\\cmd.exe" } }
-{"index":{ }}
-{ "@timestamp": "2020-12-07T11:07:09.000Z", "agent": { "id": "8a4f500d" }, "event": { "category": "process", "id": "aR3NWVOs", "sequence": 5 }, "process": { "name": "regsvr32.exe", "executable": "C:\\Windows\\System32\\regsvr32.exe" } }
-{"index":{ }}
-{ "@timestamp": "2020-12-07T11:07:10.000Z", "agent": { "id": "8a4f500d" }, "event": { "category": "process", "id": "GTSmSqgz0U", "sequence": 6, "type": "termination" }, "process": { "name": "regsvr32.exe", "executable": "C:\\Windows\\System32\\regsvr32.exe" } }
-----
-// TESTSETUP
-////
-
-[source,console]
-----
-GET /my-index-00001/_eql/search
+GET /my-index-000001/_eql/search
 {
   "query": """
     process where process.name = "regsvr32.exe"
   """
 }
 ----
+// TEST[setup:sec_logs]
 
 [[eql-search-api-request]]
 ==== {api-request-title}
@@ -54,12 +35,12 @@ GET /my-index-00001/_eql/search
 [[eql-search-api-prereqs]]
 ==== {api-prereq-title}
 
-See <<eql-requirements,EQL requirements>>.
+See <<eql-required-fields>>.
 
 [[eql-search-api-limitations]]
 ===== Limitations
 
-See <<eql-limitations,EQL limitations>>.
+See <<eql-syntax-limitations,EQL limitations>>.
 
 [[eql-search-api-path-params]]
 ==== {api-path-parms-title}
@@ -163,6 +144,9 @@ Field containing the event classification, such as `process`, `file`, or
 Defaults to `event.category`, as defined in the {ecs-ref}/ecs-event.html[Elastic
 Common Schema (ECS)]. If a data stream or index does not contain the
 `event.category` field, this value is required.
++
+The event category field is typically mapped as a <<keyword,`keyword`>> or
+<<constant-keyword,constant keyword>> field.
 
 `fetch_size`::
 (Optional, integer)
@@ -275,6 +259,9 @@ does not contain the `@timestamp` field, this value is required.
 Events in the API response are sorted by this field's value, converted to
 milliseconds since the https://en.wikipedia.org/wiki/Unix_time[Unix epoch], in
 ascending order.
+
+The timestamp field is typically mapped as a <<date,`date`>> or
+<<date_nanos,`date_nanos`>> field.
 --
 
 [[eql-search-api-wait-for-completion-timeout]]
@@ -506,17 +493,18 @@ The following EQL search request searches for events with an `event.category` of
 `file` that meet the following conditions:
 
 * A `file.name` of `cmd.exe`
-* An `agent.id` other than `8a4f526c`
+* An `process.pid` other than `2013`
 
 [source,console]
 ----
-GET /my-index-00001/_eql/search
+GET /my-index-000001/_eql/search
 {
   "query": """
-    file where (file.name == "cmd.exe" and agent.id != "8a4f526c")
+    file where (file.name == "cmd.exe" and process.pid != 2013)
   """
 }
 ----
+// TEST[setup:sec_logs]
 // TEST[s/search/search\?filter_path\=\-\*\.events\.\*fields/]
 
 The API returns the following response. Matching events in the `hits.events`
@@ -542,15 +530,12 @@ the events in ascending, lexicographic order.
     },
     "events": [
       {
-        "_index": "my-index-00001",
+        "_index": "my-index-000001",
         "_type": "_doc",
         "_id": "fwGeywNsBl8Y9Ys1x51b",
         "_score": null,
         "_source": {
           "@timestamp": "2020-12-06T11:04:07.000Z",
-          "agent": {
-            "id": "8a4f500d"
-          },
           "event": {
             "category": "file",
             "id": "dGCHwoeS",
@@ -565,20 +550,18 @@ the events in ascending, lexicographic order.
           },
           "process": {
             "name": "cmd.exe",
-            "executable": "C:\\Windows\\System32\\cmd.exe"
+            "executable": "C:\\Windows\\System32\\cmd.exe",
+            "pid": 2012
           }
         }
       },
       {
-        "_index": "my-index-00001",
+        "_index": "my-index-000001",
         "_type": "_doc",
         "_id": "AtOJ4UjUBAAx3XR5kcCM",
         "_score": null,
         "_source": {
           "@timestamp": "2020-12-07T11:07:08.000Z",
-          "agent": {
-            "id": "8a4f500d"
-          },
           "event": {
             "category": "file",
             "id": "bYA7gPay",
@@ -593,7 +576,8 @@ the events in ascending, lexicographic order.
           },
           "process": {
             "name": "cmd.exe",
-            "executable": "C:\\Windows\\System32\\cmd.exe"
+            "executable": "C:\\Windows\\System32\\cmd.exe",
+            "pid": 2012
           }
         }
       }
@@ -616,7 +600,7 @@ that:
 --
 * An `event.category` of `file`
 * A `file.name` of `cmd.exe`
-* An `agent.id` other than `8a4f526c`
+* An `process.pid` other than `2013`
 --
 . Followed by an event with:
 +
@@ -625,29 +609,24 @@ that:
 * A `process.executable` that contains the substring `regsvr32`
 --
 
-These events must also share the same `agent.id` value.
+These events must also share the same `process.pid` value.
 
 [source,console]
 ----
-GET /my-index-00001/_eql/search
+GET /my-index-000001/_eql/search
 {
   "query": """
-    sequence by agent.id
-      [ file where file.name == "cmd.exe" and agent.id != "8a4f526c" ]
+    sequence by process.pid
+      [ file where file.name == "cmd.exe" and process.pid != 2013 ]
       [ process where stringContains(process.executable, "regsvr32") ]
   """
 }
 ----
+// TEST[setup:sec_logs]
 
-The API returns the following response. The `hits.sequences.join_keys` property
-contains the shared `agent.id` value for each matching event. Matching events in
-the `hits.sequences.events` property are sorted by
-<<eql-search-api-timestamp-field,timestamp>>, converted to milliseconds since
-the https://en.wikipedia.org/wiki/Unix_time[Unix epoch], in ascending order.
-
-If two or more events share the same timestamp, the
-<<eql-search-api-tiebreaker-field,`tiebreaker_field`>> field is used to sort
-the events in ascending, lexicographic order.
+The API returns the following response. Matching sequences are included in the
+`hits.sequences` property. The `hits.sequences.join_keys` property contains the
+shared `process.pid` value for each matching event.
 
 [source,console-result]
 ----
@@ -664,11 +643,11 @@ the events in ascending, lexicographic order.
     "sequences": [
       {
         "join_keys": [
-          "8a4f500d"
+          "2012"
         ],
         "events": [
           {
-            "_index": "my-index-00001",
+            "_index": "my-index-000001",
             "_type": "_doc",
             "_id": "AtOJ4UjUBAAx3XR5kcCM",
             "_version": 1,
@@ -677,9 +656,6 @@ the events in ascending, lexicographic order.
             "_score": null,
             "_source": {
               "@timestamp": "2020-12-07T11:07:08.000Z",
-              "agent": {
-                "id": "8a4f500d"
-              },
               "event": {
                 "category": "file",
                 "id": "bYA7gPay",
@@ -692,14 +668,15 @@ the events in ascending, lexicographic order.
                 "type": "file",
                 "size": 16384
               },
-              "process": {
+              "process": { 
                 "name": "cmd.exe",
-                "executable": "C:\\Windows\\System32\\cmd.exe"
+                "executable": "C:\\Windows\\System32\\cmd.exe",
+                "pid": 2012
               }
             }
           },
           {
-            "_index": "my-index-00001",
+            "_index": "my-index-000001",
             "_type": "_doc",
             "_id": "yDwnGIJouOYGBzP0ZE9n",
             "_version": 1,
@@ -708,17 +685,15 @@ the events in ascending, lexicographic order.
             "_score": null,
             "_source": {
               "@timestamp": "2020-12-07T11:07:09.000Z",
-              "agent": {
-                "id": "8a4f500d"
-              },
               "event": {
                 "category": "process",
                 "id": "aR3NWVOs",
                 "sequence": 5
               },
-              "process": {
+              "process": { 
                 "name": "regsvr32.exe",
-                "executable": "C:\\Windows\\System32\\regsvr32.exe"
+                "executable": "C:\\Windows\\System32\\regsvr32.exe",
+                "pid": 2012
               }
             }
           }