update watcher examples for 7.x

Alerting/Sample Watches/README.md

@@ -24,11 +24,11 @@ In each watch directory the following is provided:
 
 The parent directory includes the following utility scripts:
 
-* run_test.py - A python script which can be used to run a specific test e.g. python run_test.py --test_file new_process_started/tests/test1.json. Include optional username and password with --user and --password parameters.
+* run_test.py - A python script which can be used to run a specific test e.g. python run_test.py --test_file new_process_started/tests/test1.json. Include optional username and password with --user, --password, --endpoint, --port, and --protocol parameters.
 * load_watch.sh.  Utility script for loading a watch to a local Elasticsearch cluster.  Each watch can be loaded by running `load_watch.sh <watch folder name>`.  This will also index any scripts. Username and password for the cluster can be specified as parameters e.g.
-`load_watch.sh <watch folder name> <username> <password> <protocol>`
-* run_test.sh - Runs a specified watches tests. Specify watch by directory name e.g. `run_test.sh port_scan`. Include optional username and password e.g. `run_test.sh port_scan <username> <password> <protocol>`.
-* run_all_tests.sh - Runs all tests. Include optional username and password e.g. `run_all_tests.sh <username> <password> <protocol>`.
+`load_watch.sh <watch folder name> <optional_username> <optional_password> <optional_endpoint>:<optional_port> <optional_protocol>"`
+* run_test.sh - Runs a specified watches tests. Specify watch by directory name e.g. `run_test.sh port_scan`. Include optional username and password e.g. `run_test.sh <watch folder name> <username> <password> <endpoint> <protocol>`.
+* run_all_tests.sh - Runs all tests. Include optional username and password e.g. `run_all_tests.sh <username> <password> <endpoint> <protocol>`.
 
 If username, password, and protocol are not specified, the above scripts assume the x-pack default of "elastic", "changeme", and "http" respectively.
 

Alerting/Sample Watches/cpu_iowait_hosts/mapping.json

@@ -1,6 +1,5 @@
 {
   "mappings": {
-    "doc":{
       "properties": {
         "beat":{
           "properties": {
@@ -106,6 +105,5 @@
           }
         }
       }
-    }
   }
-}
+}

Alerting/Sample Watches/cpu_iowait_hosts/watch.json

@@ -16,9 +16,6 @@
                 "indices": [
                     "metricbeat-*"
                 ],
-                "types": [
-                    "doc"
-                ],
                 "body": {
                   "size": 0,
                   "aggs": {
@@ -27,7 +24,7 @@
                         "per_minute": {
                           "date_histogram": {
                             "field": "@timestamp",
-                            "interval": "{{ctx.metadata.interval}}"
+                            "fixed_interval": "{{ctx.metadata.interval}}"
                           },
                           "aggs": {
                             "iowait": {
@@ -84,4 +81,4 @@
 	          }
         }
     }
-}
+}

Alerting/Sample Watches/errors_in_logs/mapping.json

@@ -1,6 +1,5 @@
 {
   "mappings": {
-    "doc": {
       "properties": {
         "@timestamp": {
           "type": "date",
@@ -13,6 +12,5 @@
           "type": "keyword"
         }
       }
-    }
   }
-}
+}

Alerting/Sample Watches/filesystem_usage/mapping.json

@@ -1,6 +1,5 @@
 {
   "mappings": {
-    "filesystem":{
       "properties": {
         "hostname":{
           "type": "keyword"
@@ -12,6 +11,5 @@
           "type":"date"
         }
       }
-    }
   }
-}
+}

Alerting/Sample Watches/filesystem_usage/watch.json

@@ -14,9 +14,6 @@
         "indices": [
           "logs"
         ],
-        "types": [
-          "filesystem"
-        ],
         "body": {
           "aggs": {
             "host": {
@@ -75,9 +72,9 @@
     "log": {
       "logging": {
         "text": {
-          "inline": "Some hosts are over {{ctx.payload.threshold}}% utilized:{{#ctx.payload.hosts}}{{disk_usage}}%-{{key}}:{{/ctx.payload.hosts}}"
+          "source": "Some hosts are over {{ctx.payload.threshold}}% utilized:{{#ctx.payload.hosts}}{{disk_usage}}%-{{key}}:{{/ctx.payload.hosts}}"
         }
       }
     }
   }
-}
+}

Alerting/Sample Watches/lateral_movement_in_user_comm/mapping.json

@@ -1,6 +1,5 @@
 {
   "mappings": {
-    "doc": {
       "properties": {
         "user_server": {
           "type": "keyword"
@@ -14,6 +13,5 @@
           "format": "HH:mm:ss||strict_time_no_millis"
         }
       }
-    }
   }
-}
+}

Alerting/Sample Watches/load_watch.sh

@@ -60,14 +60,15 @@ do
     fi
 done
 
-echo "Loading $1 watch "
 
-curl -H "Content-Type: application/json" -s -o /dev/null -X DELETE $endpoint:$port/_xpack/watcher/watch/$1 -u $username:$password
+echo "Removing existing $1 watch "
+curl -H "Content-Type: application/json" -s -X DELETE $protocol$endpoint:$port/_xpack/watcher/watch/$1 -u $username:$password
+echo "Loading $1 watch "
 es_response=$(curl -H "Content-Type: application/json" --w "%{http_code}" -s -o /dev/null -X PUT $protocol$endpoint:$port/_xpack/watcher/watch/$1 -u $username:$password -d @$1/watch.json)
 if [ 0 -eq $? ] && [ $es_response = "201" ]; then
-  echo "Loading $2 watch...OK"
+  echo "Loading $1 watch...OK"
   exit 0
 else
-  echo "Loading $2 watch...FAILED"
+  echo "Loading $1 watch...FAILED with response code $es_response"
   exit 1
 fi

Alerting/Sample Watches/monitoring_cluster_health/mapping.json

@@ -1,6 +1,5 @@
 {
   "mappings": {
-    "doc": {
       "date_detection": false,
       "properties": {
         "cluster_state": {
@@ -37,6 +36,5 @@
           "type": "keyword"
         }
       }
-    }
   }
-}
+}

Alerting/Sample Watches/monitoring_cluster_health/tests/test1.json

@@ -1,7 +1,7 @@
 {
   "watch_name":"monitoring_cluster_health",
   "mapping_file":"./monitoring_cluster_health/mapping.json",
-  "index":".monitoring-es-test",
+  "index":"monitoring-es-test",
   "type":"doc",
   "time_field":"timestamp",
   "watch_file":"./monitoring_cluster_health/watch.json",

Alerting/Sample Watches/monitoring_cluster_health/watch.json

@@ -12,8 +12,7 @@
   "input": {
     "search": {
       "request": {
-        "indices": ".monitoring-es-test",
-        "types": "doc",
+        "indices": "monitoring-es-test",
         "body": {
           "query": {
             "bool": {
@@ -97,4 +96,4 @@
       }
     }
   }
-}
+}

Alerting/Sample Watches/monitoring_free_disk_space/mapping.json

@@ -1,9 +1,5 @@
 {
   "mappings": {
-    "doc": {
-      "_all": {
-        "enabled": false
-      },
       "date_detection": false,
       "properties": {
         "type": {
@@ -75,6 +71,5 @@
           "format": "date_time"
         }
       }
-    }
   }
-}
+}

Alerting/Sample Watches/monitoring_free_disk_space/tests/test1.json

@@ -1,8 +1,7 @@
 {
   "watch_name":"monitoring_free_disk_space",
   "mapping_file":"./monitoring_free_disk_space/mapping.json",
-  "index":".monitoring-es-test",
-  "type":"doc",
+  "index":"monitoring-es-test",
   "time_field":"timestamp",
   "watch_file":"./monitoring_free_disk_space/watch.json",
   "comments":"Tests that each nodes host has more than lower_bound free disk space. Nodes a and c matches, b does not.",

Alerting/Sample Watches/monitoring_free_disk_space/watch.json

@@ -10,8 +10,7 @@
   "input": {
     "search": {
       "request": {
-        "indices": ".monitoring-es-test",
-        "types": "doc",
+        "indices": "monitoring-es-test",
         "body": {
           "query": {
             "bool": {
@@ -83,4 +82,4 @@
       }
     }
   }
-}
+}

Alerting/Sample Watches/new_process_started/mapping.json

@@ -1,6 +1,5 @@
 {
   "mappings": {
-    "doc": {
       "properties": {
         "@timestamp": {
           "type": "date"
@@ -12,6 +11,5 @@
           "type": "keyword"
         }
       }
-    }
   }
-}
+}

Alerting/Sample Watches/port_scan/mapping.json

@@ -1,6 +1,5 @@
 {
   "mappings": {
-    "doc": {
       "properties": {
         "source_dest": {
           "type": "keyword"
@@ -15,6 +14,5 @@
           "type":"date"
         }
       }
-    }
   }
-}
+}

Alerting/Sample Watches/port_scan/watch.json

@@ -34,7 +34,7 @@
                 "series": {
                   "date_histogram": {
                     "field": "@timestamp",
-                    "interval": "{{ctx.metadata.time_period}}"
+                    "fixed_interval": "{{ctx.metadata.time_period}}"
                   },
                   "aggs": {
                     "num_ports": {
@@ -94,9 +94,8 @@
         }
       },
       "index": {
-        "index": "connection-scans",
-        "doc_type": "scan"
+        "index": "connection-scans"
       }
     }
   }
-}
+}

Alerting/Sample Watches/run_all_tests.sh

@@ -1,2 +1,2 @@
 #!/usr/bin/env bash
-./run_test.sh '**' $1 $2 $3
+./run_test.sh '**' $1 $2 $3 $4 $5

Alerting/Sample Watches/run_test.py

@@ -3,22 +3,22 @@
 __author__ = '[email protected]'
 
 import datetime
-from elasticsearch import Elasticsearch
-from elasticsearch_xpack import XPackClient
+from elasticsearch7 import Elasticsearch
+from elasticsearch7.client.ingest import IngestClient
 import argparse
 import json
 
 parser = argparse.ArgumentParser(description='Index Connection Log data into ES with the last event at the current time')
-parser.add_argument('--host',help='host name')
-parser.add_argument('--port',help='port')
 parser.add_argument('--user',help='user')
 parser.add_argument('--password',help='password')
+parser.add_argument('--endpoint',help='endpoint')
+parser.add_argument('--port',help='port')
 parser.add_argument('--protocol',help='protocol')
 parser.add_argument('--test_file',help='test file')
 
-parser.set_defaults(host='localhost',port="9200",protocol="http",test_file='data.json',user='elastic',password='changeme')
+parser.set_defaults(endpoint='localhost',port="9200",protocol="http",test_file='data.json',user='elastic',password='changeme')
 args = parser.parse_args()
-es = Elasticsearch([args.protocol+"://"+args.host+":"+args.port],http_auth=(args.user, args.password))
+es = Elasticsearch([args.protocol+"://"+args.endpoint+":"+args.port],http_auth=(args.user, args.password))
 
 def find_item(list, key):
     for item in list:
@@ -28,44 +28,39 @@ def find_item(list, key):
 
 with open(args.test_file,'r') as test_file:
     test=json.loads(test_file.read())
-    #Load Mapping
     try:
         es.indices.delete(test['index'])
     except:
         print("Unable to delete current dataset")
         pass
     with open(test['mapping_file'],'r') as mapping_file:
         es.indices.create(index=test["index"],body=json.loads(mapping_file.read()))
-    #Load pipeline if its declared
     params={}
     if "ingest_pipeline_file" in test:
         with open(test['ingest_pipeline_file'],'r') as ingest_pipeline_file:
-            es.index(index="_ingest",doc_type="pipeline",id=test["watch_name"],body=json.loads(ingest_pipeline_file.read()))
+            pipeline=json.loads(ingest_pipeline_file.read())
+            p = IngestClient(es)
+            p.put_pipeline(id=test["watch_name"],body=pipeline)
             params["pipeline"]=test["watch_name"]
-    #Index data
     current_data=last_time=datetime.datetime.utcnow()
     i=0
     time_field = test["time_field"] if "time_field" in test else "@timestamp"
     for event in test['events']:
-        #All offsets in seconds
         event_time=current_data+datetime.timedelta(seconds=int(event['offset'] if 'offset' in event else 0))
         event[time_field]=event_time.strftime('%Y-%m-%dT%H:%M:%S.%fZ') if not time_field in event else event[time_field]
-        es.index(index=test['index'],doc_type=test['type'],body=event,id=event['id'] if "id" in event else i,params=params)
+        es.index(index=test['index'],body=event,id=event['id'] if "id" in event else i,params=params)
         i+=1
     es.indices.refresh(index=test["index"])
-    #Load Scripts
     if 'scripts' in test:
         for script in test['scripts']:
             with open(script['path'], 'r') as script_file:
                 es.put_script(id=script["name"],body=json.loads(script_file.read()))
 
-    #Load Watch and Execute
-    watcher = XPackClient(es).watcher
     with open(test['watch_file'],'r') as watch_file:
         watch=json.loads(watch_file.read())
-        watcher.put_watch(id=test["watch_name"],body=watch)
-        response=watcher.execute_watch(test["watch_name"])
-        #Confirm Matches
+        es.watcher.put_watch(id=test["watch_name"],body=watch)
+        response=es.watcher.execute_watch(id=test["watch_name"])
+
         match = test['match'] if 'match' in test else True
         print("Expected: Watch Condition: %s"%match)
         if not 'condition' in response['watch_record']['result']:
@@ -89,4 +84,4 @@ def find_item(list, key):
             sys.exit(1)
         else:
             print("TEST %s"%("PASS" if not response['watch_record']['result']['condition']['met'] else "FAIL"))
-            sys.exit(met)
+            sys.exit(met)