From dc189f17348c568a98af91d221fc68ef36bd579c Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Wed, 27 Feb 2019 15:44:07 -0500 Subject: [PATCH 01/28] Generate asciidoc for fields documentation - first draft --- scripts/generator.py | 2 + scripts/generators/asciidoc_fields.py | 162 ++++++++++++++++++++++++++ 2 files changed, 164 insertions(+) create mode 100644 scripts/generators/asciidoc_fields.py diff --git a/scripts/generator.py b/scripts/generator.py index 64d8073532..816cb8459e 100644 --- a/scripts/generator.py +++ b/scripts/generator.py @@ -5,6 +5,7 @@ from generators import csv_generator from generators import es_template from generators import beats +from generators import asciidoc_fields def main(): @@ -22,6 +23,7 @@ def main(): csv_generator.generate(ecs_flat, ecs_version) es_template.generate(ecs_flat, ecs_version) beats.generate(ecs_nested, ecs_version) + asciidoc_fields.generate(ecs_nested, ecs_version) def argument_parser(): diff --git a/scripts/generators/asciidoc_fields.py b/scripts/generators/asciidoc_fields.py new file mode 100644 index 0000000000..5e57c4b179 --- /dev/null +++ b/scripts/generators/asciidoc_fields.py @@ -0,0 +1,162 @@ +import sys + + +def generate(ecs_nested, ecs_version): + save_asciidoc('docs/fields.asciidoc', render_field_index(ecs_nested)) + save_asciidoc('docs/field-details.asciidoc', render_field_details(ecs_nested)) + +# Rendering + +# Field Index + + +def render_field_index(ecs_nested): + page_text = index_header() + for fieldset_name in sorted(ecs_nested): + page_text += render_index_row(ecs_nested[fieldset_name]) + # try: + # for fieldset_name in sorted(ecs_nested): + # print fieldset_name + # page_text += render_index_row(ecs_nested[fieldset_name]) + # except TypeError as ex: + # print ex + # print fieldset_name + # raise + page_text += table_footer() + page_text += index_footer() + return page_text + + +# def render_index_table(fieldset): +# fieldset_text = fieldset_header().format( +# fieldset_id= 'ecs-' + fieldset['name'], +# fieldset_description= fieldset['description'], +# fieldset_title= fieldset['title'] +# ) +# for field_name in sorted(fieldset['fields']): +# fieldset_text += render_field(fieldset['fields'][field_name]) +# fieldset_text += table_footer() +# return fieldset_text + + +def render_index_row(fieldset): + return index_row().format( + fieldset_id='ecs-' + fieldset['name'], + fieldset_title=fieldset['title'], + fieldset_short=fieldset.get('short', fieldset['description']) + ) + + +# Field Details + +def render_field_details(ecs_nested): + page_text = '' + for fieldset_name in sorted(ecs_nested): + page_text += render_fieldset(ecs_nested[fieldset_name]) + return page_text + + +def render_fieldset(fieldset): + fieldset_text = fieldset_header().format( + fieldset_id='ecs-' + fieldset['name'], + fieldset_description=fieldset['description'], + fieldset_title=fieldset['title'] + ) + for field_name in sorted(fieldset['fields']): + fieldset_text += render_field(fieldset['fields'][field_name]) + fieldset_text += table_footer() + return fieldset_text + + +def render_field(field): + example = '' + if 'example' in field: + example = 'example: ' + str(field['example']) + field_text = field_row().format( + field_name=field['flat_name'], + field_short=field['short'], + field_example=example, + field_level=field['level'], + field_type=field['type'], + ) + return field_text + +# Templates + + +def table_footer(): + return ''' +|===== +''' + + +# Field Index + + +def index_header(): + return ''' +[[ecs-fields]] +== {ecs} Fields + +[float] +[[ecs-fieldsets]] +=== Field Sets +[cols="<,<",options="header",] +|======================================================================= +| Field Set | Description +''' + + +def index_row(): + return ''' +| <<{fieldset_id},{fieldset_title}>> | {fieldset_short} +''' + + +def index_footer(): + return ''' +include::field-details.asciidoc[] +''' + + +# Field Details + + +def fieldset_header(): + return ''' +[[{fieldset_id}]] +=== {fieldset_title} fields + +{fieldset_description} + +[options="header"] +|===== +| Field | Description | Level / Type + +// =============================================================== +''' + + +def field_row(): + return ''' +| {field_name} +| {field_short} + +{field_example} + +| level: {field_level} + +type: {field_type} + +// =============================================================== +''' + +# File + + +def save_asciidoc(file, text): + open_mode = "wb" + if sys.version_info >= (3, 0): + open_mode = "w" + with open(file, open_mode) as outfile: + outfile.write(text) From abc83891e011e8d76e2737cb9187032411a8dbf7 Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Wed, 27 Feb 2019 15:45:09 -0500 Subject: [PATCH 02/28] Generated asciidoc files for fields --- docs/field-details.asciidoc | 2310 +++++++++++++++++++++++++++++++++++ docs/fields-gen.asciidoc | 429 ------- docs/fields.asciidoc | 102 +- 3 files changed, 2366 insertions(+), 475 deletions(-) create mode 100644 docs/field-details.asciidoc delete mode 100644 docs/fields-gen.asciidoc diff --git a/docs/field-details.asciidoc b/docs/field-details.asciidoc new file mode 100644 index 0000000000..128fb8a33d --- /dev/null +++ b/docs/field-details.asciidoc @@ -0,0 +1,2310 @@ + +[[ecs-agent]] +=== Agent fields + +The agent fields contain the data about the software entity, if any, that collects, detects, or observes events on a host, or takes measurements on a host. +Examples include Beats. Agents may also run on observers. ECS agent.* fields shall be populated with details of the agent running on the host or observer where the event happened or the measurement was taken. + +[options="header"] +|===== +| Field | Description | Level / Type + +// =============================================================== + +| agent.ephemeral_id +| Ephemeral identifier of this agent. + +example: 8a4f500f + +| level: extended + +type: keyword + +// =============================================================== + +| agent.id +| Unique identifier of this agent. + +example: 8a4f500d + +| level: core + +type: keyword + +// =============================================================== + +| agent.name +| Custom name of the agent. + +example: foo + +| level: core + +type: keyword + +// =============================================================== + +| agent.type +| Type of the agent. + +example: filebeat + +| level: core + +type: keyword + +// =============================================================== + +| agent.version +| Version of the agent. + +example: 6.0.0-rc2 + +| level: core + +type: keyword + +// =============================================================== + +|===== + +[[ecs-base]] +=== Base fields + +The `base` field set contains all fields which are on the top level. These fields are common across all types of events. + +[options="header"] +|===== +| Field | Description | Level / Type + +// =============================================================== + +| @timestamp +| Date/time when the event originated. + +example: 2016-05-23T08:05:34.853Z + +| level: core + +type: date + +// =============================================================== + +| labels +| Custom key/value pairs. + +example: {'application': 'foo-bar', 'env': 'production'} + +| level: core + +type: object + +// =============================================================== + +| message +| Log message optimized for viewing in a log viewer. + +example: Hello World + +| level: core + +type: text + +// =============================================================== + +| tags +| List of keywords used to tag each event. + +example: ["production", "env2"] + +| level: core + +type: keyword + +// =============================================================== + +|===== + +[[ecs-client]] +=== Client fields + +A client is defined as the initiator of a network connection for events regarding sessions, connections, or bidirectional flow records. +For TCP events, the client is the initiator of the TCP connection that sends the SYN packet(s). For other protocols, the client is generally the initiator or requestor in the network transaction. Some systems use the term "originator" to refer the client in TCP connections. The client fields describe details about the system acting as the client in the network event. Client fields are usually populated in conjunction with server fields. Client fields are generally not populated for packet-level events. +Client / server representations can add semantic context to an exchange, which is helpful to visualize the data in certain situations. If your context falls in that category, you should still ensure that source and destination are filled appropriately. + +[options="header"] +|===== +| Field | Description | Level / Type + +// =============================================================== + +| client.address +| Client network address. + + + +| level: extended + +type: keyword + +// =============================================================== + +| client.bytes +| Bytes sent from the client to the server. + +example: 184 + +| level: core + +type: long + +// =============================================================== + +| client.domain +| Client domain. + + + +| level: core + +type: keyword + +// =============================================================== + +| client.ip +| IP address of the client. + + + +| level: core + +type: ip + +// =============================================================== + +| client.mac +| MAC address of the client. + + + +| level: core + +type: keyword + +// =============================================================== + +| client.packets +| Packets sent from the client to the server. + +example: 12 + +| level: core + +type: long + +// =============================================================== + +| client.port +| Port of the client. + + + +| level: core + +type: long + +// =============================================================== + +|===== + +[[ecs-cloud]] +=== Cloud fields + +Fields related to the cloud or infrastructure the events are coming from. + +[options="header"] +|===== +| Field | Description | Level / Type + +// =============================================================== + +| cloud.account.id +| The cloud account or organization id. + +example: 666777888999 + +| level: extended + +type: keyword + +// =============================================================== + +| cloud.availability_zone +| Availability zone in which this host is running. + +example: us-east-1c + +| level: extended + +type: keyword + +// =============================================================== + +| cloud.instance.id +| Instance ID of the host machine. + +example: i-1234567890abcdef0 + +| level: extended + +type: keyword + +// =============================================================== + +| cloud.instance.name +| Instance name of the host machine. + + + +| level: extended + +type: keyword + +// =============================================================== + +| cloud.machine.type +| Machine type of the host machine. + +example: t2.medium + +| level: extended + +type: keyword + +// =============================================================== + +| cloud.provider +| Name of the cloud provider. + +example: ec2 + +| level: extended + +type: keyword + +// =============================================================== + +| cloud.region +| Region in which this host is running. + +example: us-east-1 + +| level: extended + +type: keyword + +// =============================================================== + +|===== + +[[ecs-container]] +=== Container fields + +Container fields are used for meta information about the specific container that is the source of information. +These fields help correlate data based containers from any runtime. + +[options="header"] +|===== +| Field | Description | Level / Type + +// =============================================================== + +| container.id +| Unique container id. + + + +| level: core + +type: keyword + +// =============================================================== + +| container.image.name +| Name of the image the container was built on. + + + +| level: extended + +type: keyword + +// =============================================================== + +| container.image.tag +| Container image tag. + + + +| level: extended + +type: keyword + +// =============================================================== + +| container.labels +| Image labels. + + + +| level: extended + +type: object + +// =============================================================== + +| container.name +| Container name. + + + +| level: extended + +type: keyword + +// =============================================================== + +| container.runtime +| Runtime managing this container. + +example: docker + +| level: extended + +type: keyword + +// =============================================================== + +|===== + +[[ecs-destination]] +=== Destination fields + +Destination fields describe details about the destination of a packet/event. +Destination fields are usually populated in conjunction with source fields. + +[options="header"] +|===== +| Field | Description | Level / Type + +// =============================================================== + +| destination.address +| Destination network address. + + + +| level: extended + +type: keyword + +// =============================================================== + +| destination.bytes +| Bytes sent from the destination to the source. + +example: 184 + +| level: core + +type: long + +// =============================================================== + +| destination.domain +| Destination domain. + + + +| level: core + +type: keyword + +// =============================================================== + +| destination.ip +| IP address of the destination. + + + +| level: core + +type: ip + +// =============================================================== + +| destination.mac +| MAC address of the destination. + + + +| level: core + +type: keyword + +// =============================================================== + +| destination.packets +| Packets sent from the destination to the source. + +example: 12 + +| level: core + +type: long + +// =============================================================== + +| destination.port +| Port of the destination. + + + +| level: core + +type: long + +// =============================================================== + +|===== + +[[ecs-ecs]] +=== ECS fields + +Meta-information specific to ECS. + +[options="header"] +|===== +| Field | Description | Level / Type + +// =============================================================== + +| ecs.version +| ECS version this event conforms to. + +example: 1.0.0-beta2 + +| level: core + +type: keyword + +// =============================================================== + +|===== + +[[ecs-error]] +=== Error fields + +These fields can represent errors of any kind. +Use them for errors that happen while fetching events or in cases where the event itself contains an error. + +[options="header"] +|===== +| Field | Description | Level / Type + +// =============================================================== + +| error.code +| Error code describing the error. + + + +| level: core + +type: keyword + +// =============================================================== + +| error.id +| Unique identifier for the error. + + + +| level: core + +type: keyword + +// =============================================================== + +| error.message +| Error message. + + + +| level: core + +type: text + +// =============================================================== + +|===== + +[[ecs-event]] +=== Event fields + +The event fields are used for context information about the log or metric event itself. +A log is defined as an event containing details of something that happened. Log events must include the time at which the thing happened. Examples of log events include a process starting on a host, a network packet being sent from a source to a destination, or a network connection between a client and a server being initiated or closed. A metric is defined as an event containing one or more numerical or categorical measurements and the time at which the measurement was taken. Examples of metric events include memory pressure measured on a host, or vulnerabilities measured on a scanned host. + +[options="header"] +|===== +| Field | Description | Level / Type + +// =============================================================== + +| event.action +| The action captured by the event. + +example: user-password-change + +| level: core + +type: keyword + +// =============================================================== + +| event.category +| Event category. + +example: user-management + +| level: core + +type: keyword + +// =============================================================== + +| event.created +| Time when the event was first read by an agent or by your pipeline. + + + +| level: core + +type: date + +// =============================================================== + +| event.dataset +| Name of the dataset. + +example: stats + +| level: core + +type: keyword + +// =============================================================== + +| event.duration +| Duration of the event in nanoseconds. + + + +| level: core + +type: long + +// =============================================================== + +| event.end +| event.end contains the date when the event ended or when the activity was last observed. + + + +| level: extended + +type: date + +// =============================================================== + +| event.hash +| Hash (perhaps logstash fingerprint) of raw field to be able to demonstrate log integrity. + +example: 123456789012345678901234567890ABCD + +| level: extended + +type: keyword + +// =============================================================== + +| event.id +| Unique ID to describe the event. + +example: 8a4f500d + +| level: core + +type: keyword + +// =============================================================== + +| event.kind +| The kind of the event. + +example: state + +| level: extended + +type: keyword + +// =============================================================== + +| event.module +| Name of the module this data is coming from. +This information is coming from the modules used in Beats or Logstash. + +example: mysql + +| level: core + +type: keyword + +// =============================================================== + +| event.original +| Raw text message of entire event. + +example: Sep 19 08:26:10 host CEF:0|Security| threatmanager|1.0|100| worm successfully stopped|10|src=10.0.0.1 dst=2.1.2.2spt=1232 + +| level: core + +type: keyword + +// =============================================================== + +| event.outcome +| The outcome of the event. + +example: success + +| level: extended + +type: keyword + +// =============================================================== + +| event.risk_score +| Risk score or priority of the event (e.g. security solutions). Use your system's original value here. + + + +| level: core + +type: float + +// =============================================================== + +| event.risk_score_norm +| Normalized risk score or priority of the event (0-100). + + + +| level: extended + +type: float + +// =============================================================== + +| event.severity +| Original severity of the event. + +example: 7 + +| level: core + +type: long + +// =============================================================== + +| event.start +| event.start contains the date when the event started or when the activity was first observed. + + + +| level: extended + +type: date + +// =============================================================== + +| event.timezone +| Event time zone. + + + +| level: extended + +type: keyword + +// =============================================================== + +| event.type +| Reserved for future usage. + + + +| level: core + +type: keyword + +// =============================================================== + +|===== + +[[ecs-file]] +=== File fields + +A file is defined as a set of information that has been created on, or has existed on a filesystem. +File objects can be associated with host events, network events, and/or file events (e.g., those produced by File Integrity Monitoring [FIM] products or services). File fields provide details about the affected file associated with the event or metric. + +[options="header"] +|===== +| Field | Description | Level / Type + +// =============================================================== + +| file.ctime +| Last time file metadata changed. + + + +| level: extended + +type: date + +// =============================================================== + +| file.device +| Device that is the source of the file. + + + +| level: extended + +type: keyword + +// =============================================================== + +| file.extension +| File extension. + +example: png + +| level: extended + +type: keyword + +// =============================================================== + +| file.gid +| Primary group ID (GID) of the file. + + + +| level: extended + +type: keyword + +// =============================================================== + +| file.group +| Primary group name of the file. + + + +| level: extended + +type: keyword + +// =============================================================== + +| file.inode +| Inode representing the file in the filesystem. + + + +| level: extended + +type: keyword + +// =============================================================== + +| file.mode +| Mode of the file in octal representation. + +example: 416 + +| level: extended + +type: keyword + +// =============================================================== + +| file.mtime +| Last time file content was modified. + + + +| level: extended + +type: date + +// =============================================================== + +| file.owner +| File owner's username. + + + +| level: extended + +type: keyword + +// =============================================================== + +| file.path +| Path to the file. + + + +| level: extended + +type: keyword + +// =============================================================== + +| file.size +| File size in bytes (field is only added when `type` is `file`). + + + +| level: extended + +type: long + +// =============================================================== + +| file.target_path +| Target path for symlinks. + + + +| level: extended + +type: keyword + +// =============================================================== + +| file.type +| File type (file, dir, or symlink). + + + +| level: extended + +type: keyword + +// =============================================================== + +| file.uid +| The user ID (UID) or security identifier (SID) of the file owner. + + + +| level: extended + +type: keyword + +// =============================================================== + +|===== + +[[ecs-geo]] +=== Geo fields + +Geo fields can carry data about a specific location related to an event. +This geolocation information can be derived from techniques such as Geo IP, or be user-supplied. + +[options="header"] +|===== +| Field | Description | Level / Type + +// =============================================================== + +| geo.city_name +| City name. + +example: Montreal + +| level: core + +type: keyword + +// =============================================================== + +| geo.continent_name +| Name of the continent. + +example: North America + +| level: core + +type: keyword + +// =============================================================== + +| geo.country_iso_code +| Country ISO code. + +example: CA + +| level: core + +type: keyword + +// =============================================================== + +| geo.country_name +| Country name. + +example: Canada + +| level: core + +type: keyword + +// =============================================================== + +| geo.location +| Longitude and latitude. + +example: { "lon": -73.614830, "lat": 45.505918 } + +| level: core + +type: geo_point + +// =============================================================== + +| geo.name +| User-defined description of a location. + +example: boston-dc + +| level: extended + +type: keyword + +// =============================================================== + +| geo.region_iso_code +| Region ISO code. + +example: CA-QC + +| level: core + +type: keyword + +// =============================================================== + +| geo.region_name +| Region name. + +example: Quebec + +| level: core + +type: keyword + +// =============================================================== + +|===== + +[[ecs-group]] +=== Group fields + +The group fields are meant to represent groups that are relevant to the event. + +[options="header"] +|===== +| Field | Description | Level / Type + +// =============================================================== + +| group.id +| Unique identifier for the group on the system/platform. + + + +| level: extended + +type: keyword + +// =============================================================== + +| group.name +| Name of the group. + + + +| level: extended + +type: keyword + +// =============================================================== + +|===== + +[[ecs-host]] +=== Host fields + +A host is defined as a general computing instance. +ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes. + +[options="header"] +|===== +| Field | Description | Level / Type + +// =============================================================== + +| host.architecture +| Operating system architecture. + +example: x86_64 + +| level: core + +type: keyword + +// =============================================================== + +| host.hostname +| Hostname of the host. + + + +| level: core + +type: keyword + +// =============================================================== + +| host.id +| Unique host id. + + + +| level: core + +type: keyword + +// =============================================================== + +| host.ip +| Host ip address. + + + +| level: core + +type: ip + +// =============================================================== + +| host.mac +| Host mac address. + + + +| level: core + +type: keyword + +// =============================================================== + +| host.name +| Name of the host. + + + +| level: core + +type: keyword + +// =============================================================== + +| host.type +| Type of host. + + + +| level: core + +type: keyword + +// =============================================================== + +|===== + +[[ecs-http]] +=== HTTP fields + +Fields related to HTTP activity. Use the `url` field set to store the url of the request. + +[options="header"] +|===== +| Field | Description | Level / Type + +// =============================================================== + +| http.request.body.bytes +| Size in bytes of the request body. + +example: 887 + +| level: extended + +type: long + +// =============================================================== + +| http.request.body.content +| The full HTTP request body. + +example: Hello world + +| level: extended + +type: keyword + +// =============================================================== + +| http.request.bytes +| Total size in bytes of the request (body and headers). + +example: 1437 + +| level: extended + +type: long + +// =============================================================== + +| http.request.method +| HTTP request method. + +example: get, post, put + +| level: extended + +type: keyword + +// =============================================================== + +| http.request.referrer +| Referrer for this HTTP request. + +example: https://blog.example.com/ + +| level: extended + +type: keyword + +// =============================================================== + +| http.response.body.bytes +| Size in bytes of the response body. + +example: 887 + +| level: extended + +type: long + +// =============================================================== + +| http.response.body.content +| The full HTTP response body. + +example: Hello world + +| level: extended + +type: keyword + +// =============================================================== + +| http.response.bytes +| Total size in bytes of the response (body and headers). + +example: 1437 + +| level: extended + +type: long + +// =============================================================== + +| http.response.status_code +| HTTP response status code. + +example: 404 + +| level: extended + +type: long + +// =============================================================== + +| http.version +| HTTP version. + +example: 1.1 + +| level: extended + +type: keyword + +// =============================================================== + +|===== + +[[ecs-log]] +=== Log fields + +Fields which are specific to log events. + +[options="header"] +|===== +| Field | Description | Level / Type + +// =============================================================== + +| log.level +| Log level of the log event. + +example: err + +| level: core + +type: keyword + +// =============================================================== + +| log.original +| Original log message with light interpretation only (encoding, newlines). + +example: Sep 19 08:26:10 localhost My log + +| level: core + +type: keyword + +// =============================================================== + +|===== + +[[ecs-network]] +=== Network fields + +The network is defined as the communication path over which a host or network event happens. +The network.* fields should be populated with details about the network activity associated with an event. + +[options="header"] +|===== +| Field | Description | Level / Type + +// =============================================================== + +| network.application +| Application level protocol name. + +example: aim + +| level: extended + +type: keyword + +// =============================================================== + +| network.bytes +| Total bytes transferred in both directions. + +example: 368 + +| level: core + +type: long + +// =============================================================== + +| network.community_id +| A hash of source and destination IPs and ports. + +example: 1:hO+sN4H+MG5MY/8hIrXPqc4ZQz0= + +| level: extended + +type: keyword + +// =============================================================== + +| network.direction +| Direction of the network traffic. + +example: inbound + +| level: core + +type: keyword + +// =============================================================== + +| network.forwarded_ip +| Host IP address when the source IP address is the proxy. + +example: 192.1.1.2 + +| level: core + +type: ip + +// =============================================================== + +| network.iana_number +| IANA Protocol Number. + +example: 6 + +| level: extended + +type: keyword + +// =============================================================== + +| network.name +| Name given by operators to sections of their network. + +example: Guest Wifi + +| level: extended + +type: keyword + +// =============================================================== + +| network.packets +| Total packets transferred in both directions. + +example: 24 + +| level: core + +type: long + +// =============================================================== + +| network.protocol +| L7 Network protocol name. + +example: http + +| level: core + +type: keyword + +// =============================================================== + +| network.transport +| Protocol Name corresponding to the field `iana_number`. + +example: tcp + +| level: core + +type: keyword + +// =============================================================== + +| network.type +| In the OSI Model this would be the Network Layer. ipv4, ipv6, ipsec, pim, etc + +example: ipv4 + +| level: core + +type: keyword + +// =============================================================== + +|===== + +[[ecs-observer]] +=== Observer fields + +An observer is defined as a special network, security, or application device used to detect, observe, or create network, security, or application-related events and metrics. +This could be a custom hardware appliance or a server that has been configured to run special network, security, or application software. Examples include firewalls, intrusion detection/prevention systems, network monitoring sensors, web application firewalls, data loss prevention systems, and APM servers. The observer.* fields shall be populated with details of the system, if any, that detects, observes and/or creates a network, security, or application event or metric. Message queues and ETL components used in processing events or metrics are not considered observers in ECS. + +[options="header"] +|===== +| Field | Description | Level / Type + +// =============================================================== + +| observer.hostname +| Hostname of the observer. + + + +| level: core + +type: keyword + +// =============================================================== + +| observer.ip +| IP address of the observer. + + + +| level: core + +type: ip + +// =============================================================== + +| observer.mac +| MAC address of the observer + + + +| level: core + +type: keyword + +// =============================================================== + +| observer.serial_number +| Observer serial number. + + + +| level: extended + +type: keyword + +// =============================================================== + +| observer.type +| The type of the observer the data is coming from. + +example: firewall + +| level: core + +type: keyword + +// =============================================================== + +| observer.vendor +| observer vendor information. + + + +| level: core + +type: keyword + +// =============================================================== + +| observer.version +| Observer version. + + + +| level: core + +type: keyword + +// =============================================================== + +|===== + +[[ecs-organization]] +=== Organization fields + +The organization fields enrich data with information about the company or entity the data is associated with. +These fields help you arrange or filter data stored in an index by one or multiple organizations. + +[options="header"] +|===== +| Field | Description | Level / Type + +// =============================================================== + +| organization.id +| Unique identifier for the organization. + + + +| level: extended + +type: keyword + +// =============================================================== + +| organization.name +| Organization name. + + + +| level: extended + +type: keyword + +// =============================================================== + +|===== + +[[ecs-os]] +=== Operating System fields + +The OS fields contain information about the operating system. + +[options="header"] +|===== +| Field | Description | Level / Type + +// =============================================================== + +| os.family +| OS family (such as redhat, debian, freebsd, windows). + +example: debian + +| level: extended + +type: keyword + +// =============================================================== + +| os.full +| Operating system name, including the version or code name. + +example: Mac OS Mojave + +| level: extended + +type: keyword + +// =============================================================== + +| os.kernel +| Operating system kernel version as a raw string. + +example: 4.4.0-112-generic + +| level: extended + +type: keyword + +// =============================================================== + +| os.name +| Operating system name, without the version. + +example: Mac OS X + +| level: extended + +type: keyword + +// =============================================================== + +| os.platform +| Operating system platform (such centos, ubuntu, windows). + +example: darwin + +| level: extended + +type: keyword + +// =============================================================== + +| os.version +| Operating system version as a raw string. + +example: 10.14.1 + +| level: extended + +type: keyword + +// =============================================================== + +|===== + +[[ecs-process]] +=== Process fields + +These fields contain information about a process. +These fields can help you correlate metrics information with a process id/name from a log message. The `process.pid` often stays in the metric itself and is copied to the global field for correlation. + +[options="header"] +|===== +| Field | Description | Level / Type + +// =============================================================== + +| process.args +| Array of process arguments. + +example: ['ssh', '-l', 'user', '10.0.0.16'] + +| level: extended + +type: keyword + +// =============================================================== + +| process.executable +| Absolute path to the process executable. + +example: /usr/bin/ssh + +| level: extended + +type: keyword + +// =============================================================== + +| process.name +| Process name. + +example: ssh + +| level: extended + +type: keyword + +// =============================================================== + +| process.pid +| Process id. + + + +| level: core + +type: long + +// =============================================================== + +| process.ppid +| Process parent id. + + + +| level: extended + +type: long + +// =============================================================== + +| process.start +| The time the process started. + +example: 2016-05-23T08:05:34.853Z + +| level: extended + +type: date + +// =============================================================== + +| process.thread.id +| Thread ID. + +example: 4242 + +| level: extended + +type: long + +// =============================================================== + +| process.title +| Process title. + + + +| level: extended + +type: keyword + +// =============================================================== + +| process.working_directory +| The working directory of the process. + +example: /home/alice + +| level: extended + +type: keyword + +// =============================================================== + +|===== + +[[ecs-related]] +=== Related fields + +This field set is meant to facilitate pivoting around a piece of data. +Some pieces of information can be seen in many places in an ECS event. To facilitate searching for them, store an array of all seen values to their corresponding field in `related.`. +A concrete example is IP addresses, which can be under host, observer, source, destination, client, server, and network.forwarded_ip. If you append all IPs to `related.ip`, you can then search for a given IP trivially, no matter where it appeared, by querying `related.ip:a.b.c.d`. + +[options="header"] +|===== +| Field | Description | Level / Type + +// =============================================================== + +| related.ip +| All of the IPs seen on your event. + + + +| level: extended + +type: ip + +// =============================================================== + +|===== + +[[ecs-server]] +=== Server fields + +A Server is defined as the responder in a network connection for events regarding sessions, connections, or bidirectional flow records. +For TCP events, the server is the receiver of the initial SYN packet(s) of the TCP connection. For other protocols, the server is generally the responder in the network transaction. Some systems actually use the term "responder" to refer the server in TCP connections. The server fields describe details about the system acting as the server in the network event. Server fields are usually populated in conjunction with client fields. Server fields are generally not populated for packet-level events. +Client / server representations can add semantic context to an exchange, which is helpful to visualize the data in certain situations. If your context falls in that category, you should still ensure that source and destination are filled appropriately. + +[options="header"] +|===== +| Field | Description | Level / Type + +// =============================================================== + +| server.address +| Server network address. + + + +| level: extended + +type: keyword + +// =============================================================== + +| server.bytes +| Bytes sent from the server to the client. + +example: 184 + +| level: core + +type: long + +// =============================================================== + +| server.domain +| Server domain. + + + +| level: core + +type: keyword + +// =============================================================== + +| server.ip +| IP address of the server. + + + +| level: core + +type: ip + +// =============================================================== + +| server.mac +| MAC address of the server. + + + +| level: core + +type: keyword + +// =============================================================== + +| server.packets +| Packets sent from the server to the client. + +example: 12 + +| level: core + +type: long + +// =============================================================== + +| server.port +| Port of the server. + + + +| level: core + +type: long + +// =============================================================== + +|===== + +[[ecs-service]] +=== Service fields + +The service fields describe the service for or from which the data was collected. +These fields help you find and correlate logs for a specific service and version. + +[options="header"] +|===== +| Field | Description | Level / Type + +// =============================================================== + +| service.ephemeral_id +| Ephemeral identifier of this service. + +example: 8a4f500f + +| level: extended + +type: keyword + +// =============================================================== + +| service.id +| Unique identifier of the running service. + +example: d37e5ebfe0ae6c4972dbe9f0174a1637bb8247f6 + +| level: core + +type: keyword + +// =============================================================== + +| service.name +| Name of the service. + +example: elasticsearch-metrics + +| level: core + +type: keyword + +// =============================================================== + +| service.state +| Current state of the service. + + + +| level: core + +type: keyword + +// =============================================================== + +| service.type +| The type of the service. + +example: elasticsearch + +| level: core + +type: keyword + +// =============================================================== + +| service.version +| Version of the service. + +example: 3.2.4 + +| level: core + +type: keyword + +// =============================================================== + +|===== + +[[ecs-source]] +=== Source fields + +Source fields describe details about the source of a packet/event. +Source fields are usually populated in conjunction with destination fields. + +[options="header"] +|===== +| Field | Description | Level / Type + +// =============================================================== + +| source.address +| Source network address. + + + +| level: extended + +type: keyword + +// =============================================================== + +| source.bytes +| Bytes sent from the source to the destination. + +example: 184 + +| level: core + +type: long + +// =============================================================== + +| source.domain +| Source domain. + + + +| level: core + +type: keyword + +// =============================================================== + +| source.ip +| IP address of the source. + + + +| level: core + +type: ip + +// =============================================================== + +| source.mac +| MAC address of the source. + + + +| level: core + +type: keyword + +// =============================================================== + +| source.packets +| Packets sent from the source to the destination. + +example: 12 + +| level: core + +type: long + +// =============================================================== + +| source.port +| Port of the source. + + + +| level: core + +type: long + +// =============================================================== + +|===== + +[[ecs-url]] +=== URL fields + +URL fields provide support for complete or partial URLs, and supports the breaking down into scheme, domain, path, and so on. + +[options="header"] +|===== +| Field | Description | Level / Type + +// =============================================================== + +| url.domain +| Domain of the url. + +example: www.elastic.co + +| level: extended + +type: keyword + +// =============================================================== + +| url.fragment +| Portion of the url after the `#`. + + + +| level: extended + +type: keyword + +// =============================================================== + +| url.full +| Full unparsed URL. + +example: https://www.elastic.co:443/search?q=elasticsearch#top + +| level: extended + +type: keyword + +// =============================================================== + +| url.original +| Unmodified original url as seen in the event source. + +example: https://www.elastic.co:443/search?q=elasticsearch#top or /search?q=elasticsearch + +| level: extended + +type: keyword + +// =============================================================== + +| url.password +| Password of the request. + + + +| level: extended + +type: keyword + +// =============================================================== + +| url.path +| Path of the request, such as "/search". + + + +| level: extended + +type: keyword + +// =============================================================== + +| url.port +| Port of the request, such as 443. + +example: 443 + +| level: extended + +type: integer + +// =============================================================== + +| url.query +| Query string of the request. + + + +| level: extended + +type: keyword + +// =============================================================== + +| url.scheme +| Scheme of the url. + +example: https + +| level: extended + +type: keyword + +// =============================================================== + +| url.username +| Username of the request. + + + +| level: extended + +type: keyword + +// =============================================================== + +|===== + +[[ecs-user]] +=== User fields + +The user fields describe information about the user that is relevant to the event. +Fields can have one entry or multiple entries. If a user has more than one id, provide an array that includes all of them. + +[options="header"] +|===== +| Field | Description | Level / Type + +// =============================================================== + +| user.email +| User email address. + + + +| level: extended + +type: keyword + +// =============================================================== + +| user.full_name +| User's full name, if available. + +example: Albert Einstein + +| level: extended + +type: keyword + +// =============================================================== + +| user.hash +| Unique user hash to correlate information for a user in anonymized form. + + + +| level: extended + +type: keyword + +// =============================================================== + +| user.id +| One or multiple unique identifiers of the user. + + + +| level: core + +type: keyword + +// =============================================================== + +| user.name +| Short name or login of the user. + +example: albert + +| level: core + +type: keyword + +// =============================================================== + +|===== + +[[ecs-user_agent]] +=== User agent fields + +The user_agent fields normally come from a browser request. +They often show up in web service logs coming from the parsed user agent string. + +[options="header"] +|===== +| Field | Description | Level / Type + +// =============================================================== + +| user_agent.device.name +| Name of the device. + +example: iPhone + +| level: extended + +type: keyword + +// =============================================================== + +| user_agent.name +| Name of the user agent. + +example: Safari + +| level: extended + +type: keyword + +// =============================================================== + +| user_agent.original +| Unparsed version of the user_agent. + +example: Mozilla/5.0 (iPhone; CPU iPhone OS 12_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.0 Mobile/15E148 Safari/604.1 + +| level: extended + +type: keyword + +// =============================================================== + +| user_agent.version +| Version of the user agent. + +example: 12.0 + +| level: extended + +type: keyword + +// =============================================================== + +|===== diff --git a/docs/fields-gen.asciidoc b/docs/fields-gen.asciidoc deleted file mode 100644 index 6d33b4febc..0000000000 --- a/docs/fields-gen.asciidoc +++ /dev/null @@ -1,429 +0,0 @@ -[[ecs-base]] -=== Base fields - -The `base` set contains top level fields that are common across all types of events -(such as `@timestamp`, `tags`, `message`, and `labels`). - - -[options="header"] -|===== -| Field | Description | Level - -// =============================================================== - -| [[@timestamp]] -| Date/time when the event originated. -For log events this is the date/time when the event was generated, and not when it was read. -Required field for all events. - -type: date - -Example: `2016-05-23T08:05:34.853Z` - -| core - -// =============================================================== - -| tags -| List of keywords used to tag each event. - -type: keyword - -Example:`["production", "env2"]` - -| core - -// =============================================================== - -| labels -| Key/value pairs. -Can be used to add meta information to events. Should not contain nested objects. -All values are stored as keyword. - -type: object - -Examples: `docker` and `k8s` labels. Examples: `{'application': 'foo-bar', 'env': 'production'}` - -| core - -// =============================================================== - -| message -| For log events the message field contains the log message. -In other use cases the message field can be used to concatenate different values -which are then freely searchable. If multiple messages exist, they can be -combined into one message. - -type: text - -Example: `Hello World` - -| core -|===== - - -[[ecs-agent]] -=== Agent fields - -The agent fields contain the data about the agent/client/shipper that created the event. - -[cols="<,<,<,<,<",options="header",] -|======================================================================= -| Field | Description | Level | Type | Example -| agent.version | Version of the agent. | core | keyword | `6.0.0-rc2` -| agent.name | Name of the agent. This is a name that can be given to an agent. This can be helpful if for example two Filebeat instances are running on the same host but a human readable separation is needed on which Filebeat instance data is coming from. If no name is given, the name is often left empty. | core | keyword | `foo` -| agent.type | Type of the agent. The agent type stays always the same and should be given by the agent used. In case of Filebeat the agent would always be Filebeat also if two Filebeat instances are run on the same machine. | core | keyword | `filebeat` -| agent.id | Unique identifier of this agent (if one exists). Example: For Beats this would be beat.id. | core | keyword | `8a4f500d` -| agent.ephemeral_id | Ephemeral identifier of this agent (if one exists). This id normally changes across restarts, but `agent.id` does not. | extended | keyword | `8a4f500f` -|======================================================================= - -Examples: In the case of Beats for logs, the agent.name is filebeat. For APM, it is the agent running in the app/service. The agent information does not change if data is sent through queuing systems like Kafka, Redis, or processing systems such as Logstash or APM Server. - -[[ecs-cloud]] -=== Cloud fields - -Fields related to the cloud or infrastructure the events are coming from. - -[cols="<,<,<,<,<",options="header",] -|======================================================================= -| Field | Description | Level | Type | Example -| cloud.provider | Name of the cloud provider. Example values are ec2, gce, or digitalocean. | extended | keyword | `ec2` -| cloud.availability_zone | Availability zone in which this host is running. | extended | keyword | `us-east-1c` -| cloud.region | Region in which this host is running. | extended | keyword | `us-east-1` -| cloud.instance.id | Instance ID of the host machine. | extended | keyword | `i-1234567890abcdef0` -| cloud.instance.name | Instance name of the host machine. | extended | keyword | -| cloud.machine.type | Machine type of the host machine. | extended | keyword | `t2.medium` -| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. - -Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | extended | keyword | `666777888999` -|======================================================================= - -Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on. - -[[ecs-container]] -=== Container fields - -Container fields are used for meta information about the specific container that -is the source of information. These fields help correlate data based containers -from any runtime. - -[cols="<,<,<,<,<",options="header",] -|======================================================================= -| Field | Description | Level | Type | Example -| container.runtime | Runtime managing this container. | extended | keyword | `docker` -| container.id | Unique container id. | core | keyword | -| container.image.name | Name of the image the container was built on. | extended | keyword | -| container.image.tag | Container image tag. | extended | keyword | -| container.name | Container name. | extended | keyword | -| container.labels | Image labels. | extended | object | -|======================================================================= - -[[ecs-destination]] -=== Destination fields - -Destination fields describe details about the destination of a packet/event. - -[cols="<,<,<,<,<",options="header",] -|======================================================================= -| Field | Description | Level | Type | Example -| destination.ip | IP address of the destination. Can be one or multiple IPv4 or IPv6 addresses. | core | ip | -| destination.port | Port of the destination. | core | long | -| destination.mac | MAC address of the destination. | core | keyword | -| destination.domain | Destination domain. | core | keyword | -|======================================================================= - -[[ecs-device]] -=== Device fields - -Device fields are used to provide additional information about the device that -is the source of the information. This could be a firewall, network device, etc. - -[cols="<,<,<,<,<",options="header",] -|======================================================================= -| Field | Description | Level | Type | Example -| device.mac | MAC address of the device | core | keyword | -| device.ip | IP address of the device. | core | ip | -| device.hostname | Hostname of the device. | core | keyword | -| device.vendor | Device vendor information. | core | keyword | -| device.version | Device version. | core | keyword | -| device.serial_number | Device serial number. | extended | keyword | -| device.type | The type of the device the data is coming from. There is no predefined list of device types. Some examples are `endpoint`, `firewall`, `ids`, `ips`, `proxy`. | core | keyword | `firewall` -|======================================================================= - -[[ecs-ecs]] -=== ECS fields - -Meta-information specific to ECS. - -[cols="<,<,<,<,<",options="header",] -|======================================================================= -| Field | Description | Level | Type | Example -| ecs.version | ECS version for this event. `ecs.version` is a required field and must exist in all events. - -Different indices may conform to different ECS versions. -This field helps integrations adjust to the correct schema version for events when you query across multiple indices. -The current version is 1.0.0-beta1 . | core | keyword | `1.0.0-beta1` -|======================================================================= - -// Can we make the version a VARIABLE so that it's always up-to-date? - -[[ecs-error]] -=== Error fields - -These fields can represent errors of any kind. Use them for errors that happen -while fetching events or in cases where the event itself contains an error. - -[cols="<,<,<,<,<",options="header",] -|======================================================================= -| Field | Description | Level | Type | Example -| error.id | Unique identifier for the error. | core | keyword | -| error.message | Error message. | core | text | -| error.code | Error code describing the error. | core | keyword | -|======================================================================= - - -[[ecs-event]] -=== Event fields - -The event fields are used for context information about the data itself. - -[cols="<,<,<,<,<",options="header",] -|======================================================================= -| Field | Description | Level | Type | Example -| event.id | Unique ID to describe the event. | core | keyword | `8a4f500d` -| event.kind | The kind of the event. This gives information about what type of information the event contains, without being specific to the contents of the event. Examples are `event`, `state`, `alarm`. Warning: In future versions of ECS, we plan to provide a list of acceptable values for this field, please use with caution. | extended | keyword | `state` -| event.category | Event category. This contains high-level information about the contents of the event. It is more generic than `event.action`, in the sense that typically a category contains multiple actions. Warning: In future versions of ECS, we plan to provide a list of acceptable values for this field, please use with caution. | core | keyword | `user-management` -| event.action | The action captured by the event. This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. | core | keyword | `user-password-change` -| event.outcome | The outcome of the event. If the event describes an action, this fields contains the outcome of that action. Examples outcomes are `success` and `failure`. Warning: In future versions of ECS, we plan to provide a list of acceptable values for this field, please use with caution. | extended | keyword | `success` -| event.type | Reserved for future usage. Please avoid using this field for user data. | core | keyword | -| event.module | Name of the module this data is coming from. This information is coming from the modules used in Beats or Logstash. | core | keyword | `mysql` -| event.dataset | Name of the dataset. The concept of a `dataset` (fileset / metricset) is used in Beats as a subset of modules. It contains the information which is currently stored in metricset.name and metricset.module or fileset.name. | core | keyword | `stats` -| event.severity | Severity describes the severity of the event. What the different severity values mean can very different between use cases. It's up to the implementer to make sure severities are consistent across events. | core | long | `7` -| event.original | Raw text message of entire event. Used to demonstrate log integrity. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. | core | (not indexed) | `Sep 19 08:26:10 host CEF:0|Security| threatmanager|1.0|100| worm successfully stopped|10|src=10.0.0.1 dst=2.1.2.2spt=1232` -| event.hash | Hash (perhaps logstash fingerprint) of raw field to be able to demonstrate log integrity. | extended | keyword | `123456789012345678901234567890ABCD` -| event.duration | Duration of the event in nanoseconds. If event.start and event.end are known this value should be the difference between the end and start time. | core | long | -| event.timezone | This field should be populated when the event's timestamp does not include timezone information already (e.g. default Syslog timestamps). It's optional otherwise. Acceptable timezone formats are: a canonical ID (e.g. "Europe/Amsterdam"), abbreviated (e.g. "EST") or an HH:mm differential (e.g. "-05:00"). | extended | keyword | -| event.created | event.created contains the date when the event was created. This timestamp is distinct from @timestamp in that @timestamp contains the processed timestamp. For logs these two timestamps can be different as the timestamp in the log line and when the event is read for example by Filebeat are not identical. `@timestamp` must contain the timestamp extracted from the log line, event.created when the log line is read. The same could apply to package capturing where @timestamp contains the timestamp extracted from the network package and event.created when the event was created. In case the two timestamps are identical, @timestamp should be used. | core | date | -| event.start | event.start contains the date when the event started or when the activity was first observed. | extended | date | -| event.end | event.end contains the date when the event ended or when the activity was last observed. | extended | date | -| event.risk_score | Risk score or priority of the event (e.g. security solutions). Use your system's original value here. | core | float | -| event.risk_score_norm | Normalized risk score or priority of the event, on a scale of 0 to 100. This is mainly useful if you use more than one system that assigns risk scores, and you want to see a normalized value across all systems. | extended | float | -|======================================================================= - -[[ecs-file]] -=== File fields - -File fields provide details about each file. - -[cols="<,<,<,<,<",options="header",] -|======================================================================= -| Field | Description | Level | Type | Example -| file.path | Path to the file. | extended | keyword | -| file.target_path | Target path for symlinks. | extended | keyword | -| file.extension | File extension. This should allow easy filtering by file extensions. | extended | keyword | `png` -| file.type | File type (file, dir, or symlink). | extended | keyword | -| file.device | Device that is the source of the file. | extended | keyword | -| file.inode | Inode representing the file in the filesystem. | extended | keyword | -| file.uid | The user ID (UID) or security identifier (SID) of the file owner. | extended | keyword | -| file.owner | File owner's username. | extended | keyword | -| file.gid | Primary group ID (GID) of the file. | extended | keyword | -| file.group | Primary group name of the file. | extended | keyword | -| file.mode | Mode of the file in octal representation. | extended | keyword | `416` | -| file.size | File size in bytes (field is only added when `type` is `file`). | extended | long | -| file.mtime | Last time file content was modified. | extended | date | -| file.ctime | Last time file metadata changed. | extended | date | -|======================================================================= - -[[ecs-geo]] -=== Geo fields - -Geo fields can carry data about a specific location related to an event or geo information derived from an IP field. - -The `geo` fields are expected to be nested at: `destination.geo`, `device.geo`, `host.geo`, `source.geo`. - -Note also that the `geo` fields are not expected to be used directly at the top level. - -[cols="<,<,<,<,<",options="header",] -|======================================================================= -| Field | Description | Level | Type | Example -| geo.continent_name | Name of the continent. | core | keyword | `North America` -| geo.country_iso_code | Country ISO code. | core | keyword | `CA` -| geo.location | Longitude and latitude. | core | geo_point | `{ "lon": -73.614830, "lat": 45.505918 }` -| geo.region_name | Region name. | core | keyword | `Quebec` -| geo.city_name | City name. | core | keyword | `Montreal` -|======================================================================= - -[[ecs-host]] -=== Host fields - -Host fields provide information related to a host. A host can be a physical -machine, a virtual machine, or a Docker container. Normally the host information -is related to the machine on which the event was generated/collected, but they -can be used differently if needed. - -[cols="<,<,<,<,<",options="header",] -|======================================================================= -| Field | Description | Level | Type | Example -| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | core | keyword | -| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | core | keyword | -| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | core | keyword | -| host.ip | Host ip address. | core | ip | -| host.mac | Host mac address. | core | keyword | -| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | core | keyword | -| host.architecture | Operating system architecture. | core | keyword | `x86_64` -|======================================================================= - -[[ecs-log]] -=== Log fields - -Fields which are specific to log events. - -[cols="<,<,<,<,<",options="header",] -|======================================================================= -| Field | Description | Level | Type | Example -| log.level | Log level of the log event. Some examples are `WARN`, `ERR`, `INFO`. | core | keyword | `ERR` | -| log.original | This is the original log message and contains the full log message before splitting it up in multiple parts. In contrast to the `message` field which can contain an extracted part of the log message, this field contains the original, full log message. It can have already some modifications applied like encoding or new lines removed to clean up the log message. This field is not indexed and doc_values are disabled so it can't be queried but the value can be retrieved from `_source`. | core | keyword | `Sep 19 08:26:10 localhost My log` | -|======================================================================= - -[[ecs-network]] -=== Network fields - -Fields related to network data. - -[cols="<,<,<,<,<",options="header",] -|======================================================================= -| Field | Description | Level | Type | Example -| network.name | Name given by operators to sections of their network. | extended | keyword | `Guest Wifi` -| network.type | In the OSI Model this would be the Network Layer. IPv4, IPv6, IPSec, PIM, etc | core | keyword | `IPv4` -| network.iana_number | IANA Protocol Number (https://www.iana.org/assignments/protocol-numbers/protocol-numbers.xhtml). Standardized list of protocols. This aligns well with NetFlow and sFlow related logs which use the IANA Protocol Number. | extended | keyword | `6` -| network.transport | Same as network.iana_number, but instead using the Keyword name of the transport layer (UDP, TCP, IPv6-ICMP, etc.) | core | keyword | `TCP` -| network.application | A name given to an application. This can be arbitrarily assigned for things like microservices, but also apply to things like skype, icq, facebook, twitter. This would be used in situations where the vendor or service can be decoded such as from the source/dest IP owners, ports, or wire format. | extended | keyword | `AIM` -| network.protocol | L7 Network protocol name. ex. http, lumberjack, transport protocol | core | keyword | `http` -| network.direction | Direction of the network traffic. Recommended values are: * inbound * outbound * unknown | core | keyword | `inbound` -| network.forwarded_ip | Host IP address when the source IP address is the proxy. | core | ip | `192.1.1.2` -| network.inbound.bytes | Network inbound bytes. | core | long | `184` -| network.inbound.packets | Network inbound packets. | core | long | `12` -| network.outbound.bytes | Network outbound bytes. | core | long | `184` -| network.outbound.packets | Network outbound packets. | core | long | `12` -| network.total.bytes | Network total bytes. The sum of inbound.bytes + outbound.bytes. | core | long | `368` -| network.total.packets | Network outbound packets. The sum of inbound.packets + outbound.packets | core | long | `24` -|======================================================================= - -[[ecs-organization]] -=== Organization fields - -The organization fields enrich data with information about the company or entity the data is associated with. These fields help you arrange or filter data stored in an index by one or multiple organizations. - -[cols="<,<,<,<,<",options="header",] -|======================================================================= -| Field | Description | Level | Type | Example -| organization.name | Organization name. | extended | keyword | -| organization.id | Unique identifier for the organization. | extended | keyword | -|======================================================================= - -[[ecs-os]] -=== Operating System fields - -The OS fields contain information about the operating system. - -The `os` fields are expected to be nested at: `device.os`, `host.os`, `user_agent.os`. -Note also that the `os` fields are not expected to be used directly at the top level. - -[cols="<,<,<,<,<",options="header",] -|======================================================================= -| Field | Description | Level | Type | Example -| os.platform | Operating system platform (such centos, ubuntu, windows). | extended | keyword | `darwin` -| os.name | Operating system name. | extended | keyword | `Mac OS X` -| os.family | OS family (such as redhat, debian, freebsd, windows). | extended | keyword | `debian` -| os.version | Operating system version as a raw string. | extended | keyword | `10.12.6-rc2` -| os.kernel | Operating system kernel version as a raw string. | extended | keyword | `4.4.0-112-generic` -|======================================================================= - -[[ecs-process]] -=== Process fields - -These fields contain information about a process. These fields can help you -correlate metrics information with a process id/name from a log message. The -`process.pid` often stays in the metric itself and is copied to the global field -for correlation. - -[cols="<,<,<,<,<",options="header",] -|======================================================================= -| Field | Description | Level | Type | Example -| process.args | Process arguments. May be filtered to protect sensitive information. | extended | keyword | `['-l', 'user', '10.0.0.16']` -| process.name | Process name. Sometimes called program name or similar. | extended | keyword | `ssh` -| process.pid | Process id. | core | long | -| process.ppid | Process parent id. | extended | long | -| process.title | Process title. The proctitle, some times the same as process name. Can also be different: for example a browser setting its title to the web page currently opened. | extended | keyword | -|======================================================================= - -[[ecs-service]] -=== Service fields - -The service fields describe the service for or from which the data was -collected. These fields help you find and correlate logs for a specific service -and version. - -[cols="<,<,<,<,<",options="header",] -|======================================================================= -| Field | Description | Level | Type | Example -| service.id | Unique identifier of the running service. This id should uniquely identify this service. This makes it possible to correlate logs and metrics for one specific service. Example: If you are experiencing issues with one redis instance, you can filter on that id to see metrics and logs for that single instance. | core | keyword | `d37e5ebfe0ae6c4972dbe9f0174a1637bb8247f6` -| service.name | Name of the service data is collected from. The name of the service is normally user given. This allows if two instances of the same service are running on the same machine they can be differentiated by the `service.name`. Also it allows for distributed services that run on multiple hosts to correlate the related instances based on the name. In the case of Elasticsearch the service.name could contain the cluster name. For Beats the service.name is by default a copy of the `service.type` field if no name is specified. | core | keyword | `elasticsearch-metrics` -| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | core | keyword | `elasticsearch` -| service.state | Current state of the service. | core | keyword | -| service.version | Version of the service the data was collected from. This allows to look at a data set only for a specific version of a service. | core | keyword | `3.2.4` -| service.ephemeral_id | Ephemeral identifier of this service (if one exists). This id normally changes across restarts, but `service.id` does not. | extended | keyword | `8a4f500f` -|======================================================================= - -[[ecs-source]] -=== Source fields - -Source fields describe details about the destination of a packet/event. - -[cols="<,<,<,<,<",options="header",] -|======================================================================= -| Field | Description | Level | Type | Example -| source.ip | IP address of the source. Can be one or multiple IPv4 or IPv6 addresses. | core | ip | -| source.port | Port of the source. | core | long | -| source.mac | MAC address of the source. | core | keyword | -| source.domain | Source domain. | core | keyword | -|======================================================================= - -[[ecs-url]] -=== URL fields - -URL fields provide a complete URL, with scheme, host, and path. - -[cols="<,<,<,<,<",options="header",] -|======================================================================= -| Field | Description | Level | Type | Example -| url.original | Full original url. The field is stored as keyword. | extended | keyword | `https://www.elastic.co:443/search?q=elasticsearch#top` -| url.scheme | Scheme of the request, such as "https". Note: The `:` is not part of the scheme. | extended | keyword | `https` -| url.domain | Domain of the request, such as "www.elastic.co". In some cases a URL may refer to an IP and/or port directly, without a domain name. In this case, the IP address would go to the `domain` field. | extended | keyword | `www.elastic.co` -| url.port | Port of the request, such as 443. | extended | integer | `443` -| url.path | Path of the request, such as "/search". | extended | keyword | -| url.query | The query field describes the query string of the request, such as "q=elasticsearch". The `?` is excluded from the query string. If a URL contains no `?`, there is no query field. If there is a `?` but no query, the query field exists with an empty string. The `exists` query can be used to differentiate between the two cases. | extended | keyword | -| url.fragment | Portion of the url after the `#`, such as "top". The `#` is not part of the fragment. | extended | keyword | -| url.username | Username of the request. | extended | keyword | -| url.password | Password of the request. | extended | keyword | -|======================================================================= - -[[ecs-user]] -=== User fields - -The user fields describe information about the user that is relevant to the event. Fields can have one entry or multiple entries. If a user has more than one id, provide an array that includes all of them. - -The `user` fields are expected to be nested at: `destination.user`, `host.user`, `source.user`. - -Note also that the `user` fields may be used directly at the top level. - -[cols="<,<,<,<,<",options="header",] -|======================================================================= -| Field | Description | Level | Type | Example -| user.id | One or multiple unique identifiers of the user. | core | keyword | -| user.name | Name of the user. The field is a keyword, and will not be tokenized. | core | keyword | -| user.email | User email address. | extended | keyword | -| user.hash | Unique user hash to correlate information for a user in anonymized form. Useful if `user.id` or `user.name` contain confidential information and cannot be used. | extended | keyword | -|======================================================================= - - diff --git a/docs/fields.asciidoc b/docs/fields.asciidoc index 29d85c04b0..7d16eb5e34 100644 --- a/docs/fields.asciidoc +++ b/docs/fields.asciidoc @@ -1,58 +1,68 @@ + [[ecs-fields]] == {ecs} Fields -// Set make to generate short description for use in the Fields overview section. -// Pull in generated field content using `include` statements - [float] -[[ecs-categories]] -=== Field categories +[[ecs-fieldsets]] +=== Field Sets [cols="<,<",options="header",] |======================================================================= -| Fields | Description -| <> | Top level fields that are common across all types of events -(such as `@timestamp`, `tags`, `message`, and `labels`). -| <> | The agent fields contain data about the -agent/client/shipper that created the event. -| <> | Fields related to the cloud or infrastructure the events are - coming from. -| <> | Container fields are used for meta information about the specific container that -is the source of information. These fields help correlate data based containers -from any runtime. -| <> | Destination fields describe details about the destination of a packet/event. -| <> | Device fields are used to provide additional information -about the device that is the source of the information. This could be a firewall, network device, etc. +| Field Set | Description + +| <> | Fields about the monitoring agent. + +| <> | All fields defined directly at the top level + +| <> | Fields about the client side of a network connection, used with server. + +| <> | Fields about the cloud resource. + +| <> | Fields describing the container that generated this event. + +| <> | Fields about the destination side of a network connection, used with source. + | <> | Meta-information specific to ECS. -| <> | These fields can represent errors of any kind. Use them for errors that happen -while fetching events or in cases where the event itself contains an error. -| <> | The event fields are used for context information about the data itself. -| <> | File fields provide details about each file. -| <> | Geo fields can carry data about a specific location related to -an event or geo information derived from an IP field. -The `geo` fields are expected to be nested at: `destination.geo`, `device.geo`, `host.geo`, `source.geo`. -| <> | Host fields provide information related to a host. A host can be a physical -machine, a virtual machine, or a Docker container. + +| <> | Fields about errors of any kind. + +| <> | Fields breaking down the event details. + +| <> | Fields describing files. + +| <> | Fields describing a location. + +| <> | User's group relevant to the event. + +| <> | Fields describing the relevant computing instance. + +| <> | Fields describing an HTTP request. + | <> | Fields which are specific to log events. -| <> | Fields related to network data. -| <> | The organization fields enrich data with - information about the company or entity the data is associated with. These fields help - you arrange or filter data stored in an index by one or multiple organizations. -| <> | The OS fields contain information about the operating system. -| <> | These fields contain information about a process. These fields can help you -correlate metrics information with a process id/name from a log message. The -`process.pid` often stays in the metric itself and is copied to the global field -for correlation. -| <> | The service fields describe the service for or from which the data was -collected. These fields help you find and correlate logs for a specific service -and version. -| <> | Source fields describe details about the destination of a packet/event. -| <> | URL fields provide a complete URL, with scheme, host, and path. -| <> | The user fields describe information about the user - that is relevant to the event. Fields can have one entry or multiple entries. - If a user has more than one id, provide an array that includes all of them. -|======================================================================= -include::fields-gen.asciidoc[] +| <> | Fields describing the communication path over which the event happened. + +| <> | Fields describing an entity observing the event from outside the host. + +| <> | Fields describing the organization or company the event is associated with. + +| <> | OS fields contain information about the operating system. + +| <> | These fields contain information about a process. + +| <> | Fields meant to facilitate pivoting around a piece of data. + +| <> | Fields about the server side of a network connection, used with client. + +| <> | Fields describing the service for or from which the data was collected. + +| <> | Fields about the source side of a network connection, used with destination. + +| <> | Fields that let you store URLs in various forms. + +| <> | Fields to describe the user relevant to the event. +| <> | Fields to describe a browser user_agent string. +|===== +include::field-details.asciidoc[] From aadd4416c983f2474dc9eee4633140f35dc3b483 Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Wed, 27 Feb 2019 16:15:49 -0500 Subject: [PATCH 03/28] Remove commented debugging code --- scripts/generators/asciidoc_fields.py | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/scripts/generators/asciidoc_fields.py b/scripts/generators/asciidoc_fields.py index 5e57c4b179..ae8e6fffbd 100644 --- a/scripts/generators/asciidoc_fields.py +++ b/scripts/generators/asciidoc_fields.py @@ -14,31 +14,11 @@ def render_field_index(ecs_nested): page_text = index_header() for fieldset_name in sorted(ecs_nested): page_text += render_index_row(ecs_nested[fieldset_name]) - # try: - # for fieldset_name in sorted(ecs_nested): - # print fieldset_name - # page_text += render_index_row(ecs_nested[fieldset_name]) - # except TypeError as ex: - # print ex - # print fieldset_name - # raise page_text += table_footer() page_text += index_footer() return page_text -# def render_index_table(fieldset): -# fieldset_text = fieldset_header().format( -# fieldset_id= 'ecs-' + fieldset['name'], -# fieldset_description= fieldset['description'], -# fieldset_title= fieldset['title'] -# ) -# for field_name in sorted(fieldset['fields']): -# fieldset_text += render_field(fieldset['fields'][field_name]) -# fieldset_text += table_footer() -# return fieldset_text - - def render_index_row(fieldset): return index_row().format( fieldset_id='ecs-' + fieldset['name'], From 1646cb09e691ae1d357309b96380187c7e4dd31c Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Thu, 28 Feb 2019 16:40:06 -0500 Subject: [PATCH 04/28] Sort fieldsets correctly in asciidoc. Base first, then the rest. --- docs/field-details.asciidoc | 94 +++++++++++++-------------- docs/fields.asciidoc | 4 +- scripts/generators/asciidoc_fields.py | 21 ++++-- scripts/tests/test_asciidoc_fields.py | 24 +++++++ 4 files changed, 90 insertions(+), 53 deletions(-) create mode 100644 scripts/tests/test_asciidoc_fields.py diff --git a/docs/field-details.asciidoc b/docs/field-details.asciidoc index 128fb8a33d..48fd72eca5 100644 --- a/docs/field-details.asciidoc +++ b/docs/field-details.asciidoc @@ -1,9 +1,8 @@ -[[ecs-agent]] -=== Agent fields +[[ecs-base]] +=== Base fields -The agent fields contain the data about the software entity, if any, that collects, detects, or observes events on a host, or takes measurements on a host. -Examples include Beats. Agents may also run on observers. ECS agent.* fields shall be populated with details of the agent running on the host or observer where the event happened or the measurement was taken. +The `base` field set contains all fields which are on the top level. These fields are common across all types of events. [options="header"] |===== @@ -11,43 +10,43 @@ Examples include Beats. Agents may also run on observers. ECS agent.* fields sha // =============================================================== -| agent.ephemeral_id -| Ephemeral identifier of this agent. +| @timestamp +| Date/time when the event originated. -example: 8a4f500f +example: 2016-05-23T08:05:34.853Z -| level: extended +| level: core -type: keyword +type: date // =============================================================== -| agent.id -| Unique identifier of this agent. +| labels +| Custom key/value pairs. -example: 8a4f500d +example: {'application': 'foo-bar', 'env': 'production'} | level: core -type: keyword +type: object // =============================================================== -| agent.name -| Custom name of the agent. +| message +| Log message optimized for viewing in a log viewer. -example: foo +example: Hello World | level: core -type: keyword +type: text // =============================================================== -| agent.type -| Type of the agent. +| tags +| List of keywords used to tag each event. -example: filebeat +example: ["production", "env2"] | level: core @@ -55,67 +54,68 @@ type: keyword // =============================================================== -| agent.version -| Version of the agent. +|===== -example: 6.0.0-rc2 +[[ecs-agent]] +=== Agent fields -| level: core +The agent fields contain the data about the software entity, if any, that collects, detects, or observes events on a host, or takes measurements on a host. +Examples include Beats. Agents may also run on observers. ECS agent.* fields shall be populated with details of the agent running on the host or observer where the event happened or the measurement was taken. -type: keyword +[options="header"] +|===== +| Field | Description | Level / Type // =============================================================== -|===== +| agent.ephemeral_id +| Ephemeral identifier of this agent. -[[ecs-base]] -=== Base fields +example: 8a4f500f -The `base` field set contains all fields which are on the top level. These fields are common across all types of events. +| level: extended -[options="header"] -|===== -| Field | Description | Level / Type +type: keyword // =============================================================== -| @timestamp -| Date/time when the event originated. +| agent.id +| Unique identifier of this agent. -example: 2016-05-23T08:05:34.853Z +example: 8a4f500d | level: core -type: date +type: keyword // =============================================================== -| labels -| Custom key/value pairs. +| agent.name +| Custom name of the agent. -example: {'application': 'foo-bar', 'env': 'production'} +example: foo | level: core -type: object +type: keyword // =============================================================== -| message -| Log message optimized for viewing in a log viewer. +| agent.type +| Type of the agent. -example: Hello World +example: filebeat | level: core -type: text +type: keyword // =============================================================== -| tags -| List of keywords used to tag each event. +| agent.version +| Version of the agent. -example: ["production", "env2"] +example: 6.0.0-rc2 | level: core diff --git a/docs/fields.asciidoc b/docs/fields.asciidoc index 7d16eb5e34..833d1aaf00 100644 --- a/docs/fields.asciidoc +++ b/docs/fields.asciidoc @@ -9,10 +9,10 @@ |======================================================================= | Field Set | Description -| <> | Fields about the monitoring agent. - | <> | All fields defined directly at the top level +| <> | Fields about the monitoring agent. + | <> | Fields about the client side of a network connection, used with server. | <> | Fields about the cloud resource. diff --git a/scripts/generators/asciidoc_fields.py b/scripts/generators/asciidoc_fields.py index ae8e6fffbd..c23d128dc7 100644 --- a/scripts/generators/asciidoc_fields.py +++ b/scripts/generators/asciidoc_fields.py @@ -5,6 +5,17 @@ def generate(ecs_nested, ecs_version): save_asciidoc('docs/fields.asciidoc', render_field_index(ecs_nested)) save_asciidoc('docs/field-details.asciidoc', render_field_details(ecs_nested)) +# Helpers + + +def sorted_by_group(dict): + triples = [] + for key in dict: + nested = dict[key] + triples += [(nested['group'], key, nested)] + return sorted(triples) + + # Rendering # Field Index @@ -12,8 +23,9 @@ def generate(ecs_nested, ecs_version): def render_field_index(ecs_nested): page_text = index_header() - for fieldset_name in sorted(ecs_nested): - page_text += render_index_row(ecs_nested[fieldset_name]) + for triple in sorted_by_group(ecs_nested): + (group, fieldset_name, fieldset) = triple + page_text += render_index_row(fieldset) page_text += table_footer() page_text += index_footer() return page_text @@ -31,8 +43,9 @@ def render_index_row(fieldset): def render_field_details(ecs_nested): page_text = '' - for fieldset_name in sorted(ecs_nested): - page_text += render_fieldset(ecs_nested[fieldset_name]) + for triple in sorted_by_group(ecs_nested): + (group, fieldset_name, fieldset) = triple + page_text += render_fieldset(fieldset) return page_text diff --git a/scripts/tests/test_asciidoc_fields.py b/scripts/tests/test_asciidoc_fields.py new file mode 100644 index 0000000000..1df32ee543 --- /dev/null +++ b/scripts/tests/test_asciidoc_fields.py @@ -0,0 +1,24 @@ +import unittest +from scripts.generators import asciidoc_fields + + +class TestGeneratorsAsciidocFields(unittest.TestCase): + # dict_add_nested + + def test_sorted_by_group(self): + dict = { + 'agent': { 'group': 2 }, + 'base': { 'group': 1 }, + 'cloud': { 'group': 2 } + } + triples = asciidoc_fields.sorted_by_group(dict) + expected_triples = [ + (1, 'base', { 'group': 1 }), + (2, 'agent', { 'group': 2 }), + (2, 'cloud', { 'group': 2 }) + ] + self.assertEqual(triples, expected_triples) + + +if __name__ == '__main__': + unittest.main() From 0908e4062bc10e1c794dc2c06371ad13f9f1f257 Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Thu, 28 Feb 2019 16:47:19 -0500 Subject: [PATCH 05/28] Preserve initial order of fields in key 'order' --- docs/field-details.asciidoc | 1517 +++++++++++++++++++++++++++---- generated/ecs/fields_flat.yml | 279 ++++++ generated/ecs/fields_nested.yml | 279 ++++++ scripts/schema_reader.py | 3 +- 4 files changed, 1885 insertions(+), 193 deletions(-) diff --git a/docs/field-details.asciidoc b/docs/field-details.asciidoc index 48fd72eca5..1f2948eee8 100644 --- a/docs/field-details.asciidoc +++ b/docs/field-details.asciidoc @@ -165,6 +165,94 @@ type: long +| level: core + +type: keyword + +// =============================================================== + +| client.geo.city_name +| City name. + +example: Montreal + +| level: core + +type: keyword + +// =============================================================== + +| client.geo.continent_name +| Name of the continent. + +example: North America + +| level: core + +type: keyword + +// =============================================================== + +| client.geo.country_iso_code +| Country ISO code. + +example: CA + +| level: core + +type: keyword + +// =============================================================== + +| client.geo.country_name +| Country name. + +example: Canada + +| level: core + +type: keyword + +// =============================================================== + +| client.geo.location +| Longitude and latitude. + +example: { "lon": -73.614830, "lat": 45.505918 } + +| level: core + +type: geo_point + +// =============================================================== + +| client.geo.name +| User-defined description of a location. + +example: boston-dc + +| level: extended + +type: keyword + +// =============================================================== + +| client.geo.region_iso_code +| Region ISO code. + +example: CA-QC + +| level: core + +type: keyword + +// =============================================================== + +| client.geo.region_name +| Region name. + +example: Quebec + | level: core type: keyword @@ -215,6 +303,83 @@ type: long // =============================================================== +| client.user.email +| User email address. + + + +| level: extended + +type: keyword + +// =============================================================== + +| client.user.full_name +| User's full name, if available. + +example: Albert Einstein + +| level: extended + +type: keyword + +// =============================================================== + +| client.user.group.id +| Unique identifier for the group on the system/platform. + + + +| level: extended + +type: keyword + +// =============================================================== + +| client.user.group.name +| Name of the group. + + + +| level: extended + +type: keyword + +// =============================================================== + +| client.user.hash +| Unique user hash to correlate information for a user in anonymized form. + + + +| level: extended + +type: keyword + +// =============================================================== + +| client.user.id +| One or multiple unique identifiers of the user. + + + +| level: core + +type: keyword + +// =============================================================== + +| client.user.name +| Short name or login of the user. + +example: albert + +| level: core + +type: keyword + +// =============================================================== + |===== [[ecs-cloud]] @@ -286,7 +451,7 @@ type: keyword | cloud.provider | Name of the cloud provider. -example: ec2 +example: aws | level: extended @@ -432,21 +597,21 @@ type: keyword // =============================================================== -| destination.ip -| IP address of the destination. - +| destination.geo.city_name +| City name. +example: Montreal | level: core -type: ip +type: keyword // =============================================================== -| destination.mac -| MAC address of the destination. - +| destination.geo.continent_name +| Name of the continent. +example: North America | level: core @@ -454,70 +619,65 @@ type: keyword // =============================================================== -| destination.packets -| Packets sent from the destination to the source. +| destination.geo.country_iso_code +| Country ISO code. -example: 12 +example: CA | level: core -type: long +type: keyword // =============================================================== -| destination.port -| Port of the destination. - +| destination.geo.country_name +| Country name. +example: Canada | level: core -type: long +type: keyword // =============================================================== -|===== +| destination.geo.location +| Longitude and latitude. -[[ecs-ecs]] -=== ECS fields +example: { "lon": -73.614830, "lat": 45.505918 } -Meta-information specific to ECS. +| level: core -[options="header"] -|===== -| Field | Description | Level / Type +type: geo_point // =============================================================== -| ecs.version -| ECS version this event conforms to. +| destination.geo.name +| User-defined description of a location. -example: 1.0.0-beta2 +example: boston-dc -| level: core +| level: extended type: keyword // =============================================================== -|===== +| destination.geo.region_iso_code +| Region ISO code. -[[ecs-error]] -=== Error fields +example: CA-QC -These fields can represent errors of any kind. -Use them for errors that happen while fetching events or in cases where the event itself contains an error. +| level: core -[options="header"] -|===== -| Field | Description | Level / Type +type: keyword // =============================================================== -| error.code -| Error code describing the error. - +| destination.geo.region_name +| Region name. +example: Quebec | level: core @@ -525,70 +685,240 @@ type: keyword // =============================================================== -| error.id -| Unique identifier for the error. +| destination.ip +| IP address of the destination. | level: core -type: keyword +type: ip // =============================================================== -| error.message -| Error message. +| destination.mac +| MAC address of the destination. | level: core -type: text +type: keyword // =============================================================== -|===== +| destination.packets +| Packets sent from the destination to the source. -[[ecs-event]] -=== Event fields +example: 12 -The event fields are used for context information about the log or metric event itself. -A log is defined as an event containing details of something that happened. Log events must include the time at which the thing happened. Examples of log events include a process starting on a host, a network packet being sent from a source to a destination, or a network connection between a client and a server being initiated or closed. A metric is defined as an event containing one or more numerical or categorical measurements and the time at which the measurement was taken. Examples of metric events include memory pressure measured on a host, or vulnerabilities measured on a scanned host. +| level: core -[options="header"] -|===== -| Field | Description | Level / Type +type: long // =============================================================== -| event.action -| The action captured by the event. +| destination.port +| Port of the destination. + -example: user-password-change | level: core -type: keyword +type: long // =============================================================== -| event.category -| Event category. +| destination.user.email +| User email address. -example: user-management -| level: core + +| level: extended type: keyword // =============================================================== -| event.created -| Time when the event was first read by an agent or by your pipeline. +| destination.user.full_name +| User's full name, if available. +example: Albert Einstein +| level: extended -| level: core +type: keyword + +// =============================================================== + +| destination.user.group.id +| Unique identifier for the group on the system/platform. + + + +| level: extended + +type: keyword + +// =============================================================== + +| destination.user.group.name +| Name of the group. + + + +| level: extended + +type: keyword + +// =============================================================== + +| destination.user.hash +| Unique user hash to correlate information for a user in anonymized form. + + + +| level: extended + +type: keyword + +// =============================================================== + +| destination.user.id +| One or multiple unique identifiers of the user. + + + +| level: core + +type: keyword + +// =============================================================== + +| destination.user.name +| Short name or login of the user. + +example: albert + +| level: core + +type: keyword + +// =============================================================== + +|===== + +[[ecs-ecs]] +=== ECS fields + +Meta-information specific to ECS. + +[options="header"] +|===== +| Field | Description | Level / Type + +// =============================================================== + +| ecs.version +| ECS version this event conforms to. + +example: 1.0.0 + +| level: core + +type: keyword + +// =============================================================== + +|===== + +[[ecs-error]] +=== Error fields + +These fields can represent errors of any kind. +Use them for errors that happen while fetching events or in cases where the event itself contains an error. + +[options="header"] +|===== +| Field | Description | Level / Type + +// =============================================================== + +| error.code +| Error code describing the error. + + + +| level: core + +type: keyword + +// =============================================================== + +| error.id +| Unique identifier for the error. + + + +| level: core + +type: keyword + +// =============================================================== + +| error.message +| Error message. + + + +| level: core + +type: text + +// =============================================================== + +|===== + +[[ecs-event]] +=== Event fields + +The event fields are used for context information about the log or metric event itself. +A log is defined as an event containing details of something that happened. Log events must include the time at which the thing happened. Examples of log events include a process starting on a host, a network packet being sent from a source to a destination, or a network connection between a client and a server being initiated or closed. A metric is defined as an event containing one or more numerical or categorical measurements and the time at which the measurement was taken. Examples of metric events include memory pressure measured on a host, or vulnerabilities measured on a scanned host. + +[options="header"] +|===== +| Field | Description | Level / Type + +// =============================================================== + +| event.action +| The action captured by the event. + +example: user-password-change + +| level: core + +type: keyword + +// =============================================================== + +| event.category +| Event category. + +example: user-management + +| level: core + +type: keyword + +// =============================================================== + +| event.created +| Time when the event was first read by an agent or by your pipeline. + + + +| level: core type: date @@ -1090,10 +1420,10 @@ type: keyword // =============================================================== -| host.hostname -| Hostname of the host. - +| host.geo.city_name +| City name. +example: Montreal | level: core @@ -1101,10 +1431,10 @@ type: keyword // =============================================================== -| host.id -| Unique host id. - +| host.geo.continent_name +| Name of the continent. +example: North America | level: core @@ -1112,21 +1442,21 @@ type: keyword // =============================================================== -| host.ip -| Host ip address. - +| host.geo.country_iso_code +| Country ISO code. +example: CA | level: core -type: ip +type: keyword // =============================================================== -| host.mac -| Host mac address. - +| host.geo.country_name +| Country name. +example: Canada | level: core @@ -1134,91 +1464,322 @@ type: keyword // =============================================================== -| host.name -| Name of the host. - +| host.geo.location +| Longitude and latitude. +example: { "lon": -73.614830, "lat": 45.505918 } | level: core -type: keyword +type: geo_point // =============================================================== -| host.type -| Type of host. - +| host.geo.name +| User-defined description of a location. +example: boston-dc -| level: core +| level: extended type: keyword // =============================================================== -|===== +| host.geo.region_iso_code +| Region ISO code. -[[ecs-http]] -=== HTTP fields +example: CA-QC -Fields related to HTTP activity. Use the `url` field set to store the url of the request. +| level: core -[options="header"] -|===== -| Field | Description | Level / Type +type: keyword // =============================================================== -| http.request.body.bytes -| Size in bytes of the request body. +| host.geo.region_name +| Region name. -example: 887 +example: Quebec -| level: extended +| level: core -type: long +type: keyword // =============================================================== -| http.request.body.content -| The full HTTP request body. +| host.hostname +| Hostname of the host. -example: Hello world -| level: extended + +| level: core type: keyword // =============================================================== -| http.request.bytes -| Total size in bytes of the request (body and headers). +| host.id +| Unique host id. -example: 1437 -| level: extended -type: long +| level: core + +type: keyword // =============================================================== -| http.request.method -| HTTP request method. +| host.ip +| Host ip address. -example: get, post, put -| level: extended -type: keyword +| level: core + +type: ip // =============================================================== -| http.request.referrer -| Referrer for this HTTP request. +| host.mac +| Host mac address. -example: https://blog.example.com/ -| level: extended + +| level: core + +type: keyword + +// =============================================================== + +| host.name +| Name of the host. + + + +| level: core + +type: keyword + +// =============================================================== + +| host.os.family +| OS family (such as redhat, debian, freebsd, windows). + +example: debian + +| level: extended + +type: keyword + +// =============================================================== + +| host.os.full +| Operating system name, including the version or code name. + +example: Mac OS Mojave + +| level: extended + +type: keyword + +// =============================================================== + +| host.os.kernel +| Operating system kernel version as a raw string. + +example: 4.4.0-112-generic + +| level: extended + +type: keyword + +// =============================================================== + +| host.os.name +| Operating system name, without the version. + +example: Mac OS X + +| level: extended + +type: keyword + +// =============================================================== + +| host.os.platform +| Operating system platform (such centos, ubuntu, windows). + +example: darwin + +| level: extended + +type: keyword + +// =============================================================== + +| host.os.version +| Operating system version as a raw string. + +example: 10.14.1 + +| level: extended + +type: keyword + +// =============================================================== + +| host.type +| Type of host. + + + +| level: core + +type: keyword + +// =============================================================== + +| host.user.email +| User email address. + + + +| level: extended + +type: keyword + +// =============================================================== + +| host.user.full_name +| User's full name, if available. + +example: Albert Einstein + +| level: extended + +type: keyword + +// =============================================================== + +| host.user.group.id +| Unique identifier for the group on the system/platform. + + + +| level: extended + +type: keyword + +// =============================================================== + +| host.user.group.name +| Name of the group. + + + +| level: extended + +type: keyword + +// =============================================================== + +| host.user.hash +| Unique user hash to correlate information for a user in anonymized form. + + + +| level: extended + +type: keyword + +// =============================================================== + +| host.user.id +| One or multiple unique identifiers of the user. + + + +| level: core + +type: keyword + +// =============================================================== + +| host.user.name +| Short name or login of the user. + +example: albert + +| level: core + +type: keyword + +// =============================================================== + +|===== + +[[ecs-http]] +=== HTTP fields + +Fields related to HTTP activity. Use the `url` field set to store the url of the request. + +[options="header"] +|===== +| Field | Description | Level / Type + +// =============================================================== + +| http.request.body.bytes +| Size in bytes of the request body. + +example: 887 + +| level: extended + +type: long + +// =============================================================== + +| http.request.body.content +| The full HTTP request body. + +example: Hello world + +| level: extended + +type: keyword + +// =============================================================== + +| http.request.bytes +| Total size in bytes of the request (body and headers). + +example: 1437 + +| level: extended + +type: long + +// =============================================================== + +| http.request.method +| HTTP request method. + +example: get, post, put + +| level: extended + +type: keyword + +// =============================================================== + +| http.request.referrer +| Referrer for this HTTP request. + +example: https://blog.example.com/ + +| level: extended type: keyword @@ -1339,21 +1900,189 @@ type: keyword // =============================================================== -| network.bytes -| Total bytes transferred in both directions. +| network.bytes +| Total bytes transferred in both directions. + +example: 368 + +| level: core + +type: long + +// =============================================================== + +| network.community_id +| A hash of source and destination IPs and ports. + +example: 1:hO+sN4H+MG5MY/8hIrXPqc4ZQz0= + +| level: extended + +type: keyword + +// =============================================================== + +| network.direction +| Direction of the network traffic. + +example: inbound + +| level: core + +type: keyword + +// =============================================================== + +| network.forwarded_ip +| Host IP address when the source IP address is the proxy. + +example: 192.1.1.2 + +| level: core + +type: ip + +// =============================================================== + +| network.iana_number +| IANA Protocol Number. + +example: 6 + +| level: extended + +type: keyword + +// =============================================================== + +| network.name +| Name given by operators to sections of their network. + +example: Guest Wifi + +| level: extended + +type: keyword + +// =============================================================== + +| network.packets +| Total packets transferred in both directions. + +example: 24 + +| level: core + +type: long + +// =============================================================== + +| network.protocol +| L7 Network protocol name. + +example: http + +| level: core + +type: keyword + +// =============================================================== + +| network.transport +| Protocol Name corresponding to the field `iana_number`. + +example: tcp + +| level: core + +type: keyword + +// =============================================================== + +| network.type +| In the OSI Model this would be the Network Layer. ipv4, ipv6, ipsec, pim, etc + +example: ipv4 + +| level: core + +type: keyword + +// =============================================================== + +|===== + +[[ecs-observer]] +=== Observer fields + +An observer is defined as a special network, security, or application device used to detect, observe, or create network, security, or application-related events and metrics. +This could be a custom hardware appliance or a server that has been configured to run special network, security, or application software. Examples include firewalls, intrusion detection/prevention systems, network monitoring sensors, web application firewalls, data loss prevention systems, and APM servers. The observer.* fields shall be populated with details of the system, if any, that detects, observes and/or creates a network, security, or application event or metric. Message queues and ETL components used in processing events or metrics are not considered observers in ECS. + +[options="header"] +|===== +| Field | Description | Level / Type + +// =============================================================== + +| observer.geo.city_name +| City name. + +example: Montreal + +| level: core + +type: keyword + +// =============================================================== + +| observer.geo.continent_name +| Name of the continent. + +example: North America + +| level: core + +type: keyword + +// =============================================================== + +| observer.geo.country_iso_code +| Country ISO code. + +example: CA + +| level: core + +type: keyword + +// =============================================================== + +| observer.geo.country_name +| Country name. + +example: Canada + +| level: core + +type: keyword + +// =============================================================== + +| observer.geo.location +| Longitude and latitude. -example: 368 +example: { "lon": -73.614830, "lat": 45.505918 } | level: core -type: long +type: geo_point // =============================================================== -| network.community_id -| A hash of source and destination IPs and ports. +| observer.geo.name +| User-defined description of a location. -example: 1:hO+sN4H+MG5MY/8hIrXPqc4ZQz0= +example: boston-dc | level: extended @@ -1361,10 +2090,10 @@ type: keyword // =============================================================== -| network.direction -| Direction of the network traffic. +| observer.geo.region_iso_code +| Region ISO code. -example: inbound +example: CA-QC | level: core @@ -1372,54 +2101,43 @@ type: keyword // =============================================================== -| network.forwarded_ip -| Host IP address when the source IP address is the proxy. +| observer.geo.region_name +| Region name. -example: 192.1.1.2 +example: Quebec | level: core -type: ip - -// =============================================================== - -| network.iana_number -| IANA Protocol Number. - -example: 6 - -| level: extended - type: keyword // =============================================================== -| network.name -| Name given by operators to sections of their network. +| observer.hostname +| Hostname of the observer. -example: Guest Wifi -| level: extended + +| level: core type: keyword // =============================================================== -| network.packets -| Total packets transferred in both directions. +| observer.ip +| IP address of the observer. + -example: 24 | level: core -type: long +type: ip // =============================================================== -| network.protocol -| L7 Network protocol name. +| observer.mac +| MAC address of the observer + -example: http | level: core @@ -1427,70 +2145,67 @@ type: keyword // =============================================================== -| network.transport -| Protocol Name corresponding to the field `iana_number`. +| observer.os.family +| OS family (such as redhat, debian, freebsd, windows). -example: tcp +example: debian -| level: core +| level: extended type: keyword // =============================================================== -| network.type -| In the OSI Model this would be the Network Layer. ipv4, ipv6, ipsec, pim, etc +| observer.os.full +| Operating system name, including the version or code name. -example: ipv4 +example: Mac OS Mojave -| level: core +| level: extended type: keyword // =============================================================== -|===== +| observer.os.kernel +| Operating system kernel version as a raw string. -[[ecs-observer]] -=== Observer fields +example: 4.4.0-112-generic -An observer is defined as a special network, security, or application device used to detect, observe, or create network, security, or application-related events and metrics. -This could be a custom hardware appliance or a server that has been configured to run special network, security, or application software. Examples include firewalls, intrusion detection/prevention systems, network monitoring sensors, web application firewalls, data loss prevention systems, and APM servers. The observer.* fields shall be populated with details of the system, if any, that detects, observes and/or creates a network, security, or application event or metric. Message queues and ETL components used in processing events or metrics are not considered observers in ECS. +| level: extended -[options="header"] -|===== -| Field | Description | Level / Type +type: keyword // =============================================================== -| observer.hostname -| Hostname of the observer. - +| observer.os.name +| Operating system name, without the version. +example: Mac OS X -| level: core +| level: extended type: keyword // =============================================================== -| observer.ip -| IP address of the observer. - +| observer.os.platform +| Operating system platform (such centos, ubuntu, windows). +example: darwin -| level: core +| level: extended -type: ip +type: keyword // =============================================================== -| observer.mac -| MAC address of the observer - +| observer.os.version +| Operating system version as a raw string. +example: 10.14.1 -| level: core +| level: extended type: keyword @@ -1836,6 +2551,94 @@ type: long +| level: core + +type: keyword + +// =============================================================== + +| server.geo.city_name +| City name. + +example: Montreal + +| level: core + +type: keyword + +// =============================================================== + +| server.geo.continent_name +| Name of the continent. + +example: North America + +| level: core + +type: keyword + +// =============================================================== + +| server.geo.country_iso_code +| Country ISO code. + +example: CA + +| level: core + +type: keyword + +// =============================================================== + +| server.geo.country_name +| Country name. + +example: Canada + +| level: core + +type: keyword + +// =============================================================== + +| server.geo.location +| Longitude and latitude. + +example: { "lon": -73.614830, "lat": 45.505918 } + +| level: core + +type: geo_point + +// =============================================================== + +| server.geo.name +| User-defined description of a location. + +example: boston-dc + +| level: extended + +type: keyword + +// =============================================================== + +| server.geo.region_iso_code +| Region ISO code. + +example: CA-QC + +| level: core + +type: keyword + +// =============================================================== + +| server.geo.region_name +| Region name. + +example: Quebec + | level: core type: keyword @@ -1847,42 +2650,119 @@ type: keyword -| level: core +| level: core + +type: ip + +// =============================================================== + +| server.mac +| MAC address of the server. + + + +| level: core + +type: keyword + +// =============================================================== + +| server.packets +| Packets sent from the server to the client. + +example: 12 + +| level: core + +type: long + +// =============================================================== + +| server.port +| Port of the server. + + + +| level: core + +type: long + +// =============================================================== + +| server.user.email +| User email address. + + + +| level: extended + +type: keyword + +// =============================================================== + +| server.user.full_name +| User's full name, if available. + +example: Albert Einstein + +| level: extended + +type: keyword + +// =============================================================== + +| server.user.group.id +| Unique identifier for the group on the system/platform. + + + +| level: extended + +type: keyword + +// =============================================================== + +| server.user.group.name +| Name of the group. + + -type: ip +| level: extended + +type: keyword // =============================================================== -| server.mac -| MAC address of the server. +| server.user.hash +| Unique user hash to correlate information for a user in anonymized form. -| level: core +| level: extended type: keyword // =============================================================== -| server.packets -| Packets sent from the server to the client. +| server.user.id +| One or multiple unique identifiers of the user. + -example: 12 | level: core -type: long +type: keyword // =============================================================== -| server.port -| Port of the server. - +| server.user.name +| Short name or login of the user. +example: albert | level: core -type: long +type: keyword // =============================================================== @@ -2007,6 +2887,94 @@ type: long +| level: core + +type: keyword + +// =============================================================== + +| source.geo.city_name +| City name. + +example: Montreal + +| level: core + +type: keyword + +// =============================================================== + +| source.geo.continent_name +| Name of the continent. + +example: North America + +| level: core + +type: keyword + +// =============================================================== + +| source.geo.country_iso_code +| Country ISO code. + +example: CA + +| level: core + +type: keyword + +// =============================================================== + +| source.geo.country_name +| Country name. + +example: Canada + +| level: core + +type: keyword + +// =============================================================== + +| source.geo.location +| Longitude and latitude. + +example: { "lon": -73.614830, "lat": 45.505918 } + +| level: core + +type: geo_point + +// =============================================================== + +| source.geo.name +| User-defined description of a location. + +example: boston-dc + +| level: extended + +type: keyword + +// =============================================================== + +| source.geo.region_iso_code +| Region ISO code. + +example: CA-QC + +| level: core + +type: keyword + +// =============================================================== + +| source.geo.region_name +| Region name. + +example: Quebec + | level: core type: keyword @@ -2057,6 +3025,83 @@ type: long // =============================================================== +| source.user.email +| User email address. + + + +| level: extended + +type: keyword + +// =============================================================== + +| source.user.full_name +| User's full name, if available. + +example: Albert Einstein + +| level: extended + +type: keyword + +// =============================================================== + +| source.user.group.id +| Unique identifier for the group on the system/platform. + + + +| level: extended + +type: keyword + +// =============================================================== + +| source.user.group.name +| Name of the group. + + + +| level: extended + +type: keyword + +// =============================================================== + +| source.user.hash +| Unique user hash to correlate information for a user in anonymized form. + + + +| level: extended + +type: keyword + +// =============================================================== + +| source.user.id +| One or multiple unique identifiers of the user. + + + +| level: core + +type: keyword + +// =============================================================== + +| source.user.name +| Short name or login of the user. + +example: albert + +| level: core + +type: keyword + +// =============================================================== + |===== [[ecs-url]] @@ -2143,7 +3188,7 @@ example: 443 | level: extended -type: integer +type: long // =============================================================== @@ -2210,6 +3255,28 @@ type: keyword example: Albert Einstein +| level: extended + +type: keyword + +// =============================================================== + +| user.group.id +| Unique identifier for the group on the system/platform. + + + +| level: extended + +type: keyword + +// =============================================================== + +| user.group.name +| Name of the group. + + + | level: extended type: keyword @@ -2296,6 +3363,72 @@ type: keyword // =============================================================== +| user_agent.os.family +| OS family (such as redhat, debian, freebsd, windows). + +example: debian + +| level: extended + +type: keyword + +// =============================================================== + +| user_agent.os.full +| Operating system name, including the version or code name. + +example: Mac OS Mojave + +| level: extended + +type: keyword + +// =============================================================== + +| user_agent.os.kernel +| Operating system kernel version as a raw string. + +example: 4.4.0-112-generic + +| level: extended + +type: keyword + +// =============================================================== + +| user_agent.os.name +| Operating system name, without the version. + +example: Mac OS X + +| level: extended + +type: keyword + +// =============================================================== + +| user_agent.os.platform +| Operating system platform (such centos, ubuntu, windows). + +example: darwin + +| level: extended + +type: keyword + +// =============================================================== + +| user_agent.os.version +| Operating system version as a raw string. + +example: 10.14.1 + +| level: extended + +type: keyword + +// =============================================================== + | user_agent.version | Version of the user agent. diff --git a/generated/ecs/fields_flat.yml b/generated/ecs/fields_flat.yml index 969104561f..d4537c2837 100644 --- a/generated/ecs/fields_flat.yml +++ b/generated/ecs/fields_flat.yml @@ -12,6 +12,7 @@ flat_name: '@timestamp' level: core name: '@timestamp' + order: 0 required: true short: Date/time when the event originated. type: date @@ -24,6 +25,7 @@ agent.ephemeral_id: ignore_above: 1024 level: extended name: ephemeral_id + order: 4 short: Ephemeral identifier of this agent. type: keyword agent.id: @@ -35,6 +37,7 @@ agent.id: ignore_above: 1024 level: core name: id + order: 3 short: Unique identifier of this agent. type: keyword agent.name: @@ -50,6 +53,7 @@ agent.name: ignore_above: 1024 level: core name: name + order: 1 short: Custom name of the agent. type: keyword agent.type: @@ -63,6 +67,7 @@ agent.type: ignore_above: 1024 level: core name: type + order: 2 short: Type of the agent. type: keyword agent.version: @@ -72,6 +77,7 @@ agent.version: ignore_above: 1024 level: core name: version + order: 0 short: Version of the agent. type: keyword client.address: @@ -85,6 +91,7 @@ client.address: ignore_above: 1024 level: extended name: address + order: 0 short: Client network address. type: keyword client.bytes: @@ -94,6 +101,7 @@ client.bytes: format: bytes level: core name: bytes + order: 5 short: Bytes sent from the client to the server. type: long client.domain: @@ -102,6 +110,7 @@ client.domain: ignore_above: 1024 level: core name: domain + order: 4 short: Client domain. type: keyword client.geo.city_name: @@ -111,6 +120,7 @@ client.geo.city_name: ignore_above: 1024 level: core name: city_name + order: 4 original_fieldset: geo short: City name. type: keyword @@ -121,6 +131,7 @@ client.geo.continent_name: ignore_above: 1024 level: core name: continent_name + order: 1 original_fieldset: geo short: Name of the continent. type: keyword @@ -131,6 +142,7 @@ client.geo.country_iso_code: ignore_above: 1024 level: core name: country_iso_code + order: 5 original_fieldset: geo short: Country ISO code. type: keyword @@ -141,6 +153,7 @@ client.geo.country_name: ignore_above: 1024 level: core name: country_name + order: 2 original_fieldset: geo short: Country name. type: keyword @@ -150,6 +163,7 @@ client.geo.location: flat_name: client.geo.location level: core name: location + order: 0 original_fieldset: geo short: Longitude and latitude. type: geo_point @@ -166,6 +180,7 @@ client.geo.name: ignore_above: 1024 level: extended name: name + order: 7 original_fieldset: geo short: User-defined description of a location. type: keyword @@ -176,6 +191,7 @@ client.geo.region_iso_code: ignore_above: 1024 level: core name: region_iso_code + order: 6 original_fieldset: geo short: Region ISO code. type: keyword @@ -186,6 +202,7 @@ client.geo.region_name: ignore_above: 1024 level: core name: region_name + order: 3 original_fieldset: geo short: Region name. type: keyword @@ -196,6 +213,7 @@ client.ip: flat_name: client.ip level: core name: ip + order: 1 short: IP address of the client. type: ip client.mac: @@ -204,6 +222,7 @@ client.mac: ignore_above: 1024 level: core name: mac + order: 3 short: MAC address of the client. type: keyword client.packets: @@ -212,6 +231,7 @@ client.packets: flat_name: client.packets level: core name: packets + order: 6 short: Packets sent from the client to the server. type: long client.port: @@ -219,6 +239,7 @@ client.port: flat_name: client.port level: core name: port + order: 2 short: Port of the client. type: long client.user.email: @@ -227,6 +248,7 @@ client.user.email: ignore_above: 1024 level: extended name: email + order: 3 original_fieldset: user short: User email address. type: keyword @@ -237,6 +259,7 @@ client.user.full_name: ignore_above: 1024 level: extended name: full_name + order: 2 original_fieldset: user short: User's full name, if available. type: keyword @@ -246,6 +269,7 @@ client.user.group.id: ignore_above: 1024 level: extended name: id + order: 0 original_fieldset: user short: Unique identifier for the group on the system/platform. type: keyword @@ -255,6 +279,7 @@ client.user.group.name: ignore_above: 1024 level: extended name: name + order: 1 original_fieldset: user short: Name of the group. type: keyword @@ -268,6 +293,7 @@ client.user.hash: ignore_above: 1024 level: extended name: hash + order: 4 original_fieldset: user short: Unique user hash to correlate information for a user in anonymized form. type: keyword @@ -277,6 +303,7 @@ client.user.id: ignore_above: 1024 level: core name: id + order: 0 original_fieldset: user short: One or multiple unique identifiers of the user. type: keyword @@ -287,6 +314,7 @@ client.user.name: ignore_above: 1024 level: core name: name + order: 1 original_fieldset: user short: Short name or login of the user. type: keyword @@ -300,6 +328,7 @@ cloud.account.id: ignore_above: 1024 level: extended name: account.id + order: 6 short: The cloud account or organization id. type: keyword cloud.availability_zone: @@ -309,6 +338,7 @@ cloud.availability_zone: ignore_above: 1024 level: extended name: availability_zone + order: 1 short: Availability zone in which this host is running. type: keyword cloud.instance.id: @@ -318,6 +348,7 @@ cloud.instance.id: ignore_above: 1024 level: extended name: instance.id + order: 3 short: Instance ID of the host machine. type: keyword cloud.instance.name: @@ -326,6 +357,7 @@ cloud.instance.name: ignore_above: 1024 level: extended name: instance.name + order: 4 short: Instance name of the host machine. type: keyword cloud.machine.type: @@ -335,6 +367,7 @@ cloud.machine.type: ignore_above: 1024 level: extended name: machine.type + order: 5 short: Machine type of the host machine. type: keyword cloud.provider: @@ -345,6 +378,7 @@ cloud.provider: ignore_above: 1024 level: extended name: provider + order: 0 short: Name of the cloud provider. type: keyword cloud.region: @@ -354,6 +388,7 @@ cloud.region: ignore_above: 1024 level: extended name: region + order: 2 short: Region in which this host is running. type: keyword container.id: @@ -362,6 +397,7 @@ container.id: ignore_above: 1024 level: core name: id + order: 1 short: Unique container id. type: keyword container.image.name: @@ -370,6 +406,7 @@ container.image.name: ignore_above: 1024 level: extended name: image.name + order: 2 short: Name of the image the container was built on. type: keyword container.image.tag: @@ -378,6 +415,7 @@ container.image.tag: ignore_above: 1024 level: extended name: image.tag + order: 3 short: Container image tag. type: keyword container.labels: @@ -386,6 +424,7 @@ container.labels: level: extended name: labels object_type: keyword + order: 5 short: Image labels. type: object container.name: @@ -394,6 +433,7 @@ container.name: ignore_above: 1024 level: extended name: name + order: 4 short: Container name. type: keyword container.runtime: @@ -403,6 +443,7 @@ container.runtime: ignore_above: 1024 level: extended name: runtime + order: 0 short: Runtime managing this container. type: keyword destination.address: @@ -416,6 +457,7 @@ destination.address: ignore_above: 1024 level: extended name: address + order: 0 short: Destination network address. type: keyword destination.bytes: @@ -425,6 +467,7 @@ destination.bytes: format: bytes level: core name: bytes + order: 5 short: Bytes sent from the destination to the source. type: long destination.domain: @@ -433,6 +476,7 @@ destination.domain: ignore_above: 1024 level: core name: domain + order: 4 short: Destination domain. type: keyword destination.geo.city_name: @@ -442,6 +486,7 @@ destination.geo.city_name: ignore_above: 1024 level: core name: city_name + order: 4 original_fieldset: geo short: City name. type: keyword @@ -452,6 +497,7 @@ destination.geo.continent_name: ignore_above: 1024 level: core name: continent_name + order: 1 original_fieldset: geo short: Name of the continent. type: keyword @@ -462,6 +508,7 @@ destination.geo.country_iso_code: ignore_above: 1024 level: core name: country_iso_code + order: 5 original_fieldset: geo short: Country ISO code. type: keyword @@ -472,6 +519,7 @@ destination.geo.country_name: ignore_above: 1024 level: core name: country_name + order: 2 original_fieldset: geo short: Country name. type: keyword @@ -481,6 +529,7 @@ destination.geo.location: flat_name: destination.geo.location level: core name: location + order: 0 original_fieldset: geo short: Longitude and latitude. type: geo_point @@ -497,6 +546,7 @@ destination.geo.name: ignore_above: 1024 level: extended name: name + order: 7 original_fieldset: geo short: User-defined description of a location. type: keyword @@ -507,6 +557,7 @@ destination.geo.region_iso_code: ignore_above: 1024 level: core name: region_iso_code + order: 6 original_fieldset: geo short: Region ISO code. type: keyword @@ -517,6 +568,7 @@ destination.geo.region_name: ignore_above: 1024 level: core name: region_name + order: 3 original_fieldset: geo short: Region name. type: keyword @@ -527,6 +579,7 @@ destination.ip: flat_name: destination.ip level: core name: ip + order: 1 short: IP address of the destination. type: ip destination.mac: @@ -535,6 +588,7 @@ destination.mac: ignore_above: 1024 level: core name: mac + order: 3 short: MAC address of the destination. type: keyword destination.packets: @@ -543,6 +597,7 @@ destination.packets: flat_name: destination.packets level: core name: packets + order: 6 short: Packets sent from the destination to the source. type: long destination.port: @@ -550,6 +605,7 @@ destination.port: flat_name: destination.port level: core name: port + order: 2 short: Port of the destination. type: long destination.user.email: @@ -558,6 +614,7 @@ destination.user.email: ignore_above: 1024 level: extended name: email + order: 3 original_fieldset: user short: User email address. type: keyword @@ -568,6 +625,7 @@ destination.user.full_name: ignore_above: 1024 level: extended name: full_name + order: 2 original_fieldset: user short: User's full name, if available. type: keyword @@ -577,6 +635,7 @@ destination.user.group.id: ignore_above: 1024 level: extended name: id + order: 0 original_fieldset: user short: Unique identifier for the group on the system/platform. type: keyword @@ -586,6 +645,7 @@ destination.user.group.name: ignore_above: 1024 level: extended name: name + order: 1 original_fieldset: user short: Name of the group. type: keyword @@ -599,6 +659,7 @@ destination.user.hash: ignore_above: 1024 level: extended name: hash + order: 4 original_fieldset: user short: Unique user hash to correlate information for a user in anonymized form. type: keyword @@ -608,6 +669,7 @@ destination.user.id: ignore_above: 1024 level: core name: id + order: 0 original_fieldset: user short: One or multiple unique identifiers of the user. type: keyword @@ -618,6 +680,7 @@ destination.user.name: ignore_above: 1024 level: core name: name + order: 1 original_fieldset: user short: Short name or login of the user. type: keyword @@ -633,6 +696,7 @@ ecs.version: ignore_above: 1024 level: core name: version + order: 0 required: true short: ECS version this event conforms to. type: keyword @@ -642,6 +706,7 @@ error.code: ignore_above: 1024 level: core name: code + order: 2 short: Error code describing the error. type: keyword error.id: @@ -650,6 +715,7 @@ error.id: ignore_above: 1024 level: core name: id + order: 0 short: Unique identifier for the error. type: keyword error.message: @@ -658,6 +724,7 @@ error.message: level: core name: message norms: false + order: 1 short: Error message. type: text event.action: @@ -671,6 +738,7 @@ event.action: ignore_above: 1024 level: core name: action + order: 3 short: The action captured by the event. type: keyword event.category: @@ -685,6 +753,7 @@ event.category: ignore_above: 1024 level: core name: category + order: 2 short: Event category. type: keyword event.created: @@ -703,6 +772,7 @@ event.created: flat_name: event.created level: core name: created + order: 13 short: Time when the event was first read by an agent or by your pipeline. type: date event.dataset: @@ -716,6 +786,7 @@ event.dataset: ignore_above: 1024 level: core name: dataset + order: 7 short: Name of the dataset. type: keyword event.duration: @@ -728,6 +799,7 @@ event.duration: input_format: nanoseconds level: core name: duration + order: 11 short: Duration of the event in nanoseconds. type: long event.end: @@ -736,6 +808,7 @@ event.end: flat_name: event.end level: extended name: end + order: 15 short: event.end contains the date when the event ended or when the activity was last observed. type: date @@ -747,6 +820,7 @@ event.hash: ignore_above: 1024 level: extended name: hash + order: 10 short: Hash (perhaps logstash fingerprint) of raw field to be able to demonstrate log integrity. type: keyword @@ -757,6 +831,7 @@ event.id: ignore_above: 1024 level: core name: id + order: 0 short: Unique ID to describe the event. type: keyword event.kind: @@ -771,6 +846,7 @@ event.kind: ignore_above: 1024 level: extended name: kind + order: 1 short: The kind of the event. type: keyword event.module: @@ -782,6 +858,7 @@ event.module: ignore_above: 1024 level: core name: module + order: 6 short: 'Name of the module this data is coming from. This information is coming from the modules used in Beats or Logstash.' @@ -799,6 +876,7 @@ event.original: index: false level: core name: original + order: 9 short: Raw text message of entire event. type: keyword event.outcome: @@ -813,6 +891,7 @@ event.outcome: ignore_above: 1024 level: extended name: outcome + order: 4 short: The outcome of the event. type: keyword event.risk_score: @@ -821,6 +900,7 @@ event.risk_score: flat_name: event.risk_score level: core name: risk_score + order: 16 short: Risk score or priority of the event (e.g. security solutions). Use your system's original value here. type: float @@ -833,6 +913,7 @@ event.risk_score_norm: flat_name: event.risk_score_norm level: extended name: risk_score_norm + order: 17 short: Normalized risk score or priority of the event (0-100). type: float event.severity: @@ -843,6 +924,7 @@ event.severity: flat_name: event.severity level: core name: severity + order: 8 short: Original severity of the event. type: long event.start: @@ -851,6 +933,7 @@ event.start: flat_name: event.start level: extended name: start + order: 14 short: event.start contains the date when the event started or when the activity was first observed. type: date @@ -865,6 +948,7 @@ event.timezone: ignore_above: 1024 level: extended name: timezone + order: 12 short: Event time zone. type: keyword event.type: @@ -875,6 +959,7 @@ event.type: ignore_above: 1024 level: core name: type + order: 5 short: Reserved for future usage. type: keyword file.ctime: @@ -882,6 +967,7 @@ file.ctime: flat_name: file.ctime level: extended name: ctime + order: 13 short: Last time file metadata changed. type: date file.device: @@ -890,6 +976,7 @@ file.device: ignore_above: 1024 level: extended name: device + order: 4 short: Device that is the source of the file. type: keyword file.extension: @@ -901,6 +988,7 @@ file.extension: ignore_above: 1024 level: extended name: extension + order: 2 short: File extension. type: keyword file.gid: @@ -909,6 +997,7 @@ file.gid: ignore_above: 1024 level: extended name: gid + order: 8 short: Primary group ID (GID) of the file. type: keyword file.group: @@ -917,6 +1006,7 @@ file.group: ignore_above: 1024 level: extended name: group + order: 9 short: Primary group name of the file. type: keyword file.inode: @@ -925,6 +1015,7 @@ file.inode: ignore_above: 1024 level: extended name: inode + order: 5 short: Inode representing the file in the filesystem. type: keyword file.mode: @@ -934,6 +1025,7 @@ file.mode: ignore_above: 1024 level: extended name: mode + order: 10 short: Mode of the file in octal representation. type: keyword file.mtime: @@ -941,6 +1033,7 @@ file.mtime: flat_name: file.mtime level: extended name: mtime + order: 12 short: Last time file content was modified. type: date file.owner: @@ -949,6 +1042,7 @@ file.owner: ignore_above: 1024 level: extended name: owner + order: 7 short: File owner's username. type: keyword file.path: @@ -957,6 +1051,7 @@ file.path: ignore_above: 1024 level: extended name: path + order: 0 short: Path to the file. type: keyword file.size: @@ -964,6 +1059,7 @@ file.size: flat_name: file.size level: extended name: size + order: 11 short: File size in bytes (field is only added when `type` is `file`). type: long file.target_path: @@ -972,6 +1068,7 @@ file.target_path: ignore_above: 1024 level: extended name: target_path + order: 1 short: Target path for symlinks. type: keyword file.type: @@ -980,6 +1077,7 @@ file.type: ignore_above: 1024 level: extended name: type + order: 3 short: File type (file, dir, or symlink). type: keyword file.uid: @@ -988,6 +1086,7 @@ file.uid: ignore_above: 1024 level: extended name: uid + order: 6 short: The user ID (UID) or security identifier (SID) of the file owner. type: keyword geo.city_name: @@ -997,6 +1096,7 @@ geo.city_name: ignore_above: 1024 level: core name: city_name + order: 4 short: City name. type: keyword geo.continent_name: @@ -1006,6 +1106,7 @@ geo.continent_name: ignore_above: 1024 level: core name: continent_name + order: 1 short: Name of the continent. type: keyword geo.country_iso_code: @@ -1015,6 +1116,7 @@ geo.country_iso_code: ignore_above: 1024 level: core name: country_iso_code + order: 5 short: Country ISO code. type: keyword geo.country_name: @@ -1024,6 +1126,7 @@ geo.country_name: ignore_above: 1024 level: core name: country_name + order: 2 short: Country name. type: keyword geo.location: @@ -1032,6 +1135,7 @@ geo.location: flat_name: geo.location level: core name: location + order: 0 short: Longitude and latitude. type: geo_point geo.name: @@ -1047,6 +1151,7 @@ geo.name: ignore_above: 1024 level: extended name: name + order: 7 short: User-defined description of a location. type: keyword geo.region_iso_code: @@ -1056,6 +1161,7 @@ geo.region_iso_code: ignore_above: 1024 level: core name: region_iso_code + order: 6 short: Region ISO code. type: keyword geo.region_name: @@ -1065,6 +1171,7 @@ geo.region_name: ignore_above: 1024 level: core name: region_name + order: 3 short: Region name. type: keyword group.id: @@ -1073,6 +1180,7 @@ group.id: ignore_above: 1024 level: extended name: id + order: 0 short: Unique identifier for the group on the system/platform. type: keyword group.name: @@ -1081,6 +1189,7 @@ group.name: ignore_above: 1024 level: extended name: name + order: 1 short: Name of the group. type: keyword host.architecture: @@ -1090,6 +1199,7 @@ host.architecture: ignore_above: 1024 level: core name: architecture + order: 6 short: Operating system architecture. type: keyword host.geo.city_name: @@ -1099,6 +1209,7 @@ host.geo.city_name: ignore_above: 1024 level: core name: city_name + order: 4 original_fieldset: geo short: City name. type: keyword @@ -1109,6 +1220,7 @@ host.geo.continent_name: ignore_above: 1024 level: core name: continent_name + order: 1 original_fieldset: geo short: Name of the continent. type: keyword @@ -1119,6 +1231,7 @@ host.geo.country_iso_code: ignore_above: 1024 level: core name: country_iso_code + order: 5 original_fieldset: geo short: Country ISO code. type: keyword @@ -1129,6 +1242,7 @@ host.geo.country_name: ignore_above: 1024 level: core name: country_name + order: 2 original_fieldset: geo short: Country name. type: keyword @@ -1138,6 +1252,7 @@ host.geo.location: flat_name: host.geo.location level: core name: location + order: 0 original_fieldset: geo short: Longitude and latitude. type: geo_point @@ -1154,6 +1269,7 @@ host.geo.name: ignore_above: 1024 level: extended name: name + order: 7 original_fieldset: geo short: User-defined description of a location. type: keyword @@ -1164,6 +1280,7 @@ host.geo.region_iso_code: ignore_above: 1024 level: core name: region_iso_code + order: 6 original_fieldset: geo short: Region ISO code. type: keyword @@ -1174,6 +1291,7 @@ host.geo.region_name: ignore_above: 1024 level: core name: region_name + order: 3 original_fieldset: geo short: Region name. type: keyword @@ -1185,6 +1303,7 @@ host.hostname: ignore_above: 1024 level: core name: hostname + order: 0 short: Hostname of the host. type: keyword host.id: @@ -1197,6 +1316,7 @@ host.id: ignore_above: 1024 level: core name: id + order: 2 short: Unique host id. type: keyword host.ip: @@ -1204,6 +1324,7 @@ host.ip: flat_name: host.ip level: core name: ip + order: 3 short: Host ip address. type: ip host.mac: @@ -1212,6 +1333,7 @@ host.mac: ignore_above: 1024 level: core name: mac + order: 4 short: Host mac address. type: keyword host.name: @@ -1223,6 +1345,7 @@ host.name: ignore_above: 1024 level: core name: name + order: 1 short: Name of the host. type: keyword host.os.family: @@ -1232,6 +1355,7 @@ host.os.family: ignore_above: 1024 level: extended name: family + order: 3 original_fieldset: os short: OS family (such as redhat, debian, freebsd, windows). type: keyword @@ -1242,6 +1366,7 @@ host.os.full: ignore_above: 1024 level: extended name: full + order: 2 original_fieldset: os short: Operating system name, including the version or code name. type: keyword @@ -1252,6 +1377,7 @@ host.os.kernel: ignore_above: 1024 level: extended name: kernel + order: 5 original_fieldset: os short: Operating system kernel version as a raw string. type: keyword @@ -1262,6 +1388,7 @@ host.os.name: ignore_above: 1024 level: extended name: name + order: 1 original_fieldset: os short: Operating system name, without the version. type: keyword @@ -1272,6 +1399,7 @@ host.os.platform: ignore_above: 1024 level: extended name: platform + order: 0 original_fieldset: os short: Operating system platform (such centos, ubuntu, windows). type: keyword @@ -1282,6 +1410,7 @@ host.os.version: ignore_above: 1024 level: extended name: version + order: 4 original_fieldset: os short: Operating system version as a raw string. type: keyword @@ -1294,6 +1423,7 @@ host.type: ignore_above: 1024 level: core name: type + order: 5 short: Type of host. type: keyword host.user.email: @@ -1302,6 +1432,7 @@ host.user.email: ignore_above: 1024 level: extended name: email + order: 3 original_fieldset: user short: User email address. type: keyword @@ -1312,6 +1443,7 @@ host.user.full_name: ignore_above: 1024 level: extended name: full_name + order: 2 original_fieldset: user short: User's full name, if available. type: keyword @@ -1321,6 +1453,7 @@ host.user.group.id: ignore_above: 1024 level: extended name: id + order: 0 original_fieldset: user short: Unique identifier for the group on the system/platform. type: keyword @@ -1330,6 +1463,7 @@ host.user.group.name: ignore_above: 1024 level: extended name: name + order: 1 original_fieldset: user short: Name of the group. type: keyword @@ -1343,6 +1477,7 @@ host.user.hash: ignore_above: 1024 level: extended name: hash + order: 4 original_fieldset: user short: Unique user hash to correlate information for a user in anonymized form. type: keyword @@ -1352,6 +1487,7 @@ host.user.id: ignore_above: 1024 level: core name: id + order: 0 original_fieldset: user short: One or multiple unique identifiers of the user. type: keyword @@ -1362,6 +1498,7 @@ host.user.name: ignore_above: 1024 level: core name: name + order: 1 original_fieldset: user short: Short name or login of the user. type: keyword @@ -1372,6 +1509,7 @@ http.request.body.bytes: format: bytes level: extended name: request.body.bytes + order: 7 short: Size in bytes of the request body. type: long http.request.body.content: @@ -1381,6 +1519,7 @@ http.request.body.content: ignore_above: 1024 level: extended name: request.body.content + order: 1 short: The full HTTP request body. type: keyword http.request.bytes: @@ -1390,6 +1529,7 @@ http.request.bytes: format: bytes level: extended name: request.bytes + order: 6 short: Total size in bytes of the request (body and headers). type: long http.request.method: @@ -1402,6 +1542,7 @@ http.request.method: ignore_above: 1024 level: extended name: request.method + order: 0 short: HTTP request method. type: keyword http.request.referrer: @@ -1411,6 +1552,7 @@ http.request.referrer: ignore_above: 1024 level: extended name: request.referrer + order: 2 short: Referrer for this HTTP request. type: keyword http.response.body.bytes: @@ -1420,6 +1562,7 @@ http.response.body.bytes: format: bytes level: extended name: response.body.bytes + order: 9 short: Size in bytes of the response body. type: long http.response.body.content: @@ -1429,6 +1572,7 @@ http.response.body.content: ignore_above: 1024 level: extended name: response.body.content + order: 4 short: The full HTTP response body. type: keyword http.response.bytes: @@ -1438,6 +1582,7 @@ http.response.bytes: format: bytes level: extended name: response.bytes + order: 8 short: Total size in bytes of the response (body and headers). type: long http.response.status_code: @@ -1446,6 +1591,7 @@ http.response.status_code: flat_name: http.response.status_code level: extended name: response.status_code + order: 3 short: HTTP response status code. type: long http.version: @@ -1455,6 +1601,7 @@ http.version: ignore_above: 1024 level: extended name: version + order: 5 short: HTTP version. type: keyword labels: @@ -1471,6 +1618,7 @@ labels: level: core name: labels object_type: keyword + order: 2 short: Custom key/value pairs. type: object log.level: @@ -1482,6 +1630,7 @@ log.level: ignore_above: 1024 level: core name: level + order: 0 short: Log level of the log event. type: keyword log.original: @@ -1502,6 +1651,7 @@ log.original: index: false level: core name: original + order: 1 short: Original log message with light interpretation only (encoding, newlines). type: keyword message: @@ -1517,6 +1667,7 @@ message: level: core name: message norms: false + order: 3 short: Log message optimized for viewing in a log viewer. type: text network.application: @@ -1532,6 +1683,7 @@ network.application: ignore_above: 1024 level: extended name: application + order: 4 short: Application level protocol name. type: keyword network.bytes: @@ -1544,6 +1696,7 @@ network.bytes: format: bytes level: core name: bytes + order: 9 short: Total bytes transferred in both directions. type: long network.community_id: @@ -1556,6 +1709,7 @@ network.community_id: ignore_above: 1024 level: extended name: community_id + order: 8 short: A hash of source and destination IPs and ports. type: keyword network.direction: @@ -1569,6 +1723,7 @@ network.direction: ignore_above: 1024 level: core name: direction + order: 6 short: Direction of the network traffic. type: keyword network.forwarded_ip: @@ -1577,6 +1732,7 @@ network.forwarded_ip: flat_name: network.forwarded_ip level: core name: forwarded_ip + order: 7 short: Host IP address when the source IP address is the proxy. type: ip network.iana_number: @@ -1588,6 +1744,7 @@ network.iana_number: ignore_above: 1024 level: extended name: iana_number + order: 2 short: IANA Protocol Number. type: keyword network.name: @@ -1597,6 +1754,7 @@ network.name: ignore_above: 1024 level: extended name: name + order: 0 short: Name given by operators to sections of their network. type: keyword network.packets: @@ -1608,6 +1766,7 @@ network.packets: flat_name: network.packets level: core name: packets + order: 10 short: Total packets transferred in both directions. type: long network.protocol: @@ -1620,6 +1779,7 @@ network.protocol: ignore_above: 1024 level: core name: protocol + order: 5 short: L7 Network protocol name. type: keyword network.transport: @@ -1633,6 +1793,7 @@ network.transport: ignore_above: 1024 level: core name: transport + order: 3 short: Protocol Name corresponding to the field `iana_number`. type: keyword network.type: @@ -1646,6 +1807,7 @@ network.type: ignore_above: 1024 level: core name: type + order: 1 short: In the OSI Model this would be the Network Layer. ipv4, ipv6, ipsec, pim, etc type: keyword @@ -1656,6 +1818,7 @@ observer.geo.city_name: ignore_above: 1024 level: core name: city_name + order: 4 original_fieldset: geo short: City name. type: keyword @@ -1666,6 +1829,7 @@ observer.geo.continent_name: ignore_above: 1024 level: core name: continent_name + order: 1 original_fieldset: geo short: Name of the continent. type: keyword @@ -1676,6 +1840,7 @@ observer.geo.country_iso_code: ignore_above: 1024 level: core name: country_iso_code + order: 5 original_fieldset: geo short: Country ISO code. type: keyword @@ -1686,6 +1851,7 @@ observer.geo.country_name: ignore_above: 1024 level: core name: country_name + order: 2 original_fieldset: geo short: Country name. type: keyword @@ -1695,6 +1861,7 @@ observer.geo.location: flat_name: observer.geo.location level: core name: location + order: 0 original_fieldset: geo short: Longitude and latitude. type: geo_point @@ -1711,6 +1878,7 @@ observer.geo.name: ignore_above: 1024 level: extended name: name + order: 7 original_fieldset: geo short: User-defined description of a location. type: keyword @@ -1721,6 +1889,7 @@ observer.geo.region_iso_code: ignore_above: 1024 level: core name: region_iso_code + order: 6 original_fieldset: geo short: Region ISO code. type: keyword @@ -1731,6 +1900,7 @@ observer.geo.region_name: ignore_above: 1024 level: core name: region_name + order: 3 original_fieldset: geo short: Region name. type: keyword @@ -1740,6 +1910,7 @@ observer.hostname: ignore_above: 1024 level: core name: hostname + order: 2 short: Hostname of the observer. type: keyword observer.ip: @@ -1747,6 +1918,7 @@ observer.ip: flat_name: observer.ip level: core name: ip + order: 1 short: IP address of the observer. type: ip observer.mac: @@ -1755,6 +1927,7 @@ observer.mac: ignore_above: 1024 level: core name: mac + order: 0 short: MAC address of the observer type: keyword observer.os.family: @@ -1764,6 +1937,7 @@ observer.os.family: ignore_above: 1024 level: extended name: family + order: 3 original_fieldset: os short: OS family (such as redhat, debian, freebsd, windows). type: keyword @@ -1774,6 +1948,7 @@ observer.os.full: ignore_above: 1024 level: extended name: full + order: 2 original_fieldset: os short: Operating system name, including the version or code name. type: keyword @@ -1784,6 +1959,7 @@ observer.os.kernel: ignore_above: 1024 level: extended name: kernel + order: 5 original_fieldset: os short: Operating system kernel version as a raw string. type: keyword @@ -1794,6 +1970,7 @@ observer.os.name: ignore_above: 1024 level: extended name: name + order: 1 original_fieldset: os short: Operating system name, without the version. type: keyword @@ -1804,6 +1981,7 @@ observer.os.platform: ignore_above: 1024 level: extended name: platform + order: 0 original_fieldset: os short: Operating system platform (such centos, ubuntu, windows). type: keyword @@ -1814,6 +1992,7 @@ observer.os.version: ignore_above: 1024 level: extended name: version + order: 4 original_fieldset: os short: Operating system version as a raw string. type: keyword @@ -1823,6 +2002,7 @@ observer.serial_number: ignore_above: 1024 level: extended name: serial_number + order: 5 short: Observer serial number. type: keyword observer.type: @@ -1835,6 +2015,7 @@ observer.type: ignore_above: 1024 level: core name: type + order: 6 short: The type of the observer the data is coming from. type: keyword observer.vendor: @@ -1843,6 +2024,7 @@ observer.vendor: ignore_above: 1024 level: core name: vendor + order: 3 short: observer vendor information. type: keyword observer.version: @@ -1851,6 +2033,7 @@ observer.version: ignore_above: 1024 level: core name: version + order: 4 short: Observer version. type: keyword organization.id: @@ -1859,6 +2042,7 @@ organization.id: ignore_above: 1024 level: extended name: id + order: 1 short: Unique identifier for the organization. type: keyword organization.name: @@ -1867,6 +2051,7 @@ organization.name: ignore_above: 1024 level: extended name: name + order: 0 short: Organization name. type: keyword os.family: @@ -1876,6 +2061,7 @@ os.family: ignore_above: 1024 level: extended name: family + order: 3 short: OS family (such as redhat, debian, freebsd, windows). type: keyword os.full: @@ -1885,6 +2071,7 @@ os.full: ignore_above: 1024 level: extended name: full + order: 2 short: Operating system name, including the version or code name. type: keyword os.kernel: @@ -1894,6 +2081,7 @@ os.kernel: ignore_above: 1024 level: extended name: kernel + order: 5 short: Operating system kernel version as a raw string. type: keyword os.name: @@ -1903,6 +2091,7 @@ os.name: ignore_above: 1024 level: extended name: name + order: 1 short: Operating system name, without the version. type: keyword os.platform: @@ -1912,6 +2101,7 @@ os.platform: ignore_above: 1024 level: extended name: platform + order: 0 short: Operating system platform (such centos, ubuntu, windows). type: keyword os.version: @@ -1921,6 +2111,7 @@ os.version: ignore_above: 1024 level: extended name: version + order: 4 short: Operating system version as a raw string. type: keyword process.args: @@ -1936,6 +2127,7 @@ process.args: ignore_above: 1024 level: extended name: args + order: 3 short: Array of process arguments. type: keyword process.executable: @@ -1945,6 +2137,7 @@ process.executable: ignore_above: 1024 level: extended name: executable + order: 4 short: Absolute path to the process executable. type: keyword process.name: @@ -1956,6 +2149,7 @@ process.name: ignore_above: 1024 level: extended name: name + order: 1 short: Process name. type: keyword process.pid: @@ -1964,6 +2158,7 @@ process.pid: flat_name: process.pid level: core name: pid + order: 0 short: Process id. type: long process.ppid: @@ -1971,6 +2166,7 @@ process.ppid: flat_name: process.ppid level: extended name: ppid + order: 2 short: Process parent id. type: long process.start: @@ -1979,6 +2175,7 @@ process.start: flat_name: process.start level: extended name: start + order: 7 short: The time the process started. type: date process.thread.id: @@ -1987,6 +2184,7 @@ process.thread.id: flat_name: process.thread.id level: extended name: thread.id + order: 6 short: Thread ID. type: long process.title: @@ -1998,6 +2196,7 @@ process.title: ignore_above: 1024 level: extended name: title + order: 5 short: Process title. type: keyword process.working_directory: @@ -2007,6 +2206,7 @@ process.working_directory: ignore_above: 1024 level: extended name: working_directory + order: 8 short: The working directory of the process. type: keyword related.ip: @@ -2014,6 +2214,7 @@ related.ip: flat_name: related.ip level: extended name: ip + order: 0 short: All of the IPs seen on your event. type: ip server.address: @@ -2027,6 +2228,7 @@ server.address: ignore_above: 1024 level: extended name: address + order: 0 short: Server network address. type: keyword server.bytes: @@ -2036,6 +2238,7 @@ server.bytes: format: bytes level: core name: bytes + order: 5 short: Bytes sent from the server to the client. type: long server.domain: @@ -2044,6 +2247,7 @@ server.domain: ignore_above: 1024 level: core name: domain + order: 4 short: Server domain. type: keyword server.geo.city_name: @@ -2053,6 +2257,7 @@ server.geo.city_name: ignore_above: 1024 level: core name: city_name + order: 4 original_fieldset: geo short: City name. type: keyword @@ -2063,6 +2268,7 @@ server.geo.continent_name: ignore_above: 1024 level: core name: continent_name + order: 1 original_fieldset: geo short: Name of the continent. type: keyword @@ -2073,6 +2279,7 @@ server.geo.country_iso_code: ignore_above: 1024 level: core name: country_iso_code + order: 5 original_fieldset: geo short: Country ISO code. type: keyword @@ -2083,6 +2290,7 @@ server.geo.country_name: ignore_above: 1024 level: core name: country_name + order: 2 original_fieldset: geo short: Country name. type: keyword @@ -2092,6 +2300,7 @@ server.geo.location: flat_name: server.geo.location level: core name: location + order: 0 original_fieldset: geo short: Longitude and latitude. type: geo_point @@ -2108,6 +2317,7 @@ server.geo.name: ignore_above: 1024 level: extended name: name + order: 7 original_fieldset: geo short: User-defined description of a location. type: keyword @@ -2118,6 +2328,7 @@ server.geo.region_iso_code: ignore_above: 1024 level: core name: region_iso_code + order: 6 original_fieldset: geo short: Region ISO code. type: keyword @@ -2128,6 +2339,7 @@ server.geo.region_name: ignore_above: 1024 level: core name: region_name + order: 3 original_fieldset: geo short: Region name. type: keyword @@ -2138,6 +2350,7 @@ server.ip: flat_name: server.ip level: core name: ip + order: 1 short: IP address of the server. type: ip server.mac: @@ -2146,6 +2359,7 @@ server.mac: ignore_above: 1024 level: core name: mac + order: 3 short: MAC address of the server. type: keyword server.packets: @@ -2154,6 +2368,7 @@ server.packets: flat_name: server.packets level: core name: packets + order: 6 short: Packets sent from the server to the client. type: long server.port: @@ -2161,6 +2376,7 @@ server.port: flat_name: server.port level: core name: port + order: 2 short: Port of the server. type: long server.user.email: @@ -2169,6 +2385,7 @@ server.user.email: ignore_above: 1024 level: extended name: email + order: 3 original_fieldset: user short: User email address. type: keyword @@ -2179,6 +2396,7 @@ server.user.full_name: ignore_above: 1024 level: extended name: full_name + order: 2 original_fieldset: user short: User's full name, if available. type: keyword @@ -2188,6 +2406,7 @@ server.user.group.id: ignore_above: 1024 level: extended name: id + order: 0 original_fieldset: user short: Unique identifier for the group on the system/platform. type: keyword @@ -2197,6 +2416,7 @@ server.user.group.name: ignore_above: 1024 level: extended name: name + order: 1 original_fieldset: user short: Name of the group. type: keyword @@ -2210,6 +2430,7 @@ server.user.hash: ignore_above: 1024 level: extended name: hash + order: 4 original_fieldset: user short: Unique user hash to correlate information for a user in anonymized form. type: keyword @@ -2219,6 +2440,7 @@ server.user.id: ignore_above: 1024 level: core name: id + order: 0 original_fieldset: user short: One or multiple unique identifiers of the user. type: keyword @@ -2229,6 +2451,7 @@ server.user.name: ignore_above: 1024 level: core name: name + order: 1 original_fieldset: user short: Short name or login of the user. type: keyword @@ -2241,6 +2464,7 @@ service.ephemeral_id: ignore_above: 1024 level: extended name: ephemeral_id + order: 5 short: Ephemeral identifier of this service. type: keyword service.id: @@ -2256,6 +2480,7 @@ service.id: ignore_above: 1024 level: core name: id + order: 0 short: Unique identifier of the running service. type: keyword service.name: @@ -2276,6 +2501,7 @@ service.name: ignore_above: 1024 level: core name: name + order: 1 short: Name of the service. type: keyword service.state: @@ -2284,6 +2510,7 @@ service.state: ignore_above: 1024 level: core name: state + order: 3 short: Current state of the service. type: keyword service.type: @@ -2299,6 +2526,7 @@ service.type: ignore_above: 1024 level: core name: type + order: 2 short: The type of the service. type: keyword service.version: @@ -2310,6 +2538,7 @@ service.version: ignore_above: 1024 level: core name: version + order: 4 short: Version of the service. type: keyword source.address: @@ -2323,6 +2552,7 @@ source.address: ignore_above: 1024 level: extended name: address + order: 0 short: Source network address. type: keyword source.bytes: @@ -2332,6 +2562,7 @@ source.bytes: format: bytes level: core name: bytes + order: 5 short: Bytes sent from the source to the destination. type: long source.domain: @@ -2340,6 +2571,7 @@ source.domain: ignore_above: 1024 level: core name: domain + order: 4 short: Source domain. type: keyword source.geo.city_name: @@ -2349,6 +2581,7 @@ source.geo.city_name: ignore_above: 1024 level: core name: city_name + order: 4 original_fieldset: geo short: City name. type: keyword @@ -2359,6 +2592,7 @@ source.geo.continent_name: ignore_above: 1024 level: core name: continent_name + order: 1 original_fieldset: geo short: Name of the continent. type: keyword @@ -2369,6 +2603,7 @@ source.geo.country_iso_code: ignore_above: 1024 level: core name: country_iso_code + order: 5 original_fieldset: geo short: Country ISO code. type: keyword @@ -2379,6 +2614,7 @@ source.geo.country_name: ignore_above: 1024 level: core name: country_name + order: 2 original_fieldset: geo short: Country name. type: keyword @@ -2388,6 +2624,7 @@ source.geo.location: flat_name: source.geo.location level: core name: location + order: 0 original_fieldset: geo short: Longitude and latitude. type: geo_point @@ -2404,6 +2641,7 @@ source.geo.name: ignore_above: 1024 level: extended name: name + order: 7 original_fieldset: geo short: User-defined description of a location. type: keyword @@ -2414,6 +2652,7 @@ source.geo.region_iso_code: ignore_above: 1024 level: core name: region_iso_code + order: 6 original_fieldset: geo short: Region ISO code. type: keyword @@ -2424,6 +2663,7 @@ source.geo.region_name: ignore_above: 1024 level: core name: region_name + order: 3 original_fieldset: geo short: Region name. type: keyword @@ -2434,6 +2674,7 @@ source.ip: flat_name: source.ip level: core name: ip + order: 1 short: IP address of the source. type: ip source.mac: @@ -2442,6 +2683,7 @@ source.mac: ignore_above: 1024 level: core name: mac + order: 3 short: MAC address of the source. type: keyword source.packets: @@ -2450,6 +2692,7 @@ source.packets: flat_name: source.packets level: core name: packets + order: 6 short: Packets sent from the source to the destination. type: long source.port: @@ -2457,6 +2700,7 @@ source.port: flat_name: source.port level: core name: port + order: 2 short: Port of the source. type: long source.user.email: @@ -2465,6 +2709,7 @@ source.user.email: ignore_above: 1024 level: extended name: email + order: 3 original_fieldset: user short: User email address. type: keyword @@ -2475,6 +2720,7 @@ source.user.full_name: ignore_above: 1024 level: extended name: full_name + order: 2 original_fieldset: user short: User's full name, if available. type: keyword @@ -2484,6 +2730,7 @@ source.user.group.id: ignore_above: 1024 level: extended name: id + order: 0 original_fieldset: user short: Unique identifier for the group on the system/platform. type: keyword @@ -2493,6 +2740,7 @@ source.user.group.name: ignore_above: 1024 level: extended name: name + order: 1 original_fieldset: user short: Name of the group. type: keyword @@ -2506,6 +2754,7 @@ source.user.hash: ignore_above: 1024 level: extended name: hash + order: 4 original_fieldset: user short: Unique user hash to correlate information for a user in anonymized form. type: keyword @@ -2515,6 +2764,7 @@ source.user.id: ignore_above: 1024 level: core name: id + order: 0 original_fieldset: user short: One or multiple unique identifiers of the user. type: keyword @@ -2525,6 +2775,7 @@ source.user.name: ignore_above: 1024 level: core name: name + order: 1 original_fieldset: user short: Short name or login of the user. type: keyword @@ -2535,6 +2786,7 @@ tags: ignore_above: 1024 level: core name: tags + order: 1 short: List of keywords used to tag each event. type: keyword url.domain: @@ -2547,6 +2799,7 @@ url.domain: ignore_above: 1024 level: extended name: domain + order: 3 short: Domain of the url. type: keyword url.fragment: @@ -2557,6 +2810,7 @@ url.fragment: ignore_above: 1024 level: extended name: fragment + order: 7 short: Portion of the url after the `#`. type: keyword url.full: @@ -2567,6 +2821,7 @@ url.full: ignore_above: 1024 level: extended name: full + order: 1 short: Full unparsed URL. type: keyword url.original: @@ -2581,6 +2836,7 @@ url.original: ignore_above: 1024 level: extended name: original + order: 0 short: Unmodified original url as seen in the event source. type: keyword url.password: @@ -2589,6 +2845,7 @@ url.password: ignore_above: 1024 level: extended name: password + order: 9 short: Password of the request. type: keyword url.path: @@ -2597,6 +2854,7 @@ url.path: ignore_above: 1024 level: extended name: path + order: 5 short: Path of the request, such as "/search". type: keyword url.port: @@ -2605,6 +2863,7 @@ url.port: flat_name: url.port level: extended name: port + order: 4 short: Port of the request, such as 443. type: long url.query: @@ -2619,6 +2878,7 @@ url.query: ignore_above: 1024 level: extended name: query + order: 6 short: Query string of the request. type: keyword url.scheme: @@ -2630,6 +2890,7 @@ url.scheme: ignore_above: 1024 level: extended name: scheme + order: 2 short: Scheme of the url. type: keyword url.username: @@ -2638,6 +2899,7 @@ url.username: ignore_above: 1024 level: extended name: username + order: 8 short: Username of the request. type: keyword user.email: @@ -2646,6 +2908,7 @@ user.email: ignore_above: 1024 level: extended name: email + order: 3 short: User email address. type: keyword user.full_name: @@ -2655,6 +2918,7 @@ user.full_name: ignore_above: 1024 level: extended name: full_name + order: 2 short: User's full name, if available. type: keyword user.group.id: @@ -2663,6 +2927,7 @@ user.group.id: ignore_above: 1024 level: extended name: id + order: 0 original_fieldset: group short: Unique identifier for the group on the system/platform. type: keyword @@ -2672,6 +2937,7 @@ user.group.name: ignore_above: 1024 level: extended name: name + order: 1 original_fieldset: group short: Name of the group. type: keyword @@ -2685,6 +2951,7 @@ user.hash: ignore_above: 1024 level: extended name: hash + order: 4 short: Unique user hash to correlate information for a user in anonymized form. type: keyword user.id: @@ -2693,6 +2960,7 @@ user.id: ignore_above: 1024 level: core name: id + order: 0 short: One or multiple unique identifiers of the user. type: keyword user.name: @@ -2702,6 +2970,7 @@ user.name: ignore_above: 1024 level: core name: name + order: 1 short: Short name or login of the user. type: keyword user_agent.device.name: @@ -2711,6 +2980,7 @@ user_agent.device.name: ignore_above: 1024 level: extended name: device.name + order: 3 short: Name of the device. type: keyword user_agent.name: @@ -2720,6 +2990,7 @@ user_agent.name: ignore_above: 1024 level: extended name: name + order: 1 short: Name of the user agent. type: keyword user_agent.original: @@ -2730,6 +3001,7 @@ user_agent.original: ignore_above: 1024 level: extended name: original + order: 0 short: Unparsed version of the user_agent. type: keyword user_agent.os.family: @@ -2739,6 +3011,7 @@ user_agent.os.family: ignore_above: 1024 level: extended name: family + order: 3 original_fieldset: os short: OS family (such as redhat, debian, freebsd, windows). type: keyword @@ -2749,6 +3022,7 @@ user_agent.os.full: ignore_above: 1024 level: extended name: full + order: 2 original_fieldset: os short: Operating system name, including the version or code name. type: keyword @@ -2759,6 +3033,7 @@ user_agent.os.kernel: ignore_above: 1024 level: extended name: kernel + order: 5 original_fieldset: os short: Operating system kernel version as a raw string. type: keyword @@ -2769,6 +3044,7 @@ user_agent.os.name: ignore_above: 1024 level: extended name: name + order: 1 original_fieldset: os short: Operating system name, without the version. type: keyword @@ -2779,6 +3055,7 @@ user_agent.os.platform: ignore_above: 1024 level: extended name: platform + order: 0 original_fieldset: os short: Operating system platform (such centos, ubuntu, windows). type: keyword @@ -2789,6 +3066,7 @@ user_agent.os.version: ignore_above: 1024 level: extended name: version + order: 4 original_fieldset: os short: Operating system version as a raw string. type: keyword @@ -2799,5 +3077,6 @@ user_agent.version: ignore_above: 1024 level: extended name: version + order: 2 short: Version of the user agent. type: keyword diff --git a/generated/ecs/fields_nested.yml b/generated/ecs/fields_nested.yml index 39ef1172ae..648e80f276 100644 --- a/generated/ecs/fields_nested.yml +++ b/generated/ecs/fields_nested.yml @@ -16,6 +16,7 @@ agent: ignore_above: 1024 level: extended name: ephemeral_id + order: 4 short: Ephemeral identifier of this agent. type: keyword id: @@ -27,6 +28,7 @@ agent: ignore_above: 1024 level: core name: id + order: 3 short: Unique identifier of this agent. type: keyword name: @@ -42,6 +44,7 @@ agent: ignore_above: 1024 level: core name: name + order: 1 short: Custom name of the agent. type: keyword type: @@ -55,6 +58,7 @@ agent: ignore_above: 1024 level: core name: type + order: 2 short: Type of the agent. type: keyword version: @@ -64,6 +68,7 @@ agent: ignore_above: 1024 level: core name: version + order: 0 short: Version of the agent. type: keyword footnote: 'Examples: In the case of Beats for logs, the agent.name is filebeat. @@ -94,6 +99,7 @@ base: flat_name: '@timestamp' level: core name: '@timestamp' + order: 0 required: true short: Date/time when the event originated. type: date @@ -111,6 +117,7 @@ base: level: core name: labels object_type: keyword + order: 2 short: Custom key/value pairs. type: object message: @@ -126,6 +133,7 @@ base: level: core name: message norms: false + order: 3 short: Log message optimized for viewing in a log viewer. type: text tags: @@ -135,6 +143,7 @@ base: ignore_above: 1024 level: core name: tags + order: 1 short: List of keywords used to tag each event. type: keyword group: 1 @@ -172,6 +181,7 @@ client: ignore_above: 1024 level: extended name: address + order: 0 short: Client network address. type: keyword bytes: @@ -181,6 +191,7 @@ client: format: bytes level: core name: bytes + order: 5 short: Bytes sent from the client to the server. type: long domain: @@ -189,6 +200,7 @@ client: ignore_above: 1024 level: core name: domain + order: 4 short: Client domain. type: keyword geo.city_name: @@ -198,6 +210,7 @@ client: ignore_above: 1024 level: core name: city_name + order: 4 original_fieldset: geo short: City name. type: keyword @@ -208,6 +221,7 @@ client: ignore_above: 1024 level: core name: continent_name + order: 1 original_fieldset: geo short: Name of the continent. type: keyword @@ -218,6 +232,7 @@ client: ignore_above: 1024 level: core name: country_iso_code + order: 5 original_fieldset: geo short: Country ISO code. type: keyword @@ -228,6 +243,7 @@ client: ignore_above: 1024 level: core name: country_name + order: 2 original_fieldset: geo short: Country name. type: keyword @@ -237,6 +253,7 @@ client: flat_name: client.geo.location level: core name: location + order: 0 original_fieldset: geo short: Longitude and latitude. type: geo_point @@ -253,6 +270,7 @@ client: ignore_above: 1024 level: extended name: name + order: 7 original_fieldset: geo short: User-defined description of a location. type: keyword @@ -263,6 +281,7 @@ client: ignore_above: 1024 level: core name: region_iso_code + order: 6 original_fieldset: geo short: Region ISO code. type: keyword @@ -273,6 +292,7 @@ client: ignore_above: 1024 level: core name: region_name + order: 3 original_fieldset: geo short: Region name. type: keyword @@ -283,6 +303,7 @@ client: flat_name: client.ip level: core name: ip + order: 1 short: IP address of the client. type: ip mac: @@ -291,6 +312,7 @@ client: ignore_above: 1024 level: core name: mac + order: 3 short: MAC address of the client. type: keyword packets: @@ -299,6 +321,7 @@ client: flat_name: client.packets level: core name: packets + order: 6 short: Packets sent from the client to the server. type: long port: @@ -306,6 +329,7 @@ client: flat_name: client.port level: core name: port + order: 2 short: Port of the client. type: long user.email: @@ -314,6 +338,7 @@ client: ignore_above: 1024 level: extended name: email + order: 3 original_fieldset: user short: User email address. type: keyword @@ -324,6 +349,7 @@ client: ignore_above: 1024 level: extended name: full_name + order: 2 original_fieldset: user short: User's full name, if available. type: keyword @@ -333,6 +359,7 @@ client: ignore_above: 1024 level: extended name: id + order: 0 original_fieldset: user short: Unique identifier for the group on the system/platform. type: keyword @@ -342,6 +369,7 @@ client: ignore_above: 1024 level: extended name: name + order: 1 original_fieldset: user short: Name of the group. type: keyword @@ -355,6 +383,7 @@ client: ignore_above: 1024 level: extended name: hash + order: 4 original_fieldset: user short: Unique user hash to correlate information for a user in anonymized form. type: keyword @@ -364,6 +393,7 @@ client: ignore_above: 1024 level: core name: id + order: 0 original_fieldset: user short: One or multiple unique identifiers of the user. type: keyword @@ -374,6 +404,7 @@ client: ignore_above: 1024 level: core name: name + order: 1 original_fieldset: user short: Short name or login of the user. type: keyword @@ -397,6 +428,7 @@ cloud: ignore_above: 1024 level: extended name: account.id + order: 6 short: The cloud account or organization id. type: keyword availability_zone: @@ -406,6 +438,7 @@ cloud: ignore_above: 1024 level: extended name: availability_zone + order: 1 short: Availability zone in which this host is running. type: keyword instance.id: @@ -415,6 +448,7 @@ cloud: ignore_above: 1024 level: extended name: instance.id + order: 3 short: Instance ID of the host machine. type: keyword instance.name: @@ -423,6 +457,7 @@ cloud: ignore_above: 1024 level: extended name: instance.name + order: 4 short: Instance name of the host machine. type: keyword machine.type: @@ -432,6 +467,7 @@ cloud: ignore_above: 1024 level: extended name: machine.type + order: 5 short: Machine type of the host machine. type: keyword provider: @@ -442,6 +478,7 @@ cloud: ignore_above: 1024 level: extended name: provider + order: 0 short: Name of the cloud provider. type: keyword region: @@ -451,6 +488,7 @@ cloud: ignore_above: 1024 level: extended name: region + order: 2 short: Region in which this host is running. type: keyword footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from @@ -476,6 +514,7 @@ container: ignore_above: 1024 level: core name: id + order: 1 short: Unique container id. type: keyword image.name: @@ -484,6 +523,7 @@ container: ignore_above: 1024 level: extended name: image.name + order: 2 short: Name of the image the container was built on. type: keyword image.tag: @@ -492,6 +532,7 @@ container: ignore_above: 1024 level: extended name: image.tag + order: 3 short: Container image tag. type: keyword labels: @@ -500,6 +541,7 @@ container: level: extended name: labels object_type: keyword + order: 5 short: Image labels. type: object name: @@ -508,6 +550,7 @@ container: ignore_above: 1024 level: extended name: name + order: 4 short: Container name. type: keyword runtime: @@ -517,6 +560,7 @@ container: ignore_above: 1024 level: extended name: runtime + order: 0 short: Runtime managing this container. type: keyword group: 2 @@ -541,6 +585,7 @@ destination: ignore_above: 1024 level: extended name: address + order: 0 short: Destination network address. type: keyword bytes: @@ -550,6 +595,7 @@ destination: format: bytes level: core name: bytes + order: 5 short: Bytes sent from the destination to the source. type: long domain: @@ -558,6 +604,7 @@ destination: ignore_above: 1024 level: core name: domain + order: 4 short: Destination domain. type: keyword geo.city_name: @@ -567,6 +614,7 @@ destination: ignore_above: 1024 level: core name: city_name + order: 4 original_fieldset: geo short: City name. type: keyword @@ -577,6 +625,7 @@ destination: ignore_above: 1024 level: core name: continent_name + order: 1 original_fieldset: geo short: Name of the continent. type: keyword @@ -587,6 +636,7 @@ destination: ignore_above: 1024 level: core name: country_iso_code + order: 5 original_fieldset: geo short: Country ISO code. type: keyword @@ -597,6 +647,7 @@ destination: ignore_above: 1024 level: core name: country_name + order: 2 original_fieldset: geo short: Country name. type: keyword @@ -606,6 +657,7 @@ destination: flat_name: destination.geo.location level: core name: location + order: 0 original_fieldset: geo short: Longitude and latitude. type: geo_point @@ -622,6 +674,7 @@ destination: ignore_above: 1024 level: extended name: name + order: 7 original_fieldset: geo short: User-defined description of a location. type: keyword @@ -632,6 +685,7 @@ destination: ignore_above: 1024 level: core name: region_iso_code + order: 6 original_fieldset: geo short: Region ISO code. type: keyword @@ -642,6 +696,7 @@ destination: ignore_above: 1024 level: core name: region_name + order: 3 original_fieldset: geo short: Region name. type: keyword @@ -652,6 +707,7 @@ destination: flat_name: destination.ip level: core name: ip + order: 1 short: IP address of the destination. type: ip mac: @@ -660,6 +716,7 @@ destination: ignore_above: 1024 level: core name: mac + order: 3 short: MAC address of the destination. type: keyword packets: @@ -668,6 +725,7 @@ destination: flat_name: destination.packets level: core name: packets + order: 6 short: Packets sent from the destination to the source. type: long port: @@ -675,6 +733,7 @@ destination: flat_name: destination.port level: core name: port + order: 2 short: Port of the destination. type: long user.email: @@ -683,6 +742,7 @@ destination: ignore_above: 1024 level: extended name: email + order: 3 original_fieldset: user short: User email address. type: keyword @@ -693,6 +753,7 @@ destination: ignore_above: 1024 level: extended name: full_name + order: 2 original_fieldset: user short: User's full name, if available. type: keyword @@ -702,6 +763,7 @@ destination: ignore_above: 1024 level: extended name: id + order: 0 original_fieldset: user short: Unique identifier for the group on the system/platform. type: keyword @@ -711,6 +773,7 @@ destination: ignore_above: 1024 level: extended name: name + order: 1 original_fieldset: user short: Name of the group. type: keyword @@ -724,6 +787,7 @@ destination: ignore_above: 1024 level: extended name: hash + order: 4 original_fieldset: user short: Unique user hash to correlate information for a user in anonymized form. type: keyword @@ -733,6 +797,7 @@ destination: ignore_above: 1024 level: core name: id + order: 0 original_fieldset: user short: One or multiple unique identifiers of the user. type: keyword @@ -743,6 +808,7 @@ destination: ignore_above: 1024 level: core name: name + order: 1 original_fieldset: user short: Short name or login of the user. type: keyword @@ -767,6 +833,7 @@ ecs: ignore_above: 1024 level: core name: version + order: 0 required: true short: ECS version this event conforms to. type: keyword @@ -787,6 +854,7 @@ error: ignore_above: 1024 level: core name: code + order: 2 short: Error code describing the error. type: keyword id: @@ -795,6 +863,7 @@ error: ignore_above: 1024 level: core name: id + order: 0 short: Unique identifier for the error. type: keyword message: @@ -803,6 +872,7 @@ error: level: core name: message norms: false + order: 1 short: Error message. type: text group: 2 @@ -835,6 +905,7 @@ event: ignore_above: 1024 level: core name: action + order: 3 short: The action captured by the event. type: keyword category: @@ -849,6 +920,7 @@ event: ignore_above: 1024 level: core name: category + order: 2 short: Event category. type: keyword created: @@ -867,6 +939,7 @@ event: flat_name: event.created level: core name: created + order: 13 short: Time when the event was first read by an agent or by your pipeline. type: date dataset: @@ -880,6 +953,7 @@ event: ignore_above: 1024 level: core name: dataset + order: 7 short: Name of the dataset. type: keyword duration: @@ -892,6 +966,7 @@ event: input_format: nanoseconds level: core name: duration + order: 11 short: Duration of the event in nanoseconds. type: long end: @@ -900,6 +975,7 @@ event: flat_name: event.end level: extended name: end + order: 15 short: event.end contains the date when the event ended or when the activity was last observed. type: date @@ -911,6 +987,7 @@ event: ignore_above: 1024 level: extended name: hash + order: 10 short: Hash (perhaps logstash fingerprint) of raw field to be able to demonstrate log integrity. type: keyword @@ -921,6 +998,7 @@ event: ignore_above: 1024 level: core name: id + order: 0 short: Unique ID to describe the event. type: keyword kind: @@ -935,6 +1013,7 @@ event: ignore_above: 1024 level: extended name: kind + order: 1 short: The kind of the event. type: keyword module: @@ -946,6 +1025,7 @@ event: ignore_above: 1024 level: core name: module + order: 6 short: 'Name of the module this data is coming from. This information is coming from the modules used in Beats or Logstash.' @@ -963,6 +1043,7 @@ event: index: false level: core name: original + order: 9 short: Raw text message of entire event. type: keyword outcome: @@ -977,6 +1058,7 @@ event: ignore_above: 1024 level: extended name: outcome + order: 4 short: The outcome of the event. type: keyword risk_score: @@ -985,6 +1067,7 @@ event: flat_name: event.risk_score level: core name: risk_score + order: 16 short: Risk score or priority of the event (e.g. security solutions). Use your system's original value here. type: float @@ -997,6 +1080,7 @@ event: flat_name: event.risk_score_norm level: extended name: risk_score_norm + order: 17 short: Normalized risk score or priority of the event (0-100). type: float severity: @@ -1007,6 +1091,7 @@ event: flat_name: event.severity level: core name: severity + order: 8 short: Original severity of the event. type: long start: @@ -1015,6 +1100,7 @@ event: flat_name: event.start level: extended name: start + order: 14 short: event.start contains the date when the event started or when the activity was first observed. type: date @@ -1029,6 +1115,7 @@ event: ignore_above: 1024 level: extended name: timezone + order: 12 short: Event time zone. type: keyword type: @@ -1039,6 +1126,7 @@ event: ignore_above: 1024 level: core name: type + order: 5 short: Reserved for future usage. type: keyword group: 2 @@ -1061,6 +1149,7 @@ file: flat_name: file.ctime level: extended name: ctime + order: 13 short: Last time file metadata changed. type: date device: @@ -1069,6 +1158,7 @@ file: ignore_above: 1024 level: extended name: device + order: 4 short: Device that is the source of the file. type: keyword extension: @@ -1080,6 +1170,7 @@ file: ignore_above: 1024 level: extended name: extension + order: 2 short: File extension. type: keyword gid: @@ -1088,6 +1179,7 @@ file: ignore_above: 1024 level: extended name: gid + order: 8 short: Primary group ID (GID) of the file. type: keyword group: @@ -1096,6 +1188,7 @@ file: ignore_above: 1024 level: extended name: group + order: 9 short: Primary group name of the file. type: keyword inode: @@ -1104,6 +1197,7 @@ file: ignore_above: 1024 level: extended name: inode + order: 5 short: Inode representing the file in the filesystem. type: keyword mode: @@ -1113,6 +1207,7 @@ file: ignore_above: 1024 level: extended name: mode + order: 10 short: Mode of the file in octal representation. type: keyword mtime: @@ -1120,6 +1215,7 @@ file: flat_name: file.mtime level: extended name: mtime + order: 12 short: Last time file content was modified. type: date owner: @@ -1128,6 +1224,7 @@ file: ignore_above: 1024 level: extended name: owner + order: 7 short: File owner's username. type: keyword path: @@ -1136,6 +1233,7 @@ file: ignore_above: 1024 level: extended name: path + order: 0 short: Path to the file. type: keyword size: @@ -1143,6 +1241,7 @@ file: flat_name: file.size level: extended name: size + order: 11 short: File size in bytes (field is only added when `type` is `file`). type: long target_path: @@ -1151,6 +1250,7 @@ file: ignore_above: 1024 level: extended name: target_path + order: 1 short: Target path for symlinks. type: keyword type: @@ -1159,6 +1259,7 @@ file: ignore_above: 1024 level: extended name: type + order: 3 short: File type (file, dir, or symlink). type: keyword uid: @@ -1167,6 +1268,7 @@ file: ignore_above: 1024 level: extended name: uid + order: 6 short: The user ID (UID) or security identifier (SID) of the file owner. type: keyword group: 2 @@ -1189,6 +1291,7 @@ geo: ignore_above: 1024 level: core name: city_name + order: 4 short: City name. type: keyword continent_name: @@ -1198,6 +1301,7 @@ geo: ignore_above: 1024 level: core name: continent_name + order: 1 short: Name of the continent. type: keyword country_iso_code: @@ -1207,6 +1311,7 @@ geo: ignore_above: 1024 level: core name: country_iso_code + order: 5 short: Country ISO code. type: keyword country_name: @@ -1216,6 +1321,7 @@ geo: ignore_above: 1024 level: core name: country_name + order: 2 short: Country name. type: keyword location: @@ -1224,6 +1330,7 @@ geo: flat_name: geo.location level: core name: location + order: 0 short: Longitude and latitude. type: geo_point name: @@ -1239,6 +1346,7 @@ geo: ignore_above: 1024 level: extended name: name + order: 7 short: User-defined description of a location. type: keyword region_iso_code: @@ -1248,6 +1356,7 @@ geo: ignore_above: 1024 level: core name: region_iso_code + order: 6 short: Region ISO code. type: keyword region_name: @@ -1257,6 +1366,7 @@ geo: ignore_above: 1024 level: core name: region_name + order: 3 short: Region name. type: keyword group: 2 @@ -1284,6 +1394,7 @@ group: ignore_above: 1024 level: extended name: id + order: 0 short: Unique identifier for the group on the system/platform. type: keyword name: @@ -1292,6 +1403,7 @@ group: ignore_above: 1024 level: extended name: name + order: 1 short: Name of the group. type: keyword group: 2 @@ -1318,6 +1430,7 @@ host: ignore_above: 1024 level: core name: architecture + order: 6 short: Operating system architecture. type: keyword geo.city_name: @@ -1327,6 +1440,7 @@ host: ignore_above: 1024 level: core name: city_name + order: 4 original_fieldset: geo short: City name. type: keyword @@ -1337,6 +1451,7 @@ host: ignore_above: 1024 level: core name: continent_name + order: 1 original_fieldset: geo short: Name of the continent. type: keyword @@ -1347,6 +1462,7 @@ host: ignore_above: 1024 level: core name: country_iso_code + order: 5 original_fieldset: geo short: Country ISO code. type: keyword @@ -1357,6 +1473,7 @@ host: ignore_above: 1024 level: core name: country_name + order: 2 original_fieldset: geo short: Country name. type: keyword @@ -1366,6 +1483,7 @@ host: flat_name: host.geo.location level: core name: location + order: 0 original_fieldset: geo short: Longitude and latitude. type: geo_point @@ -1382,6 +1500,7 @@ host: ignore_above: 1024 level: extended name: name + order: 7 original_fieldset: geo short: User-defined description of a location. type: keyword @@ -1392,6 +1511,7 @@ host: ignore_above: 1024 level: core name: region_iso_code + order: 6 original_fieldset: geo short: Region ISO code. type: keyword @@ -1402,6 +1522,7 @@ host: ignore_above: 1024 level: core name: region_name + order: 3 original_fieldset: geo short: Region name. type: keyword @@ -1413,6 +1534,7 @@ host: ignore_above: 1024 level: core name: hostname + order: 0 short: Hostname of the host. type: keyword id: @@ -1425,6 +1547,7 @@ host: ignore_above: 1024 level: core name: id + order: 2 short: Unique host id. type: keyword ip: @@ -1432,6 +1555,7 @@ host: flat_name: host.ip level: core name: ip + order: 3 short: Host ip address. type: ip mac: @@ -1440,6 +1564,7 @@ host: ignore_above: 1024 level: core name: mac + order: 4 short: Host mac address. type: keyword name: @@ -1452,6 +1577,7 @@ host: ignore_above: 1024 level: core name: name + order: 1 short: Name of the host. type: keyword os.family: @@ -1461,6 +1587,7 @@ host: ignore_above: 1024 level: extended name: family + order: 3 original_fieldset: os short: OS family (such as redhat, debian, freebsd, windows). type: keyword @@ -1471,6 +1598,7 @@ host: ignore_above: 1024 level: extended name: full + order: 2 original_fieldset: os short: Operating system name, including the version or code name. type: keyword @@ -1481,6 +1609,7 @@ host: ignore_above: 1024 level: extended name: kernel + order: 5 original_fieldset: os short: Operating system kernel version as a raw string. type: keyword @@ -1491,6 +1620,7 @@ host: ignore_above: 1024 level: extended name: name + order: 1 original_fieldset: os short: Operating system name, without the version. type: keyword @@ -1501,6 +1631,7 @@ host: ignore_above: 1024 level: extended name: platform + order: 0 original_fieldset: os short: Operating system platform (such centos, ubuntu, windows). type: keyword @@ -1511,6 +1642,7 @@ host: ignore_above: 1024 level: extended name: version + order: 4 original_fieldset: os short: Operating system version as a raw string. type: keyword @@ -1524,6 +1656,7 @@ host: ignore_above: 1024 level: core name: type + order: 5 short: Type of host. type: keyword user.email: @@ -1532,6 +1665,7 @@ host: ignore_above: 1024 level: extended name: email + order: 3 original_fieldset: user short: User email address. type: keyword @@ -1542,6 +1676,7 @@ host: ignore_above: 1024 level: extended name: full_name + order: 2 original_fieldset: user short: User's full name, if available. type: keyword @@ -1551,6 +1686,7 @@ host: ignore_above: 1024 level: extended name: id + order: 0 original_fieldset: user short: Unique identifier for the group on the system/platform. type: keyword @@ -1560,6 +1696,7 @@ host: ignore_above: 1024 level: extended name: name + order: 1 original_fieldset: user short: Name of the group. type: keyword @@ -1573,6 +1710,7 @@ host: ignore_above: 1024 level: extended name: hash + order: 4 original_fieldset: user short: Unique user hash to correlate information for a user in anonymized form. type: keyword @@ -1582,6 +1720,7 @@ host: ignore_above: 1024 level: core name: id + order: 0 original_fieldset: user short: One or multiple unique identifiers of the user. type: keyword @@ -1592,6 +1731,7 @@ host: ignore_above: 1024 level: core name: name + order: 1 original_fieldset: user short: Short name or login of the user. type: keyword @@ -1612,6 +1752,7 @@ http: format: bytes level: extended name: request.body.bytes + order: 7 short: Size in bytes of the request body. type: long request.body.content: @@ -1621,6 +1762,7 @@ http: ignore_above: 1024 level: extended name: request.body.content + order: 1 short: The full HTTP request body. type: keyword request.bytes: @@ -1630,6 +1772,7 @@ http: format: bytes level: extended name: request.bytes + order: 6 short: Total size in bytes of the request (body and headers). type: long request.method: @@ -1642,6 +1785,7 @@ http: ignore_above: 1024 level: extended name: request.method + order: 0 short: HTTP request method. type: keyword request.referrer: @@ -1651,6 +1795,7 @@ http: ignore_above: 1024 level: extended name: request.referrer + order: 2 short: Referrer for this HTTP request. type: keyword response.body.bytes: @@ -1660,6 +1805,7 @@ http: format: bytes level: extended name: response.body.bytes + order: 9 short: Size in bytes of the response body. type: long response.body.content: @@ -1669,6 +1815,7 @@ http: ignore_above: 1024 level: extended name: response.body.content + order: 4 short: The full HTTP response body. type: keyword response.bytes: @@ -1678,6 +1825,7 @@ http: format: bytes level: extended name: response.bytes + order: 8 short: Total size in bytes of the response (body and headers). type: long response.status_code: @@ -1686,6 +1834,7 @@ http: flat_name: http.response.status_code level: extended name: response.status_code + order: 3 short: HTTP response status code. type: long version: @@ -1695,6 +1844,7 @@ http: ignore_above: 1024 level: extended name: version + order: 5 short: HTTP version. type: keyword group: 2 @@ -1715,6 +1865,7 @@ log: ignore_above: 1024 level: core name: level + order: 0 short: Log level of the log event. type: keyword original: @@ -1735,6 +1886,7 @@ log: index: false level: core name: original + order: 1 short: Original log message with light interpretation only (encoding, newlines). type: keyword group: 2 @@ -1763,6 +1915,7 @@ network: ignore_above: 1024 level: extended name: application + order: 4 short: Application level protocol name. type: keyword bytes: @@ -1775,6 +1928,7 @@ network: format: bytes level: core name: bytes + order: 9 short: Total bytes transferred in both directions. type: long community_id: @@ -1788,6 +1942,7 @@ network: ignore_above: 1024 level: extended name: community_id + order: 8 short: A hash of source and destination IPs and ports. type: keyword direction: @@ -1802,6 +1957,7 @@ network: ignore_above: 1024 level: core name: direction + order: 6 short: Direction of the network traffic. type: keyword forwarded_ip: @@ -1810,6 +1966,7 @@ network: flat_name: network.forwarded_ip level: core name: forwarded_ip + order: 7 short: Host IP address when the source IP address is the proxy. type: ip iana_number: @@ -1821,6 +1978,7 @@ network: ignore_above: 1024 level: extended name: iana_number + order: 2 short: IANA Protocol Number. type: keyword name: @@ -1830,6 +1988,7 @@ network: ignore_above: 1024 level: extended name: name + order: 0 short: Name given by operators to sections of their network. type: keyword packets: @@ -1841,6 +2000,7 @@ network: flat_name: network.packets level: core name: packets + order: 10 short: Total packets transferred in both directions. type: long protocol: @@ -1853,6 +2013,7 @@ network: ignore_above: 1024 level: core name: protocol + order: 5 short: L7 Network protocol name. type: keyword transport: @@ -1866,6 +2027,7 @@ network: ignore_above: 1024 level: core name: transport + order: 3 short: Protocol Name corresponding to the field `iana_number`. type: keyword type: @@ -1879,6 +2041,7 @@ network: ignore_above: 1024 level: core name: type + order: 1 short: In the OSI Model this would be the Network Layer. ipv4, ipv6, ipsec, pim, etc type: keyword @@ -1909,6 +2072,7 @@ observer: ignore_above: 1024 level: core name: city_name + order: 4 original_fieldset: geo short: City name. type: keyword @@ -1919,6 +2083,7 @@ observer: ignore_above: 1024 level: core name: continent_name + order: 1 original_fieldset: geo short: Name of the continent. type: keyword @@ -1929,6 +2094,7 @@ observer: ignore_above: 1024 level: core name: country_iso_code + order: 5 original_fieldset: geo short: Country ISO code. type: keyword @@ -1939,6 +2105,7 @@ observer: ignore_above: 1024 level: core name: country_name + order: 2 original_fieldset: geo short: Country name. type: keyword @@ -1948,6 +2115,7 @@ observer: flat_name: observer.geo.location level: core name: location + order: 0 original_fieldset: geo short: Longitude and latitude. type: geo_point @@ -1964,6 +2132,7 @@ observer: ignore_above: 1024 level: extended name: name + order: 7 original_fieldset: geo short: User-defined description of a location. type: keyword @@ -1974,6 +2143,7 @@ observer: ignore_above: 1024 level: core name: region_iso_code + order: 6 original_fieldset: geo short: Region ISO code. type: keyword @@ -1984,6 +2154,7 @@ observer: ignore_above: 1024 level: core name: region_name + order: 3 original_fieldset: geo short: Region name. type: keyword @@ -1993,6 +2164,7 @@ observer: ignore_above: 1024 level: core name: hostname + order: 2 short: Hostname of the observer. type: keyword ip: @@ -2000,6 +2172,7 @@ observer: flat_name: observer.ip level: core name: ip + order: 1 short: IP address of the observer. type: ip mac: @@ -2008,6 +2181,7 @@ observer: ignore_above: 1024 level: core name: mac + order: 0 short: MAC address of the observer type: keyword os.family: @@ -2017,6 +2191,7 @@ observer: ignore_above: 1024 level: extended name: family + order: 3 original_fieldset: os short: OS family (such as redhat, debian, freebsd, windows). type: keyword @@ -2027,6 +2202,7 @@ observer: ignore_above: 1024 level: extended name: full + order: 2 original_fieldset: os short: Operating system name, including the version or code name. type: keyword @@ -2037,6 +2213,7 @@ observer: ignore_above: 1024 level: extended name: kernel + order: 5 original_fieldset: os short: Operating system kernel version as a raw string. type: keyword @@ -2047,6 +2224,7 @@ observer: ignore_above: 1024 level: extended name: name + order: 1 original_fieldset: os short: Operating system name, without the version. type: keyword @@ -2057,6 +2235,7 @@ observer: ignore_above: 1024 level: extended name: platform + order: 0 original_fieldset: os short: Operating system platform (such centos, ubuntu, windows). type: keyword @@ -2067,6 +2246,7 @@ observer: ignore_above: 1024 level: extended name: version + order: 4 original_fieldset: os short: Operating system version as a raw string. type: keyword @@ -2076,6 +2256,7 @@ observer: ignore_above: 1024 level: extended name: serial_number + order: 5 short: Observer serial number. type: keyword type: @@ -2088,6 +2269,7 @@ observer: ignore_above: 1024 level: core name: type + order: 6 short: The type of the observer the data is coming from. type: keyword vendor: @@ -2096,6 +2278,7 @@ observer: ignore_above: 1024 level: core name: vendor + order: 3 short: observer vendor information. type: keyword version: @@ -2104,6 +2287,7 @@ observer: ignore_above: 1024 level: core name: version + order: 4 short: Observer version. type: keyword group: 2 @@ -2125,6 +2309,7 @@ organization: ignore_above: 1024 level: extended name: id + order: 1 short: Unique identifier for the organization. type: keyword name: @@ -2133,6 +2318,7 @@ organization: ignore_above: 1024 level: extended name: name + order: 0 short: Organization name. type: keyword group: 2 @@ -2151,6 +2337,7 @@ os: ignore_above: 1024 level: extended name: family + order: 3 short: OS family (such as redhat, debian, freebsd, windows). type: keyword full: @@ -2160,6 +2347,7 @@ os: ignore_above: 1024 level: extended name: full + order: 2 short: Operating system name, including the version or code name. type: keyword kernel: @@ -2169,6 +2357,7 @@ os: ignore_above: 1024 level: extended name: kernel + order: 5 short: Operating system kernel version as a raw string. type: keyword name: @@ -2178,6 +2367,7 @@ os: ignore_above: 1024 level: extended name: name + order: 1 short: Operating system name, without the version. type: keyword platform: @@ -2187,6 +2377,7 @@ os: ignore_above: 1024 level: extended name: platform + order: 0 short: Operating system platform (such centos, ubuntu, windows). type: keyword version: @@ -2196,6 +2387,7 @@ os: ignore_above: 1024 level: extended name: version + order: 4 short: Operating system version as a raw string. type: keyword group: 2 @@ -2230,6 +2422,7 @@ process: ignore_above: 1024 level: extended name: args + order: 3 short: Array of process arguments. type: keyword executable: @@ -2239,6 +2432,7 @@ process: ignore_above: 1024 level: extended name: executable + order: 4 short: Absolute path to the process executable. type: keyword name: @@ -2250,6 +2444,7 @@ process: ignore_above: 1024 level: extended name: name + order: 1 short: Process name. type: keyword pid: @@ -2258,6 +2453,7 @@ process: flat_name: process.pid level: core name: pid + order: 0 short: Process id. type: long ppid: @@ -2265,6 +2461,7 @@ process: flat_name: process.ppid level: extended name: ppid + order: 2 short: Process parent id. type: long start: @@ -2273,6 +2470,7 @@ process: flat_name: process.start level: extended name: start + order: 7 short: The time the process started. type: date thread.id: @@ -2281,6 +2479,7 @@ process: flat_name: process.thread.id level: extended name: thread.id + order: 6 short: Thread ID. type: long title: @@ -2292,6 +2491,7 @@ process: ignore_above: 1024 level: extended name: title + order: 5 short: Process title. type: keyword working_directory: @@ -2301,6 +2501,7 @@ process: ignore_above: 1024 level: extended name: working_directory + order: 8 short: The working directory of the process. type: keyword group: 2 @@ -2326,6 +2527,7 @@ related: flat_name: related.ip level: extended name: ip + order: 0 short: All of the IPs seen on your event. type: ip group: 2 @@ -2362,6 +2564,7 @@ server: ignore_above: 1024 level: extended name: address + order: 0 short: Server network address. type: keyword bytes: @@ -2371,6 +2574,7 @@ server: format: bytes level: core name: bytes + order: 5 short: Bytes sent from the server to the client. type: long domain: @@ -2379,6 +2583,7 @@ server: ignore_above: 1024 level: core name: domain + order: 4 short: Server domain. type: keyword geo.city_name: @@ -2388,6 +2593,7 @@ server: ignore_above: 1024 level: core name: city_name + order: 4 original_fieldset: geo short: City name. type: keyword @@ -2398,6 +2604,7 @@ server: ignore_above: 1024 level: core name: continent_name + order: 1 original_fieldset: geo short: Name of the continent. type: keyword @@ -2408,6 +2615,7 @@ server: ignore_above: 1024 level: core name: country_iso_code + order: 5 original_fieldset: geo short: Country ISO code. type: keyword @@ -2418,6 +2626,7 @@ server: ignore_above: 1024 level: core name: country_name + order: 2 original_fieldset: geo short: Country name. type: keyword @@ -2427,6 +2636,7 @@ server: flat_name: server.geo.location level: core name: location + order: 0 original_fieldset: geo short: Longitude and latitude. type: geo_point @@ -2443,6 +2653,7 @@ server: ignore_above: 1024 level: extended name: name + order: 7 original_fieldset: geo short: User-defined description of a location. type: keyword @@ -2453,6 +2664,7 @@ server: ignore_above: 1024 level: core name: region_iso_code + order: 6 original_fieldset: geo short: Region ISO code. type: keyword @@ -2463,6 +2675,7 @@ server: ignore_above: 1024 level: core name: region_name + order: 3 original_fieldset: geo short: Region name. type: keyword @@ -2473,6 +2686,7 @@ server: flat_name: server.ip level: core name: ip + order: 1 short: IP address of the server. type: ip mac: @@ -2481,6 +2695,7 @@ server: ignore_above: 1024 level: core name: mac + order: 3 short: MAC address of the server. type: keyword packets: @@ -2489,6 +2704,7 @@ server: flat_name: server.packets level: core name: packets + order: 6 short: Packets sent from the server to the client. type: long port: @@ -2496,6 +2712,7 @@ server: flat_name: server.port level: core name: port + order: 2 short: Port of the server. type: long user.email: @@ -2504,6 +2721,7 @@ server: ignore_above: 1024 level: extended name: email + order: 3 original_fieldset: user short: User email address. type: keyword @@ -2514,6 +2732,7 @@ server: ignore_above: 1024 level: extended name: full_name + order: 2 original_fieldset: user short: User's full name, if available. type: keyword @@ -2523,6 +2742,7 @@ server: ignore_above: 1024 level: extended name: id + order: 0 original_fieldset: user short: Unique identifier for the group on the system/platform. type: keyword @@ -2532,6 +2752,7 @@ server: ignore_above: 1024 level: extended name: name + order: 1 original_fieldset: user short: Name of the group. type: keyword @@ -2545,6 +2766,7 @@ server: ignore_above: 1024 level: extended name: hash + order: 4 original_fieldset: user short: Unique user hash to correlate information for a user in anonymized form. type: keyword @@ -2554,6 +2776,7 @@ server: ignore_above: 1024 level: core name: id + order: 0 original_fieldset: user short: One or multiple unique identifiers of the user. type: keyword @@ -2564,6 +2787,7 @@ server: ignore_above: 1024 level: core name: name + order: 1 original_fieldset: user short: Short name or login of the user. type: keyword @@ -2588,6 +2812,7 @@ service: ignore_above: 1024 level: extended name: ephemeral_id + order: 5 short: Ephemeral identifier of this service. type: keyword id: @@ -2603,6 +2828,7 @@ service: ignore_above: 1024 level: core name: id + order: 0 short: Unique identifier of the running service. type: keyword name: @@ -2623,6 +2849,7 @@ service: ignore_above: 1024 level: core name: name + order: 1 short: Name of the service. type: keyword state: @@ -2631,6 +2858,7 @@ service: ignore_above: 1024 level: core name: state + order: 3 short: Current state of the service. type: keyword type: @@ -2646,6 +2874,7 @@ service: ignore_above: 1024 level: core name: type + order: 2 short: The type of the service. type: keyword version: @@ -2657,6 +2886,7 @@ service: ignore_above: 1024 level: core name: version + order: 4 short: Version of the service. type: keyword group: 2 @@ -2681,6 +2911,7 @@ source: ignore_above: 1024 level: extended name: address + order: 0 short: Source network address. type: keyword bytes: @@ -2690,6 +2921,7 @@ source: format: bytes level: core name: bytes + order: 5 short: Bytes sent from the source to the destination. type: long domain: @@ -2698,6 +2930,7 @@ source: ignore_above: 1024 level: core name: domain + order: 4 short: Source domain. type: keyword geo.city_name: @@ -2707,6 +2940,7 @@ source: ignore_above: 1024 level: core name: city_name + order: 4 original_fieldset: geo short: City name. type: keyword @@ -2717,6 +2951,7 @@ source: ignore_above: 1024 level: core name: continent_name + order: 1 original_fieldset: geo short: Name of the continent. type: keyword @@ -2727,6 +2962,7 @@ source: ignore_above: 1024 level: core name: country_iso_code + order: 5 original_fieldset: geo short: Country ISO code. type: keyword @@ -2737,6 +2973,7 @@ source: ignore_above: 1024 level: core name: country_name + order: 2 original_fieldset: geo short: Country name. type: keyword @@ -2746,6 +2983,7 @@ source: flat_name: source.geo.location level: core name: location + order: 0 original_fieldset: geo short: Longitude and latitude. type: geo_point @@ -2762,6 +3000,7 @@ source: ignore_above: 1024 level: extended name: name + order: 7 original_fieldset: geo short: User-defined description of a location. type: keyword @@ -2772,6 +3011,7 @@ source: ignore_above: 1024 level: core name: region_iso_code + order: 6 original_fieldset: geo short: Region ISO code. type: keyword @@ -2782,6 +3022,7 @@ source: ignore_above: 1024 level: core name: region_name + order: 3 original_fieldset: geo short: Region name. type: keyword @@ -2792,6 +3033,7 @@ source: flat_name: source.ip level: core name: ip + order: 1 short: IP address of the source. type: ip mac: @@ -2800,6 +3042,7 @@ source: ignore_above: 1024 level: core name: mac + order: 3 short: MAC address of the source. type: keyword packets: @@ -2808,6 +3051,7 @@ source: flat_name: source.packets level: core name: packets + order: 6 short: Packets sent from the source to the destination. type: long port: @@ -2815,6 +3059,7 @@ source: flat_name: source.port level: core name: port + order: 2 short: Port of the source. type: long user.email: @@ -2823,6 +3068,7 @@ source: ignore_above: 1024 level: extended name: email + order: 3 original_fieldset: user short: User email address. type: keyword @@ -2833,6 +3079,7 @@ source: ignore_above: 1024 level: extended name: full_name + order: 2 original_fieldset: user short: User's full name, if available. type: keyword @@ -2842,6 +3089,7 @@ source: ignore_above: 1024 level: extended name: id + order: 0 original_fieldset: user short: Unique identifier for the group on the system/platform. type: keyword @@ -2851,6 +3099,7 @@ source: ignore_above: 1024 level: extended name: name + order: 1 original_fieldset: user short: Name of the group. type: keyword @@ -2864,6 +3113,7 @@ source: ignore_above: 1024 level: extended name: hash + order: 4 original_fieldset: user short: Unique user hash to correlate information for a user in anonymized form. type: keyword @@ -2873,6 +3123,7 @@ source: ignore_above: 1024 level: core name: id + order: 0 original_fieldset: user short: One or multiple unique identifiers of the user. type: keyword @@ -2883,6 +3134,7 @@ source: ignore_above: 1024 level: core name: name + order: 1 original_fieldset: user short: Short name or login of the user. type: keyword @@ -2906,6 +3158,7 @@ url: ignore_above: 1024 level: extended name: domain + order: 3 short: Domain of the url. type: keyword fragment: @@ -2916,6 +3169,7 @@ url: ignore_above: 1024 level: extended name: fragment + order: 7 short: Portion of the url after the `#`. type: keyword full: @@ -2927,6 +3181,7 @@ url: ignore_above: 1024 level: extended name: full + order: 1 short: Full unparsed URL. type: keyword original: @@ -2941,6 +3196,7 @@ url: ignore_above: 1024 level: extended name: original + order: 0 short: Unmodified original url as seen in the event source. type: keyword password: @@ -2949,6 +3205,7 @@ url: ignore_above: 1024 level: extended name: password + order: 9 short: Password of the request. type: keyword path: @@ -2957,6 +3214,7 @@ url: ignore_above: 1024 level: extended name: path + order: 5 short: Path of the request, such as "/search". type: keyword port: @@ -2965,6 +3223,7 @@ url: flat_name: url.port level: extended name: port + order: 4 short: Port of the request, such as 443. type: long query: @@ -2979,6 +3238,7 @@ url: ignore_above: 1024 level: extended name: query + order: 6 short: Query string of the request. type: keyword scheme: @@ -2990,6 +3250,7 @@ url: ignore_above: 1024 level: extended name: scheme + order: 2 short: Scheme of the url. type: keyword username: @@ -2998,6 +3259,7 @@ url: ignore_above: 1024 level: extended name: username + order: 8 short: Username of the request. type: keyword group: 2 @@ -3019,6 +3281,7 @@ user: ignore_above: 1024 level: extended name: email + order: 3 short: User email address. type: keyword full_name: @@ -3028,6 +3291,7 @@ user: ignore_above: 1024 level: extended name: full_name + order: 2 short: User's full name, if available. type: keyword group.id: @@ -3036,6 +3300,7 @@ user: ignore_above: 1024 level: extended name: id + order: 0 original_fieldset: group short: Unique identifier for the group on the system/platform. type: keyword @@ -3045,6 +3310,7 @@ user: ignore_above: 1024 level: extended name: name + order: 1 original_fieldset: group short: Name of the group. type: keyword @@ -3058,6 +3324,7 @@ user: ignore_above: 1024 level: extended name: hash + order: 4 short: Unique user hash to correlate information for a user in anonymized form. type: keyword id: @@ -3066,6 +3333,7 @@ user: ignore_above: 1024 level: core name: id + order: 0 short: One or multiple unique identifiers of the user. type: keyword name: @@ -3075,6 +3343,7 @@ user: ignore_above: 1024 level: core name: name + order: 1 short: Short name or login of the user. type: keyword group: 2 @@ -3103,6 +3372,7 @@ user_agent: ignore_above: 1024 level: extended name: device.name + order: 3 short: Name of the device. type: keyword name: @@ -3112,6 +3382,7 @@ user_agent: ignore_above: 1024 level: extended name: name + order: 1 short: Name of the user agent. type: keyword original: @@ -3122,6 +3393,7 @@ user_agent: ignore_above: 1024 level: extended name: original + order: 0 short: Unparsed version of the user_agent. type: keyword os.family: @@ -3131,6 +3403,7 @@ user_agent: ignore_above: 1024 level: extended name: family + order: 3 original_fieldset: os short: OS family (such as redhat, debian, freebsd, windows). type: keyword @@ -3141,6 +3414,7 @@ user_agent: ignore_above: 1024 level: extended name: full + order: 2 original_fieldset: os short: Operating system name, including the version or code name. type: keyword @@ -3151,6 +3425,7 @@ user_agent: ignore_above: 1024 level: extended name: kernel + order: 5 original_fieldset: os short: Operating system kernel version as a raw string. type: keyword @@ -3161,6 +3436,7 @@ user_agent: ignore_above: 1024 level: extended name: name + order: 1 original_fieldset: os short: Operating system name, without the version. type: keyword @@ -3171,6 +3447,7 @@ user_agent: ignore_above: 1024 level: extended name: platform + order: 0 original_fieldset: os short: Operating system platform (such centos, ubuntu, windows). type: keyword @@ -3181,6 +3458,7 @@ user_agent: ignore_above: 1024 level: extended name: version + order: 4 original_fieldset: os short: Operating system version as a raw string. type: keyword @@ -3191,6 +3469,7 @@ user_agent: ignore_above: 1024 level: extended name: version + order: 2 short: Version of the user agent. type: keyword group: 2 diff --git a/scripts/schema_reader.py b/scripts/schema_reader.py index 66d2a4b5b6..5ee45c5e09 100644 --- a/scripts/schema_reader.py +++ b/scripts/schema_reader.py @@ -70,7 +70,8 @@ def schema_fields_as_dictionary(schema): """Re-nest the array of field names as a dictionary of 'fieldname' => { field definition }""" field_array = schema.pop('fields') schema['fields'] = {} - for field in field_array: + for order, field in enumerate(field_array): + field['order'] = order schema['fields'][field['name']] = field # Field definitions From 47a231419e81bdab3ab5df9fdb7a65f204e7548e Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Thu, 28 Feb 2019 21:47:07 -0500 Subject: [PATCH 06/28] Make the sorting function a little more flexible --- scripts/generators/asciidoc_fields.py | 22 +++++++++------ scripts/tests/test_asciidoc_fields.py | 40 +++++++++++++++++++-------- 2 files changed, 43 insertions(+), 19 deletions(-) diff --git a/scripts/generators/asciidoc_fields.py b/scripts/generators/asciidoc_fields.py index c23d128dc7..9a286634e8 100644 --- a/scripts/generators/asciidoc_fields.py +++ b/scripts/generators/asciidoc_fields.py @@ -8,12 +8,20 @@ def generate(ecs_nested, ecs_version): # Helpers -def sorted_by_group(dict): - triples = [] +def sorted_by_keys(dict, sort_keys): + if not isinstance(sort_keys, list): + sort_keys = [sort_keys] + tuples = [] for key in dict: nested = dict[key] - triples += [(nested['group'], key, nested)] - return sorted(triples) + + sort_criteria = [] + for sort_key in sort_keys: + sort_criteria.append(nested[sort_key]) + sort_criteria.append(nested) + tuples.append(sort_criteria) + + return list(map(lambda t: t[-1], sorted(tuples))) # Rendering @@ -23,8 +31,7 @@ def sorted_by_group(dict): def render_field_index(ecs_nested): page_text = index_header() - for triple in sorted_by_group(ecs_nested): - (group, fieldset_name, fieldset) = triple + for fieldset in sorted_by_keys(ecs_nested, ['group', 'name']): page_text += render_index_row(fieldset) page_text += table_footer() page_text += index_footer() @@ -43,8 +50,7 @@ def render_index_row(fieldset): def render_field_details(ecs_nested): page_text = '' - for triple in sorted_by_group(ecs_nested): - (group, fieldset_name, fieldset) = triple + for fieldset in sorted_by_keys(ecs_nested, ['group', 'name']): page_text += render_fieldset(fieldset) return page_text diff --git a/scripts/tests/test_asciidoc_fields.py b/scripts/tests/test_asciidoc_fields.py index 1df32ee543..e26a5e3b9a 100644 --- a/scripts/tests/test_asciidoc_fields.py +++ b/scripts/tests/test_asciidoc_fields.py @@ -3,21 +3,39 @@ class TestGeneratorsAsciidocFields(unittest.TestCase): - # dict_add_nested - def test_sorted_by_group(self): + def test_sorted_by_one_key(self): dict = { - 'agent': { 'group': 2 }, - 'base': { 'group': 1 }, - 'cloud': { 'group': 2 } + '@timestamp': { 'order': 0, 'name': '@timestamp' }, + 'message': { 'order': 3, 'name': 'message' }, + 'labels': { 'order': 1, 'name': 'labels' }, + 'tags': { 'order': 2, 'name': 'tags' } } - triples = asciidoc_fields.sorted_by_group(dict) - expected_triples = [ - (1, 'base', { 'group': 1 }), - (2, 'agent', { 'group': 2 }), - (2, 'cloud', { 'group': 2 }) + expected = [ + { 'order': 0, 'name': '@timestamp' }, + { 'order': 1, 'name': 'labels' }, + { 'order': 2, 'name': 'tags' }, + { 'order': 3, 'name': 'message' } ] - self.assertEqual(triples, expected_triples) + result = asciidoc_fields.sorted_by_keys(dict, 'order') + self.assertEqual(result, expected) + result = asciidoc_fields.sorted_by_keys(dict, ['order']) + self.assertEqual(result, expected) + + + def test_sorted_by_multiple_keys(self): + dict = { + 'cloud': { 'group': 2, 'name': 'cloud' }, + 'agent': { 'group': 2, 'name': 'agent' }, + 'base': { 'group': 1, 'name': 'base' }, + } + expected = [ + { 'group': 1, 'name': 'base' }, + { 'group': 2, 'name': 'agent' }, + { 'group': 2, 'name': 'cloud' } + ] + result = asciidoc_fields.sorted_by_keys(dict, ['group', 'name']) + self.assertEqual(result, expected) if __name__ == '__main__': From 3eee17201ef47d0e336ee5d7971261f0d6d35f7b Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Thu, 28 Feb 2019 21:49:54 -0500 Subject: [PATCH 07/28] Restore sort order from files in schemas/* when listing fields. --- docs/field-details.asciidoc | 1774 ++++++++++++------------- scripts/generators/asciidoc_fields.py | 4 +- 2 files changed, 889 insertions(+), 889 deletions(-) diff --git a/docs/field-details.asciidoc b/docs/field-details.asciidoc index 1f2948eee8..599eb48605 100644 --- a/docs/field-details.asciidoc +++ b/docs/field-details.asciidoc @@ -21,6 +21,17 @@ type: date // =============================================================== +| tags +| List of keywords used to tag each event. + +example: ["production", "env2"] + +| level: core + +type: keyword + +// =============================================================== + | labels | Custom key/value pairs. @@ -43,17 +54,6 @@ type: text // =============================================================== -| tags -| List of keywords used to tag each event. - -example: ["production", "env2"] - -| level: core - -type: keyword - -// =============================================================== - |===== [[ecs-agent]] @@ -68,21 +68,10 @@ Examples include Beats. Agents may also run on observers. ECS agent.* fields sha // =============================================================== -| agent.ephemeral_id -| Ephemeral identifier of this agent. - -example: 8a4f500f - -| level: extended - -type: keyword - -// =============================================================== - -| agent.id -| Unique identifier of this agent. +| agent.version +| Version of the agent. -example: 8a4f500d +example: 6.0.0-rc2 | level: core @@ -112,10 +101,10 @@ type: keyword // =============================================================== -| agent.version -| Version of the agent. +| agent.id +| Unique identifier of this agent. -example: 6.0.0-rc2 +example: 8a4f500d | level: core @@ -123,6 +112,17 @@ type: keyword // =============================================================== +| agent.ephemeral_id +| Ephemeral identifier of this agent. + +example: 8a4f500f + +| level: extended + +type: keyword + +// =============================================================== + |===== [[ecs-client]] @@ -149,19 +149,19 @@ type: keyword // =============================================================== -| client.bytes -| Bytes sent from the client to the server. +| client.geo.location +| Longitude and latitude. -example: 184 +example: { "lon": -73.614830, "lat": 45.505918 } | level: core -type: long +type: geo_point // =============================================================== -| client.domain -| Client domain. +| client.user.id +| One or multiple unique identifiers of the user. @@ -171,43 +171,43 @@ type: keyword // =============================================================== -| client.geo.city_name -| City name. +| client.user.group.id +| Unique identifier for the group on the system/platform. -example: Montreal -| level: core + +| level: extended type: keyword // =============================================================== -| client.geo.continent_name -| Name of the continent. +| client.ip +| IP address of the client. + -example: North America | level: core -type: keyword +type: ip // =============================================================== -| client.geo.country_iso_code -| Country ISO code. +| client.user.group.name +| Name of the group. -example: CA -| level: core + +| level: extended type: keyword // =============================================================== -| client.geo.country_name -| Country name. +| client.geo.continent_name +| Name of the continent. -example: Canada +example: North America | level: core @@ -215,32 +215,32 @@ type: keyword // =============================================================== -| client.geo.location -| Longitude and latitude. +| client.user.name +| Short name or login of the user. -example: { "lon": -73.614830, "lat": 45.505918 } +example: albert | level: core -type: geo_point +type: keyword // =============================================================== -| client.geo.name -| User-defined description of a location. +| client.port +| Port of the client. -example: boston-dc -| level: extended -type: keyword +| level: core + +type: long // =============================================================== -| client.geo.region_iso_code -| Region ISO code. +| client.geo.country_name +| Country name. -example: CA-QC +example: Canada | level: core @@ -248,63 +248,63 @@ type: keyword // =============================================================== -| client.geo.region_name -| Region name. +| client.user.full_name +| User's full name, if available. -example: Quebec +example: Albert Einstein -| level: core +| level: extended type: keyword // =============================================================== -| client.ip -| IP address of the client. +| client.mac +| MAC address of the client. | level: core -type: ip +type: keyword // =============================================================== -| client.mac -| MAC address of the client. +| client.user.email +| User email address. -| level: core +| level: extended type: keyword // =============================================================== -| client.packets -| Packets sent from the client to the server. +| client.geo.region_name +| Region name. -example: 12 +example: Quebec | level: core -type: long +type: keyword // =============================================================== -| client.port -| Port of the client. +| client.domain +| Client domain. | level: core -type: long +type: keyword // =============================================================== -| client.user.email -| User email address. +| client.user.hash +| Unique user hash to correlate information for a user in anonymized form. @@ -314,54 +314,54 @@ type: keyword // =============================================================== -| client.user.full_name -| User's full name, if available. +| client.geo.city_name +| City name. -example: Albert Einstein +example: Montreal -| level: extended +| level: core type: keyword // =============================================================== -| client.user.group.id -| Unique identifier for the group on the system/platform. - +| client.bytes +| Bytes sent from the client to the server. +example: 184 -| level: extended +| level: core -type: keyword +type: long // =============================================================== -| client.user.group.name -| Name of the group. - +| client.geo.country_iso_code +| Country ISO code. +example: CA -| level: extended +| level: core type: keyword // =============================================================== -| client.user.hash -| Unique user hash to correlate information for a user in anonymized form. - +| client.packets +| Packets sent from the client to the server. +example: 12 -| level: extended +| level: core -type: keyword +type: long // =============================================================== -| client.user.id -| One or multiple unique identifiers of the user. - +| client.geo.region_iso_code +| Region ISO code. +example: CA-QC | level: core @@ -369,12 +369,12 @@ type: keyword // =============================================================== -| client.user.name -| Short name or login of the user. +| client.geo.name +| User-defined description of a location. -example: albert +example: boston-dc -| level: core +| level: extended type: keyword @@ -393,10 +393,10 @@ Fields related to the cloud or infrastructure the events are coming from. // =============================================================== -| cloud.account.id -| The cloud account or organization id. +| cloud.provider +| Name of the cloud provider. -example: 666777888999 +example: aws | level: extended @@ -415,6 +415,17 @@ type: keyword // =============================================================== +| cloud.region +| Region in which this host is running. + +example: us-east-1 + +| level: extended + +type: keyword + +// =============================================================== + | cloud.instance.id | Instance ID of the host machine. @@ -448,21 +459,10 @@ type: keyword // =============================================================== -| cloud.provider -| Name of the cloud provider. - -example: aws - -| level: extended - -type: keyword - -// =============================================================== - -| cloud.region -| Region in which this host is running. +| cloud.account.id +| The cloud account or organization id. -example: us-east-1 +example: 666777888999 | level: extended @@ -484,6 +484,17 @@ These fields help correlate data based containers from any runtime. // =============================================================== +| container.runtime +| Runtime managing this container. + +example: docker + +| level: extended + +type: keyword + +// =============================================================== + | container.id | Unique container id. @@ -517,17 +528,6 @@ type: keyword // =============================================================== -| container.labels -| Image labels. - - - -| level: extended - -type: object - -// =============================================================== - | container.name | Container name. @@ -539,14 +539,14 @@ type: keyword // =============================================================== -| container.runtime -| Runtime managing this container. +| container.labels +| Image labels. + -example: docker | level: extended -type: keyword +type: object // =============================================================== @@ -575,19 +575,19 @@ type: keyword // =============================================================== -| destination.bytes -| Bytes sent from the destination to the source. +| destination.geo.location +| Longitude and latitude. -example: 184 +example: { "lon": -73.614830, "lat": 45.505918 } | level: core -type: long +type: geo_point // =============================================================== -| destination.domain -| Destination domain. +| destination.user.id +| One or multiple unique identifiers of the user. @@ -597,43 +597,43 @@ type: keyword // =============================================================== -| destination.geo.city_name -| City name. +| destination.user.group.id +| Unique identifier for the group on the system/platform. -example: Montreal -| level: core + +| level: extended type: keyword // =============================================================== -| destination.geo.continent_name -| Name of the continent. +| destination.ip +| IP address of the destination. + -example: North America | level: core -type: keyword +type: ip // =============================================================== -| destination.geo.country_iso_code -| Country ISO code. +| destination.user.group.name +| Name of the group. -example: CA -| level: core + +| level: extended type: keyword // =============================================================== -| destination.geo.country_name -| Country name. +| destination.geo.continent_name +| Name of the continent. -example: Canada +example: North America | level: core @@ -641,32 +641,32 @@ type: keyword // =============================================================== -| destination.geo.location -| Longitude and latitude. +| destination.user.name +| Short name or login of the user. -example: { "lon": -73.614830, "lat": 45.505918 } +example: albert | level: core -type: geo_point +type: keyword // =============================================================== -| destination.geo.name -| User-defined description of a location. +| destination.port +| Port of the destination. -example: boston-dc -| level: extended -type: keyword +| level: core + +type: long // =============================================================== -| destination.geo.region_iso_code -| Region ISO code. +| destination.geo.country_name +| Country name. -example: CA-QC +example: Canada | level: core @@ -674,63 +674,63 @@ type: keyword // =============================================================== -| destination.geo.region_name -| Region name. +| destination.user.full_name +| User's full name, if available. -example: Quebec +example: Albert Einstein -| level: core +| level: extended type: keyword // =============================================================== -| destination.ip -| IP address of the destination. +| destination.mac +| MAC address of the destination. | level: core -type: ip +type: keyword // =============================================================== -| destination.mac -| MAC address of the destination. +| destination.user.email +| User email address. -| level: core +| level: extended type: keyword // =============================================================== -| destination.packets -| Packets sent from the destination to the source. +| destination.geo.region_name +| Region name. -example: 12 +example: Quebec | level: core -type: long +type: keyword // =============================================================== -| destination.port -| Port of the destination. +| destination.domain +| Destination domain. | level: core -type: long +type: keyword // =============================================================== -| destination.user.email -| User email address. +| destination.user.hash +| Unique user hash to correlate information for a user in anonymized form. @@ -740,54 +740,54 @@ type: keyword // =============================================================== -| destination.user.full_name -| User's full name, if available. +| destination.geo.city_name +| City name. -example: Albert Einstein +example: Montreal -| level: extended +| level: core type: keyword // =============================================================== -| destination.user.group.id -| Unique identifier for the group on the system/platform. - +| destination.bytes +| Bytes sent from the destination to the source. +example: 184 -| level: extended +| level: core -type: keyword +type: long // =============================================================== -| destination.user.group.name -| Name of the group. - +| destination.geo.country_iso_code +| Country ISO code. +example: CA -| level: extended +| level: core type: keyword // =============================================================== -| destination.user.hash -| Unique user hash to correlate information for a user in anonymized form. - +| destination.packets +| Packets sent from the destination to the source. +example: 12 -| level: extended +| level: core -type: keyword +type: long // =============================================================== -| destination.user.id -| One or multiple unique identifiers of the user. - +| destination.geo.region_iso_code +| Region ISO code. +example: CA-QC | level: core @@ -795,12 +795,12 @@ type: keyword // =============================================================== -| destination.user.name -| Short name or login of the user. +| destination.geo.name +| User-defined description of a location. -example: albert +example: boston-dc -| level: core +| level: extended type: keyword @@ -844,8 +844,8 @@ Use them for errors that happen while fetching events or in cases where the even // =============================================================== -| error.code -| Error code describing the error. +| error.id +| Unique identifier for the error. @@ -855,25 +855,25 @@ type: keyword // =============================================================== -| error.id -| Unique identifier for the error. +| error.message +| Error message. | level: core -type: keyword +type: text // =============================================================== -| error.message -| Error message. +| error.code +| Error code describing the error. | level: core -type: text +type: keyword // =============================================================== @@ -891,10 +891,10 @@ A log is defined as an event containing details of something that happened. Log // =============================================================== -| event.action -| The action captured by the event. +| event.id +| Unique ID to describe the event. -example: user-password-change +example: 8a4f500d | level: core @@ -902,32 +902,21 @@ type: keyword // =============================================================== -| event.category -| Event category. +| event.kind +| The kind of the event. -example: user-management +example: state -| level: core +| level: extended type: keyword // =============================================================== -| event.created -| Time when the event was first read by an agent or by your pipeline. - - - -| level: core - -type: date - -// =============================================================== - -| event.dataset -| Name of the dataset. +| event.category +| Event category. -example: stats +example: user-management | level: core @@ -935,43 +924,44 @@ type: keyword // =============================================================== -| event.duration -| Duration of the event in nanoseconds. - +| event.action +| The action captured by the event. +example: user-password-change | level: core -type: long +type: keyword // =============================================================== -| event.end -| event.end contains the date when the event ended or when the activity was last observed. - +| event.outcome +| The outcome of the event. +example: success | level: extended -type: date +type: keyword // =============================================================== -| event.hash -| Hash (perhaps logstash fingerprint) of raw field to be able to demonstrate log integrity. +| event.type +| Reserved for future usage. -example: 123456789012345678901234567890ABCD -| level: extended + +| level: core type: keyword // =============================================================== -| event.id -| Unique ID to describe the event. +| event.module +| Name of the module this data is coming from. +This information is coming from the modules used in Beats or Logstash. -example: 8a4f500d +example: mysql | level: core @@ -979,26 +969,25 @@ type: keyword // =============================================================== -| event.kind -| The kind of the event. +| event.dataset +| Name of the dataset. -example: state +example: stats -| level: extended +| level: core type: keyword // =============================================================== -| event.module -| Name of the module this data is coming from. -This information is coming from the modules used in Beats or Logstash. +| event.severity +| Original severity of the event. -example: mysql +example: 7 | level: core -type: keyword +type: long // =============================================================== @@ -1013,10 +1002,10 @@ type: keyword // =============================================================== -| event.outcome -| The outcome of the event. +| event.hash +| Hash (perhaps logstash fingerprint) of raw field to be able to demonstrate log integrity. -example: success +example: 123456789012345678901234567890ABCD | level: extended @@ -1024,36 +1013,36 @@ type: keyword // =============================================================== -| event.risk_score -| Risk score or priority of the event (e.g. security solutions). Use your system's original value here. +| event.duration +| Duration of the event in nanoseconds. | level: core -type: float +type: long // =============================================================== -| event.risk_score_norm -| Normalized risk score or priority of the event (0-100). +| event.timezone +| Event time zone. | level: extended -type: float +type: keyword // =============================================================== -| event.severity -| Original severity of the event. +| event.created +| Time when the event was first read by an agent or by your pipeline. + -example: 7 | level: core -type: long +type: date // =============================================================== @@ -1068,25 +1057,36 @@ type: date // =============================================================== -| event.timezone -| Event time zone. +| event.end +| event.end contains the date when the event ended or when the activity was last observed. | level: extended -type: keyword +type: date // =============================================================== -| event.type -| Reserved for future usage. +| event.risk_score +| Risk score or priority of the event (e.g. security solutions). Use your system's original value here. | level: core -type: keyword +type: float + +// =============================================================== + +| event.risk_score_norm +| Normalized risk score or priority of the event (0-100). + + + +| level: extended + +type: float // =============================================================== @@ -1104,19 +1104,19 @@ File objects can be associated with host events, network events, and/or file eve // =============================================================== -| file.ctime -| Last time file metadata changed. +| file.path +| Path to the file. | level: extended -type: date +type: keyword // =============================================================== -| file.device -| Device that is the source of the file. +| file.target_path +| Target path for symlinks. @@ -1137,8 +1137,8 @@ type: keyword // =============================================================== -| file.gid -| Primary group ID (GID) of the file. +| file.type +| File type (file, dir, or symlink). @@ -1148,8 +1148,8 @@ type: keyword // =============================================================== -| file.group -| Primary group name of the file. +| file.device +| Device that is the source of the file. @@ -1170,10 +1170,10 @@ type: keyword // =============================================================== -| file.mode -| Mode of the file in octal representation. +| file.uid +| The user ID (UID) or security identifier (SID) of the file owner. + -example: 416 | level: extended @@ -1181,19 +1181,19 @@ type: keyword // =============================================================== -| file.mtime -| Last time file content was modified. +| file.owner +| File owner's username. | level: extended -type: date +type: keyword // =============================================================== -| file.owner -| File owner's username. +| file.gid +| Primary group ID (GID) of the file. @@ -1203,8 +1203,8 @@ type: keyword // =============================================================== -| file.path -| Path to the file. +| file.group +| Primary group name of the file. @@ -1214,47 +1214,47 @@ type: keyword // =============================================================== -| file.size -| File size in bytes (field is only added when `type` is `file`). - +| file.mode +| Mode of the file in octal representation. +example: 416 | level: extended -type: long +type: keyword // =============================================================== -| file.target_path -| Target path for symlinks. +| file.size +| File size in bytes (field is only added when `type` is `file`). | level: extended -type: keyword +type: long // =============================================================== -| file.type -| File type (file, dir, or symlink). +| file.mtime +| Last time file content was modified. | level: extended -type: keyword +type: date // =============================================================== -| file.uid -| The user ID (UID) or security identifier (SID) of the file owner. +| file.ctime +| Last time file metadata changed. | level: extended -type: keyword +type: date // =============================================================== @@ -1272,14 +1272,14 @@ This geolocation information can be derived from techniques such as Geo IP, or b // =============================================================== -| geo.city_name -| City name. +| geo.location +| Longitude and latitude. -example: Montreal +example: { "lon": -73.614830, "lat": 45.505918 } | level: core -type: keyword +type: geo_point // =============================================================== @@ -1294,10 +1294,10 @@ type: keyword // =============================================================== -| geo.country_iso_code -| Country ISO code. +| geo.country_name +| Country name. -example: CA +example: Canada | level: core @@ -1305,10 +1305,10 @@ type: keyword // =============================================================== -| geo.country_name -| Country name. +| geo.region_name +| Region name. -example: Canada +example: Quebec | level: core @@ -1316,23 +1316,23 @@ type: keyword // =============================================================== -| geo.location -| Longitude and latitude. +| geo.city_name +| City name. -example: { "lon": -73.614830, "lat": 45.505918 } +example: Montreal | level: core -type: geo_point +type: keyword // =============================================================== -| geo.name -| User-defined description of a location. +| geo.country_iso_code +| Country ISO code. -example: boston-dc +example: CA -| level: extended +| level: core type: keyword @@ -1349,12 +1349,12 @@ type: keyword // =============================================================== -| geo.region_name -| Region name. +| geo.name +| User-defined description of a location. -example: Quebec +example: boston-dc -| level: core +| level: extended type: keyword @@ -1409,10 +1409,10 @@ ECS host.* fields should be populated with details about the host on which the e // =============================================================== -| host.architecture -| Operating system architecture. +| host.hostname +| Hostname of the host. + -example: x86_64 | level: core @@ -1420,21 +1420,21 @@ type: keyword // =============================================================== -| host.geo.city_name -| City name. +| host.geo.location +| Longitude and latitude. -example: Montreal +example: { "lon": -73.614830, "lat": 45.505918 } | level: core -type: keyword +type: geo_point // =============================================================== -| host.geo.continent_name -| Name of the continent. +| host.user.id +| One or multiple unique identifiers of the user. + -example: North America | level: core @@ -1442,21 +1442,32 @@ type: keyword // =============================================================== -| host.geo.country_iso_code -| Country ISO code. +| host.user.group.id +| Unique identifier for the group on the system/platform. -example: CA -| level: core + +| level: extended type: keyword // =============================================================== -| host.geo.country_name -| Country name. +| host.os.platform +| Operating system platform (such centos, ubuntu, windows). + +example: darwin + +| level: extended + +type: keyword + +// =============================================================== + +| host.name +| Name of the host. + -example: Canada | level: core @@ -1464,21 +1475,32 @@ type: keyword // =============================================================== -| host.geo.location -| Longitude and latitude. +| host.user.group.name +| Name of the group. -example: { "lon": -73.614830, "lat": 45.505918 } + + +| level: extended + +type: keyword + +// =============================================================== + +| host.geo.continent_name +| Name of the continent. + +example: North America | level: core -type: geo_point +type: keyword // =============================================================== -| host.geo.name -| User-defined description of a location. +| host.os.name +| Operating system name, without the version. -example: boston-dc +example: Mac OS X | level: extended @@ -1486,10 +1508,10 @@ type: keyword // =============================================================== -| host.geo.region_iso_code -| Region ISO code. +| host.user.name +| Short name or login of the user. -example: CA-QC +example: albert | level: core @@ -1497,10 +1519,10 @@ type: keyword // =============================================================== -| host.geo.region_name -| Region name. +| host.id +| Unique host id. + -example: Quebec | level: core @@ -1508,10 +1530,10 @@ type: keyword // =============================================================== -| host.hostname -| Hostname of the host. - +| host.geo.country_name +| Country name. +example: Canada | level: core @@ -1519,45 +1541,45 @@ type: keyword // =============================================================== -| host.id -| Unique host id. - +| host.os.full +| Operating system name, including the version or code name. +example: Mac OS Mojave -| level: core +| level: extended type: keyword // =============================================================== -| host.ip -| Host ip address. - +| host.user.full_name +| User's full name, if available. +example: Albert Einstein -| level: core +| level: extended -type: ip +type: keyword // =============================================================== -| host.mac -| Host mac address. +| host.ip +| Host ip address. | level: core -type: keyword +type: ip // =============================================================== -| host.name -| Name of the host. +| host.user.email +| User email address. -| level: core +| level: extended type: keyword @@ -1574,32 +1596,32 @@ type: keyword // =============================================================== -| host.os.full -| Operating system name, including the version or code name. +| host.geo.region_name +| Region name. -example: Mac OS Mojave +example: Quebec -| level: extended +| level: core type: keyword // =============================================================== -| host.os.kernel -| Operating system kernel version as a raw string. +| host.mac +| Host mac address. -example: 4.4.0-112-generic -| level: extended + +| level: core type: keyword // =============================================================== -| host.os.name -| Operating system name, without the version. +| host.user.hash +| Unique user hash to correlate information for a user in anonymized form. + -example: Mac OS X | level: extended @@ -1607,12 +1629,12 @@ type: keyword // =============================================================== -| host.os.platform -| Operating system platform (such centos, ubuntu, windows). +| host.geo.city_name +| City name. -example: darwin +example: Montreal -| level: extended +| level: core type: keyword @@ -1640,43 +1662,21 @@ type: keyword // =============================================================== -| host.user.email -| User email address. - - - -| level: extended - -type: keyword - -// =============================================================== - -| host.user.full_name -| User's full name, if available. - -example: Albert Einstein - -| level: extended - -type: keyword - -// =============================================================== - -| host.user.group.id -| Unique identifier for the group on the system/platform. - +| host.geo.country_iso_code +| Country ISO code. +example: CA -| level: extended +| level: core type: keyword // =============================================================== -| host.user.group.name -| Name of the group. - +| host.os.kernel +| Operating system kernel version as a raw string. +example: 4.4.0-112-generic | level: extended @@ -1684,21 +1684,21 @@ type: keyword // =============================================================== -| host.user.hash -| Unique user hash to correlate information for a user in anonymized form. - +| host.architecture +| Operating system architecture. +example: x86_64 -| level: extended +| level: core type: keyword // =============================================================== -| host.user.id -| One or multiple unique identifiers of the user. - +| host.geo.region_iso_code +| Region ISO code. +example: CA-QC | level: core @@ -1706,12 +1706,12 @@ type: keyword // =============================================================== -| host.user.name -| Short name or login of the user. +| host.geo.name +| User-defined description of a location. -example: albert +example: boston-dc -| level: core +| level: extended type: keyword @@ -1730,14 +1730,14 @@ Fields related to HTTP activity. Use the `url` field set to store the url of the // =============================================================== -| http.request.body.bytes -| Size in bytes of the request body. +| http.request.method +| HTTP request method. -example: 887 +example: get, post, put | level: extended -type: long +type: keyword // =============================================================== @@ -1752,32 +1752,32 @@ type: keyword // =============================================================== -| http.request.bytes -| Total size in bytes of the request (body and headers). +| http.request.referrer +| Referrer for this HTTP request. -example: 1437 +example: https://blog.example.com/ | level: extended -type: long +type: keyword // =============================================================== -| http.request.method -| HTTP request method. +| http.response.status_code +| HTTP response status code. -example: get, post, put +example: 404 | level: extended -type: keyword +type: long // =============================================================== -| http.request.referrer -| Referrer for this HTTP request. +| http.response.body.content +| The full HTTP response body. -example: https://blog.example.com/ +example: Hello world | level: extended @@ -1785,32 +1785,32 @@ type: keyword // =============================================================== -| http.response.body.bytes -| Size in bytes of the response body. +| http.version +| HTTP version. -example: 887 +example: 1.1 | level: extended -type: long +type: keyword // =============================================================== -| http.response.body.content -| The full HTTP response body. +| http.request.bytes +| Total size in bytes of the request (body and headers). -example: Hello world +example: 1437 | level: extended -type: keyword +type: long // =============================================================== -| http.response.bytes -| Total size in bytes of the response (body and headers). +| http.request.body.bytes +| Size in bytes of the request body. -example: 1437 +example: 887 | level: extended @@ -1818,10 +1818,10 @@ type: long // =============================================================== -| http.response.status_code -| HTTP response status code. +| http.response.bytes +| Total size in bytes of the response (body and headers). -example: 404 +example: 1437 | level: extended @@ -1829,14 +1829,14 @@ type: long // =============================================================== -| http.version -| HTTP version. +| http.response.body.bytes +| Size in bytes of the response body. -example: 1.1 +example: 887 | level: extended -type: keyword +type: long // =============================================================== @@ -1889,10 +1889,10 @@ The network.* fields should be populated with details about the network activity // =============================================================== -| network.application -| Application level protocol name. +| network.name +| Name given by operators to sections of their network. -example: aim +example: Guest Wifi | level: extended @@ -1900,21 +1900,21 @@ type: keyword // =============================================================== -| network.bytes -| Total bytes transferred in both directions. +| network.type +| In the OSI Model this would be the Network Layer. ipv4, ipv6, ipsec, pim, etc -example: 368 +example: ipv4 | level: core -type: long +type: keyword // =============================================================== -| network.community_id -| A hash of source and destination IPs and ports. +| network.iana_number +| IANA Protocol Number. -example: 1:hO+sN4H+MG5MY/8hIrXPqc4ZQz0= +example: 6 | level: extended @@ -1922,10 +1922,10 @@ type: keyword // =============================================================== -| network.direction -| Direction of the network traffic. +| network.transport +| Protocol Name corresponding to the field `iana_number`. -example: inbound +example: tcp | level: core @@ -1933,80 +1933,80 @@ type: keyword // =============================================================== -| network.forwarded_ip -| Host IP address when the source IP address is the proxy. +| network.application +| Application level protocol name. -example: 192.1.1.2 +example: aim -| level: core +| level: extended -type: ip +type: keyword // =============================================================== -| network.iana_number -| IANA Protocol Number. +| network.protocol +| L7 Network protocol name. -example: 6 +example: http -| level: extended +| level: core type: keyword // =============================================================== -| network.name -| Name given by operators to sections of their network. +| network.direction +| Direction of the network traffic. -example: Guest Wifi +example: inbound -| level: extended +| level: core type: keyword // =============================================================== -| network.packets -| Total packets transferred in both directions. +| network.forwarded_ip +| Host IP address when the source IP address is the proxy. -example: 24 +example: 192.1.1.2 | level: core -type: long +type: ip // =============================================================== -| network.protocol -| L7 Network protocol name. +| network.community_id +| A hash of source and destination IPs and ports. -example: http +example: 1:hO+sN4H+MG5MY/8hIrXPqc4ZQz0= -| level: core +| level: extended type: keyword // =============================================================== -| network.transport -| Protocol Name corresponding to the field `iana_number`. +| network.bytes +| Total bytes transferred in both directions. -example: tcp +example: 368 | level: core -type: keyword +type: long // =============================================================== -| network.type -| In the OSI Model this would be the Network Layer. ipv4, ipv6, ipsec, pim, etc +| network.packets +| Total packets transferred in both directions. -example: ipv4 +example: 24 | level: core -type: keyword +type: long // =============================================================== @@ -2024,10 +2024,10 @@ This could be a custom hardware appliance or a server that has been configured t // =============================================================== -| observer.geo.city_name -| City name. +| observer.mac +| MAC address of the observer + -example: Montreal | level: core @@ -2035,54 +2035,54 @@ type: keyword // =============================================================== -| observer.geo.continent_name -| Name of the continent. +| observer.geo.location +| Longitude and latitude. -example: North America +example: { "lon": -73.614830, "lat": 45.505918 } | level: core -type: keyword +type: geo_point // =============================================================== -| observer.geo.country_iso_code -| Country ISO code. +| observer.os.platform +| Operating system platform (such centos, ubuntu, windows). -example: CA +example: darwin -| level: core +| level: extended type: keyword // =============================================================== -| observer.geo.country_name -| Country name. +| observer.ip +| IP address of the observer. + -example: Canada | level: core -type: keyword +type: ip // =============================================================== -| observer.geo.location -| Longitude and latitude. +| observer.geo.continent_name +| Name of the continent. -example: { "lon": -73.614830, "lat": 45.505918 } +example: North America | level: core -type: geo_point +type: keyword // =============================================================== -| observer.geo.name -| User-defined description of a location. +| observer.os.name +| Operating system name, without the version. -example: boston-dc +example: Mac OS X | level: extended @@ -2090,10 +2090,10 @@ type: keyword // =============================================================== -| observer.geo.region_iso_code -| Region ISO code. +| observer.hostname +| Hostname of the observer. + -example: CA-QC | level: core @@ -2101,10 +2101,10 @@ type: keyword // =============================================================== -| observer.geo.region_name -| Region name. +| observer.geo.country_name +| Country name. -example: Quebec +example: Canada | level: core @@ -2112,76 +2112,76 @@ type: keyword // =============================================================== -| observer.hostname -| Hostname of the observer. - +| observer.os.full +| Operating system name, including the version or code name. +example: Mac OS Mojave -| level: core +| level: extended type: keyword // =============================================================== -| observer.ip -| IP address of the observer. +| observer.vendor +| observer vendor information. | level: core -type: ip +type: keyword // =============================================================== -| observer.mac -| MAC address of the observer - +| observer.os.family +| OS family (such as redhat, debian, freebsd, windows). +example: debian -| level: core +| level: extended type: keyword // =============================================================== -| observer.os.family -| OS family (such as redhat, debian, freebsd, windows). +| observer.geo.region_name +| Region name. -example: debian +example: Quebec -| level: extended +| level: core type: keyword // =============================================================== -| observer.os.full -| Operating system name, including the version or code name. +| observer.version +| Observer version. -example: Mac OS Mojave -| level: extended + +| level: core type: keyword // =============================================================== -| observer.os.kernel -| Operating system kernel version as a raw string. +| observer.geo.city_name +| City name. -example: 4.4.0-112-generic +example: Montreal -| level: extended +| level: core type: keyword // =============================================================== -| observer.os.name -| Operating system name, without the version. +| observer.os.version +| Operating system version as a raw string. -example: Mac OS X +example: 10.14.1 | level: extended @@ -2189,10 +2189,10 @@ type: keyword // =============================================================== -| observer.os.platform -| Operating system platform (such centos, ubuntu, windows). +| observer.serial_number +| Observer serial number. + -example: darwin | level: extended @@ -2200,21 +2200,21 @@ type: keyword // =============================================================== -| observer.os.version -| Operating system version as a raw string. +| observer.geo.country_iso_code +| Country ISO code. -example: 10.14.1 +example: CA -| level: extended +| level: core type: keyword // =============================================================== -| observer.serial_number -| Observer serial number. - +| observer.os.kernel +| Operating system kernel version as a raw string. +example: 4.4.0-112-generic | level: extended @@ -2233,10 +2233,10 @@ type: keyword // =============================================================== -| observer.vendor -| observer vendor information. - +| observer.geo.region_iso_code +| Region ISO code. +example: CA-QC | level: core @@ -2244,12 +2244,12 @@ type: keyword // =============================================================== -| observer.version -| Observer version. - +| observer.geo.name +| User-defined description of a location. +example: boston-dc -| level: core +| level: extended type: keyword @@ -2269,8 +2269,8 @@ These fields help you arrange or filter data stored in an index by one or multip // =============================================================== -| organization.id -| Unique identifier for the organization. +| organization.name +| Organization name. @@ -2280,8 +2280,8 @@ type: keyword // =============================================================== -| organization.name -| Organization name. +| organization.id +| Unique identifier for the organization. @@ -2304,10 +2304,10 @@ The OS fields contain information about the operating system. // =============================================================== -| os.family -| OS family (such as redhat, debian, freebsd, windows). +| os.platform +| Operating system platform (such centos, ubuntu, windows). -example: debian +example: darwin | level: extended @@ -2315,10 +2315,10 @@ type: keyword // =============================================================== -| os.full -| Operating system name, including the version or code name. +| os.name +| Operating system name, without the version. -example: Mac OS Mojave +example: Mac OS X | level: extended @@ -2326,10 +2326,10 @@ type: keyword // =============================================================== -| os.kernel -| Operating system kernel version as a raw string. +| os.full +| Operating system name, including the version or code name. -example: 4.4.0-112-generic +example: Mac OS Mojave | level: extended @@ -2337,10 +2337,10 @@ type: keyword // =============================================================== -| os.name -| Operating system name, without the version. +| os.family +| OS family (such as redhat, debian, freebsd, windows). -example: Mac OS X +example: debian | level: extended @@ -2348,10 +2348,10 @@ type: keyword // =============================================================== -| os.platform -| Operating system platform (such centos, ubuntu, windows). +| os.version +| Operating system version as a raw string. -example: darwin +example: 10.14.1 | level: extended @@ -2359,10 +2359,10 @@ type: keyword // =============================================================== -| os.version -| Operating system version as a raw string. +| os.kernel +| Operating system kernel version as a raw string. -example: 10.14.1 +example: 4.4.0-112-generic | level: extended @@ -2384,25 +2384,14 @@ These fields can help you correlate metrics information with a process id/name f // =============================================================== -| process.args -| Array of process arguments. - -example: ['ssh', '-l', 'user', '10.0.0.16'] - -| level: extended - -type: keyword - -// =============================================================== +| process.pid +| Process id. -| process.executable -| Absolute path to the process executable. -example: /usr/bin/ssh -| level: extended +| level: core -type: keyword +type: long // =============================================================== @@ -2417,36 +2406,47 @@ type: keyword // =============================================================== -| process.pid -| Process id. +| process.ppid +| Process parent id. -| level: core +| level: extended type: long // =============================================================== -| process.ppid -| Process parent id. +| process.args +| Array of process arguments. + +example: ['ssh', '-l', 'user', '10.0.0.16'] + +| level: extended + +type: keyword + +// =============================================================== +| process.executable +| Absolute path to the process executable. +example: /usr/bin/ssh | level: extended -type: long +type: keyword // =============================================================== -| process.start -| The time the process started. +| process.title +| Process title. + -example: 2016-05-23T08:05:34.853Z | level: extended -type: date +type: keyword // =============================================================== @@ -2461,14 +2461,14 @@ type: long // =============================================================== -| process.title -| Process title. - +| process.start +| The time the process started. +example: 2016-05-23T08:05:34.853Z | level: extended -type: keyword +type: date // =============================================================== @@ -2535,19 +2535,19 @@ type: keyword // =============================================================== -| server.bytes -| Bytes sent from the server to the client. +| server.geo.location +| Longitude and latitude. -example: 184 +example: { "lon": -73.614830, "lat": 45.505918 } | level: core -type: long +type: geo_point // =============================================================== -| server.domain -| Server domain. +| server.user.id +| One or multiple unique identifiers of the user. @@ -2557,43 +2557,43 @@ type: keyword // =============================================================== -| server.geo.city_name -| City name. +| server.user.group.id +| Unique identifier for the group on the system/platform. -example: Montreal -| level: core + +| level: extended type: keyword // =============================================================== -| server.geo.continent_name -| Name of the continent. +| server.ip +| IP address of the server. + -example: North America | level: core -type: keyword +type: ip // =============================================================== -| server.geo.country_iso_code -| Country ISO code. +| server.user.group.name +| Name of the group. -example: CA -| level: core + +| level: extended type: keyword // =============================================================== -| server.geo.country_name -| Country name. +| server.geo.continent_name +| Name of the continent. -example: Canada +example: North America | level: core @@ -2601,32 +2601,32 @@ type: keyword // =============================================================== -| server.geo.location -| Longitude and latitude. +| server.user.name +| Short name or login of the user. -example: { "lon": -73.614830, "lat": 45.505918 } +example: albert | level: core -type: geo_point +type: keyword // =============================================================== -| server.geo.name -| User-defined description of a location. +| server.port +| Port of the server. -example: boston-dc -| level: extended -type: keyword +| level: core + +type: long // =============================================================== -| server.geo.region_iso_code -| Region ISO code. +| server.geo.country_name +| Country name. -example: CA-QC +example: Canada | level: core @@ -2634,63 +2634,63 @@ type: keyword // =============================================================== -| server.geo.region_name -| Region name. +| server.user.full_name +| User's full name, if available. -example: Quebec +example: Albert Einstein -| level: core +| level: extended type: keyword // =============================================================== -| server.ip -| IP address of the server. +| server.mac +| MAC address of the server. | level: core -type: ip +type: keyword // =============================================================== -| server.mac -| MAC address of the server. +| server.user.email +| User email address. -| level: core +| level: extended type: keyword // =============================================================== -| server.packets -| Packets sent from the server to the client. +| server.geo.region_name +| Region name. -example: 12 +example: Quebec | level: core -type: long +type: keyword // =============================================================== -| server.port -| Port of the server. +| server.domain +| Server domain. | level: core -type: long +type: keyword // =============================================================== -| server.user.email -| User email address. +| server.user.hash +| Unique user hash to correlate information for a user in anonymized form. @@ -2700,54 +2700,54 @@ type: keyword // =============================================================== -| server.user.full_name -| User's full name, if available. +| server.geo.city_name +| City name. -example: Albert Einstein +example: Montreal -| level: extended +| level: core type: keyword // =============================================================== -| server.user.group.id -| Unique identifier for the group on the system/platform. - +| server.bytes +| Bytes sent from the server to the client. +example: 184 -| level: extended +| level: core -type: keyword +type: long // =============================================================== -| server.user.group.name -| Name of the group. - +| server.geo.country_iso_code +| Country ISO code. +example: CA -| level: extended +| level: core type: keyword // =============================================================== -| server.user.hash -| Unique user hash to correlate information for a user in anonymized form. - +| server.packets +| Packets sent from the server to the client. +example: 12 -| level: extended +| level: core -type: keyword +type: long // =============================================================== -| server.user.id -| One or multiple unique identifiers of the user. - +| server.geo.region_iso_code +| Region ISO code. +example: CA-QC | level: core @@ -2755,12 +2755,12 @@ type: keyword // =============================================================== -| server.user.name -| Short name or login of the user. +| server.geo.name +| User-defined description of a location. -example: albert +example: boston-dc -| level: core +| level: extended type: keyword @@ -2780,17 +2780,6 @@ These fields help you find and correlate logs for a specific service and version // =============================================================== -| service.ephemeral_id -| Ephemeral identifier of this service. - -example: 8a4f500f - -| level: extended - -type: keyword - -// =============================================================== - | service.id | Unique identifier of the running service. @@ -2813,10 +2802,10 @@ type: keyword // =============================================================== -| service.state -| Current state of the service. - +| service.type +| The type of the service. +example: elasticsearch | level: core @@ -2824,10 +2813,10 @@ type: keyword // =============================================================== -| service.type -| The type of the service. +| service.state +| Current state of the service. + -example: elasticsearch | level: core @@ -2846,6 +2835,17 @@ type: keyword // =============================================================== +| service.ephemeral_id +| Ephemeral identifier of this service. + +example: 8a4f500f + +| level: extended + +type: keyword + +// =============================================================== + |===== [[ecs-source]] @@ -2871,19 +2871,19 @@ type: keyword // =============================================================== -| source.bytes -| Bytes sent from the source to the destination. +| source.geo.location +| Longitude and latitude. -example: 184 +example: { "lon": -73.614830, "lat": 45.505918 } | level: core -type: long +type: geo_point // =============================================================== -| source.domain -| Source domain. +| source.user.id +| One or multiple unique identifiers of the user. @@ -2893,43 +2893,43 @@ type: keyword // =============================================================== -| source.geo.city_name -| City name. +| source.user.group.id +| Unique identifier for the group on the system/platform. -example: Montreal -| level: core + +| level: extended type: keyword // =============================================================== -| source.geo.continent_name -| Name of the continent. +| source.ip +| IP address of the source. + -example: North America | level: core -type: keyword +type: ip // =============================================================== -| source.geo.country_iso_code -| Country ISO code. +| source.user.group.name +| Name of the group. -example: CA -| level: core + +| level: extended type: keyword // =============================================================== -| source.geo.country_name -| Country name. +| source.geo.continent_name +| Name of the continent. -example: Canada +example: North America | level: core @@ -2937,32 +2937,32 @@ type: keyword // =============================================================== -| source.geo.location -| Longitude and latitude. +| source.user.name +| Short name or login of the user. -example: { "lon": -73.614830, "lat": 45.505918 } +example: albert | level: core -type: geo_point +type: keyword // =============================================================== -| source.geo.name -| User-defined description of a location. +| source.port +| Port of the source. -example: boston-dc -| level: extended -type: keyword +| level: core + +type: long // =============================================================== -| source.geo.region_iso_code -| Region ISO code. +| source.geo.country_name +| Country name. -example: CA-QC +example: Canada | level: core @@ -2970,63 +2970,63 @@ type: keyword // =============================================================== -| source.geo.region_name -| Region name. +| source.user.full_name +| User's full name, if available. -example: Quebec +example: Albert Einstein -| level: core +| level: extended type: keyword // =============================================================== -| source.ip -| IP address of the source. +| source.mac +| MAC address of the source. | level: core -type: ip +type: keyword // =============================================================== -| source.mac -| MAC address of the source. +| source.user.email +| User email address. -| level: core +| level: extended type: keyword // =============================================================== -| source.packets -| Packets sent from the source to the destination. +| source.geo.region_name +| Region name. -example: 12 +example: Quebec | level: core -type: long +type: keyword // =============================================================== -| source.port -| Port of the source. +| source.domain +| Source domain. | level: core -type: long +type: keyword // =============================================================== -| source.user.email -| User email address. +| source.user.hash +| Unique user hash to correlate information for a user in anonymized form. @@ -3036,54 +3036,54 @@ type: keyword // =============================================================== -| source.user.full_name -| User's full name, if available. +| source.geo.city_name +| City name. -example: Albert Einstein +example: Montreal -| level: extended +| level: core type: keyword // =============================================================== -| source.user.group.id -| Unique identifier for the group on the system/platform. - +| source.bytes +| Bytes sent from the source to the destination. +example: 184 -| level: extended +| level: core -type: keyword +type: long // =============================================================== -| source.user.group.name -| Name of the group. - +| source.geo.country_iso_code +| Country ISO code. +example: CA -| level: extended +| level: core type: keyword // =============================================================== -| source.user.hash -| Unique user hash to correlate information for a user in anonymized form. - +| source.packets +| Packets sent from the source to the destination. +example: 12 -| level: extended +| level: core -type: keyword +type: long // =============================================================== -| source.user.id -| One or multiple unique identifiers of the user. - +| source.geo.region_iso_code +| Region ISO code. +example: CA-QC | level: core @@ -3091,12 +3091,12 @@ type: keyword // =============================================================== -| source.user.name -| Short name or login of the user. +| source.geo.name +| User-defined description of a location. -example: albert +example: boston-dc -| level: core +| level: extended type: keyword @@ -3115,10 +3115,10 @@ URL fields provide support for complete or partial URLs, and supports the breaki // =============================================================== -| url.domain -| Domain of the url. +| url.original +| Unmodified original url as seen in the event source. -example: www.elastic.co +example: https://www.elastic.co:443/search?q=elasticsearch#top or /search?q=elasticsearch | level: extended @@ -3126,10 +3126,10 @@ type: keyword // =============================================================== -| url.fragment -| Portion of the url after the `#`. - +| url.full +| Full unparsed URL. +example: https://www.elastic.co:443/search?q=elasticsearch#top | level: extended @@ -3137,10 +3137,10 @@ type: keyword // =============================================================== -| url.full -| Full unparsed URL. +| url.scheme +| Scheme of the url. -example: https://www.elastic.co:443/search?q=elasticsearch#top +example: https | level: extended @@ -3148,10 +3148,10 @@ type: keyword // =============================================================== -| url.original -| Unmodified original url as seen in the event source. +| url.domain +| Domain of the url. -example: https://www.elastic.co:443/search?q=elasticsearch#top or /search?q=elasticsearch +example: www.elastic.co | level: extended @@ -3159,14 +3159,14 @@ type: keyword // =============================================================== -| url.password -| Password of the request. - +| url.port +| Port of the request, such as 443. +example: 443 | level: extended -type: keyword +type: long // =============================================================== @@ -3181,19 +3181,19 @@ type: keyword // =============================================================== -| url.port -| Port of the request, such as 443. +| url.query +| Query string of the request. + -example: 443 | level: extended -type: long +type: keyword // =============================================================== -| url.query -| Query string of the request. +| url.fragment +| Portion of the url after the `#`. @@ -3203,10 +3203,10 @@ type: keyword // =============================================================== -| url.scheme -| Scheme of the url. +| url.username +| Username of the request. + -example: https | level: extended @@ -3214,8 +3214,8 @@ type: keyword // =============================================================== -| url.username -| Username of the request. +| url.password +| Password of the request. @@ -3239,21 +3239,21 @@ Fields can have one entry or multiple entries. If a user has more than one id, p // =============================================================== -| user.email -| User email address. +| user.id +| One or multiple unique identifiers of the user. -| level: extended +| level: core type: keyword // =============================================================== -| user.full_name -| User's full name, if available. +| user.group.id +| Unique identifier for the group on the system/platform. + -example: Albert Einstein | level: extended @@ -3261,8 +3261,8 @@ type: keyword // =============================================================== -| user.group.id -| Unique identifier for the group on the system/platform. +| user.group.name +| Name of the group. @@ -3272,21 +3272,21 @@ type: keyword // =============================================================== -| user.group.name -| Name of the group. - +| user.name +| Short name or login of the user. +example: albert -| level: extended +| level: core type: keyword // =============================================================== -| user.hash -| Unique user hash to correlate information for a user in anonymized form. - +| user.full_name +| User's full name, if available. +example: Albert Einstein | level: extended @@ -3294,23 +3294,23 @@ type: keyword // =============================================================== -| user.id -| One or multiple unique identifiers of the user. +| user.email +| User email address. -| level: core +| level: extended type: keyword // =============================================================== -| user.name -| Short name or login of the user. +| user.hash +| Unique user hash to correlate information for a user in anonymized form. -example: albert -| level: core + +| level: extended type: keyword @@ -3330,10 +3330,10 @@ They often show up in web service logs coming from the parsed user agent string. // =============================================================== -| user_agent.device.name -| Name of the device. +| user_agent.original +| Unparsed version of the user_agent. -example: iPhone +example: Mozilla/5.0 (iPhone; CPU iPhone OS 12_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.0 Mobile/15E148 Safari/604.1 | level: extended @@ -3341,10 +3341,10 @@ type: keyword // =============================================================== -| user_agent.name -| Name of the user agent. +| user_agent.os.platform +| Operating system platform (such centos, ubuntu, windows). -example: Safari +example: darwin | level: extended @@ -3352,10 +3352,10 @@ type: keyword // =============================================================== -| user_agent.original -| Unparsed version of the user_agent. +| user_agent.name +| Name of the user agent. -example: Mozilla/5.0 (iPhone; CPU iPhone OS 12_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.0 Mobile/15E148 Safari/604.1 +example: Safari | level: extended @@ -3363,10 +3363,10 @@ type: keyword // =============================================================== -| user_agent.os.family -| OS family (such as redhat, debian, freebsd, windows). +| user_agent.os.name +| Operating system name, without the version. -example: debian +example: Mac OS X | level: extended @@ -3374,10 +3374,10 @@ type: keyword // =============================================================== -| user_agent.os.full -| Operating system name, including the version or code name. +| user_agent.version +| Version of the user agent. -example: Mac OS Mojave +example: 12.0 | level: extended @@ -3385,10 +3385,10 @@ type: keyword // =============================================================== -| user_agent.os.kernel -| Operating system kernel version as a raw string. +| user_agent.os.full +| Operating system name, including the version or code name. -example: 4.4.0-112-generic +example: Mac OS Mojave | level: extended @@ -3396,10 +3396,10 @@ type: keyword // =============================================================== -| user_agent.os.name -| Operating system name, without the version. +| user_agent.device.name +| Name of the device. -example: Mac OS X +example: iPhone | level: extended @@ -3407,10 +3407,10 @@ type: keyword // =============================================================== -| user_agent.os.platform -| Operating system platform (such centos, ubuntu, windows). +| user_agent.os.family +| OS family (such as redhat, debian, freebsd, windows). -example: darwin +example: debian | level: extended @@ -3429,10 +3429,10 @@ type: keyword // =============================================================== -| user_agent.version -| Version of the user agent. +| user_agent.os.kernel +| Operating system kernel version as a raw string. -example: 12.0 +example: 4.4.0-112-generic | level: extended diff --git a/scripts/generators/asciidoc_fields.py b/scripts/generators/asciidoc_fields.py index 9a286634e8..32524d9a1d 100644 --- a/scripts/generators/asciidoc_fields.py +++ b/scripts/generators/asciidoc_fields.py @@ -61,8 +61,8 @@ def render_fieldset(fieldset): fieldset_description=fieldset['description'], fieldset_title=fieldset['title'] ) - for field_name in sorted(fieldset['fields']): - fieldset_text += render_field(fieldset['fields'][field_name]) + for field in sorted_by_keys(fieldset['fields'], 'order'): + fieldset_text += render_field(field) fieldset_text += table_footer() return fieldset_text From 0f3bd597cb595967c9fc8da483ff34ceb50e49c0 Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Thu, 28 Feb 2019 22:01:01 -0500 Subject: [PATCH 08/28] Add a check that short field descriptions are single line --- scripts/schema_reader.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/scripts/schema_reader.py b/scripts/schema_reader.py index 5ee45c5e09..099487658b 100644 --- a/scripts/schema_reader.py +++ b/scripts/schema_reader.py @@ -79,8 +79,8 @@ def schema_fields_as_dictionary(schema): def field_cleanup_values(field, prefix): dict_clean_string_values(field) - field_set_defaults(field) field_set_flat_name(field, prefix) + field_set_defaults(field) def field_set_flat_name(field, prefix): @@ -88,7 +88,6 @@ def field_set_flat_name(field, prefix): def field_set_defaults(field): - dict_set_default(field, 'short', field['description']) if field['type'] == 'keyword': dict_set_default(field, 'ignore_above', 1024) if field['type'] == 'text': @@ -96,6 +95,11 @@ def field_set_defaults(field): if field['type'] == 'object': dict_set_default(field, 'object_type', 'keyword') + dict_set_default(field, 'short', field['description']) + if "\n" in field['short']: + raise ValueError("Short descriptions must be single line.\nField: {}\n{}".format(field['flat_name'], field)) + # print(" Short descriptions must be single line. Field: {}".format(field['flat_name'])) + if 'index' in field and not field['index']: dict_set_default(field, 'doc_values', False) if 'multi_fields' in field: From ed1ee432b7c82e650ada545759c3dc392857fa7f Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Thu, 28 Feb 2019 22:18:31 -0500 Subject: [PATCH 09/28] Fix one field short description, based on new check --- docs/field-details.asciidoc | 1 - generated/ecs/fields_nested.yml | 4 +--- schemas/event.yml | 1 + 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/docs/field-details.asciidoc b/docs/field-details.asciidoc index 599eb48605..6158439ef6 100644 --- a/docs/field-details.asciidoc +++ b/docs/field-details.asciidoc @@ -959,7 +959,6 @@ type: keyword | event.module | Name of the module this data is coming from. -This information is coming from the modules used in Beats or Logstash. example: mysql diff --git a/generated/ecs/fields_nested.yml b/generated/ecs/fields_nested.yml index 648e80f276..f4bf0bd082 100644 --- a/generated/ecs/fields_nested.yml +++ b/generated/ecs/fields_nested.yml @@ -1026,9 +1026,7 @@ event: level: core name: module order: 6 - short: 'Name of the module this data is coming from. - - This information is coming from the modules used in Beats or Logstash.' + short: Name of the module this data is coming from. type: keyword original: description: 'Raw text message of entire event. Used to demonstrate log integrity. diff --git a/schemas/event.yml b/schemas/event.yml index aae5a38393..aa16008d7d 100644 --- a/schemas/event.yml +++ b/schemas/event.yml @@ -84,6 +84,7 @@ - name: module level: core type: keyword + short: Name of the module this data is coming from. description: > Name of the module this data is coming from. From a0e3148c7d16ce5fd26b7010a8e04eeadfc8ffc4 Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Thu, 28 Feb 2019 22:19:26 -0500 Subject: [PATCH 10/28] Default fieldset short desc to full desc, add single line check --- generated/ecs/fields_flat.yml | 4 +--- generated/ecs/fields_nested.yml | 2 ++ scripts/schema_reader.py | 4 ++++ 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/generated/ecs/fields_flat.yml b/generated/ecs/fields_flat.yml index d4537c2837..89d4b968a4 100644 --- a/generated/ecs/fields_flat.yml +++ b/generated/ecs/fields_flat.yml @@ -859,9 +859,7 @@ event.module: level: core name: module order: 6 - short: 'Name of the module this data is coming from. - - This information is coming from the modules used in Beats or Logstash.' + short: Name of the module this data is coming from. type: keyword event.original: description: 'Raw text message of entire event. Used to demonstrate log integrity. diff --git a/generated/ecs/fields_nested.yml b/generated/ecs/fields_nested.yml index f4bf0bd082..9d62f21376 100644 --- a/generated/ecs/fields_nested.yml +++ b/generated/ecs/fields_nested.yml @@ -840,6 +840,7 @@ ecs: group: 2 name: ecs prefix: ecs. + short: Meta-information specific to ECS. title: ECS type: group error: @@ -1890,6 +1891,7 @@ log: group: 2 name: log prefix: log. + short: Fields which are specific to log events. title: Log type: group network: diff --git a/scripts/schema_reader.py b/scripts/schema_reader.py index 099487658b..f972927cef 100644 --- a/scripts/schema_reader.py +++ b/scripts/schema_reader.py @@ -57,6 +57,10 @@ def schema_cleanup_values(schema): def schema_set_default_values(schema): schema['type'] = 'group' dict_set_default(schema, 'group', 2) + dict_set_default(schema, 'short', schema['description']) + if "\n" in schema['short']: + raise ValueError("Short descriptions must be single line.\nFieldset: {}\n{}".format(schema['name'], schema)) + # print("Short descriptions must be single line. Fieldset: {}".format(schema['name'])) def schema_set_fieldset_prefix(schema): From 86fa351c26e1aaa6d5c3033c547842aaa83bd68a Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Thu, 28 Feb 2019 22:20:41 -0500 Subject: [PATCH 11/28] Run generator for everything, code format --- generated/legacy/fields.yml | 1 + scripts/tests/test_asciidoc_fields.py | 29 +++++++++++++-------------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/generated/legacy/fields.yml b/generated/legacy/fields.yml index 1fb2711b83..fb935dc0db 100644 --- a/generated/legacy/fields.yml +++ b/generated/legacy/fields.yml @@ -528,6 +528,7 @@ - name: module level: core type: keyword + short: Name of the module this data is coming from. description: > Name of the module this data is coming from. diff --git a/scripts/tests/test_asciidoc_fields.py b/scripts/tests/test_asciidoc_fields.py index e26a5e3b9a..dc8161ac36 100644 --- a/scripts/tests/test_asciidoc_fields.py +++ b/scripts/tests/test_asciidoc_fields.py @@ -6,33 +6,32 @@ class TestGeneratorsAsciidocFields(unittest.TestCase): def test_sorted_by_one_key(self): dict = { - '@timestamp': { 'order': 0, 'name': '@timestamp' }, - 'message': { 'order': 3, 'name': 'message' }, - 'labels': { 'order': 1, 'name': 'labels' }, - 'tags': { 'order': 2, 'name': 'tags' } + '@timestamp': {'order': 0, 'name': '@timestamp'}, + 'message': {'order': 3, 'name': 'message'}, + 'labels': {'order': 1, 'name': 'labels'}, + 'tags': {'order': 2, 'name': 'tags'} } expected = [ - { 'order': 0, 'name': '@timestamp' }, - { 'order': 1, 'name': 'labels' }, - { 'order': 2, 'name': 'tags' }, - { 'order': 3, 'name': 'message' } + {'order': 0, 'name': '@timestamp'}, + {'order': 1, 'name': 'labels'}, + {'order': 2, 'name': 'tags'}, + {'order': 3, 'name': 'message'} ] result = asciidoc_fields.sorted_by_keys(dict, 'order') self.assertEqual(result, expected) result = asciidoc_fields.sorted_by_keys(dict, ['order']) self.assertEqual(result, expected) - def test_sorted_by_multiple_keys(self): dict = { - 'cloud': { 'group': 2, 'name': 'cloud' }, - 'agent': { 'group': 2, 'name': 'agent' }, - 'base': { 'group': 1, 'name': 'base' }, + 'cloud': {'group': 2, 'name': 'cloud'}, + 'agent': {'group': 2, 'name': 'agent'}, + 'base': {'group': 1, 'name': 'base'}, } expected = [ - { 'group': 1, 'name': 'base' }, - { 'group': 2, 'name': 'agent' }, - { 'group': 2, 'name': 'cloud' } + {'group': 1, 'name': 'base'}, + {'group': 2, 'name': 'agent'}, + {'group': 2, 'name': 'cloud'} ] result = asciidoc_fields.sorted_by_keys(dict, ['group', 'name']) self.assertEqual(result, expected) From 955e6f11f2d06e3fae4e061cece0d926310aa075 Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Thu, 28 Feb 2019 22:29:30 -0500 Subject: [PATCH 12/28] Adjust tests. Setting schema defaults now depends on 'description' to be there --- scripts/tests/test_schema_reader.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/tests/test_schema_reader.py b/scripts/tests/test_schema_reader.py index bb7de86a05..f5fca996a9 100644 --- a/scripts/tests/test_schema_reader.py +++ b/scripts/tests/test_schema_reader.py @@ -38,14 +38,14 @@ def test_schema_set_fieldset_prefix_not_at_root(self): {'prefix': 'myfieldset.', 'root': False, 'name': 'myfieldset'}) def test_set_default_values_defaults(self): - schema = {} + schema = {'description': '...'} schema_reader.schema_set_default_values(schema) - self.assertEqual(schema, {'group': 2, 'type': 'group'}) + self.assertEqual(schema, {'group': 2, 'type': 'group', 'description': '...', 'short': '...'}) def test_set_default_values_no_overwrite(self): - schema = {'group': 1} + schema = {'group': 1, 'description': '...'} schema_reader.schema_set_default_values(schema) - self.assertEqual(schema, {'group': 1, 'type': 'group'}) + self.assertEqual(schema, {'group': 1, 'type': 'group', 'description': '...', 'short': '...'}) # field definitions From 375f79ef8aeac0710afc7124fb58a71c5cbc71d7 Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Thu, 28 Feb 2019 23:46:48 -0500 Subject: [PATCH 13/28] Simplify summary table, add first draft of field details below table --- docs/field-details.asciidoc | 5507 +++++++++++++++++-------- scripts/generators/asciidoc_fields.py | 93 +- 2 files changed, 3916 insertions(+), 1684 deletions(-) diff --git a/docs/field-details.asciidoc b/docs/field-details.asciidoc index 6158439ef6..053c6f740d 100644 --- a/docs/field-details.asciidoc +++ b/docs/field-details.asciidoc @@ -4,127 +4,212 @@ The `base` field set contains all fields which are on the top level. These fields are common across all types of events. +==== Fields summary + [options="header"] |===== -| Field | Description | Level / Type +| Field | Description | Type (Level) // =============================================================== | @timestamp | Date/time when the event originated. +| date -example: 2016-05-23T08:05:34.853Z - -| level: core - -type: date +(core) // =============================================================== | tags | List of keywords used to tag each event. +| keyword -example: ["production", "env2"] - -| level: core - -type: keyword +(core) // =============================================================== | labels | Custom key/value pairs. +| object -example: {'application': 'foo-bar', 'env': 'production'} - -| level: core - -type: object +(core) // =============================================================== | message | Log message optimized for viewing in a log viewer. +| text -example: Hello World - -| level: core - -type: text +(core) // =============================================================== |===== +[[ecs-base-details]] +==== Field details + +===== @timestamp + +Level: core + +Datatype: date + +Date/time when the event originated. +This is the date/time extracted from the event, typically representing when the event was generated by the source. +If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. +Required field for all events. + +example: `2016-05-23T08:05:34.853Z` + +===== tags + +Level: core + +Datatype: keyword + +List of keywords used to tag each event. + +example: `["production", "env2"]` + +===== labels + +Level: core + +Datatype: object + +Custom key/value pairs. +Can be used to add meta information to events. Should not contain nested objects. All values are stored as keyword. +Example: `docker` and `k8s` labels. + +example: `{'application': 'foo-bar', 'env': 'production'}` + +===== message + +Level: core + +Datatype: text + +For log events the message field contains the log message, optimized for viewing in a log viewer. +For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. +If multiple messages exist, they can be combined into one message. + +example: `Hello World` + [[ecs-agent]] === Agent fields The agent fields contain the data about the software entity, if any, that collects, detects, or observes events on a host, or takes measurements on a host. Examples include Beats. Agents may also run on observers. ECS agent.* fields shall be populated with details of the agent running on the host or observer where the event happened or the measurement was taken. +==== Fields summary + [options="header"] |===== -| Field | Description | Level / Type +| Field | Description | Type (Level) // =============================================================== | agent.version | Version of the agent. +| keyword -example: 6.0.0-rc2 - -| level: core - -type: keyword +(core) // =============================================================== | agent.name | Custom name of the agent. +| keyword -example: foo - -| level: core - -type: keyword +(core) // =============================================================== | agent.type | Type of the agent. +| keyword -example: filebeat - -| level: core - -type: keyword +(core) // =============================================================== | agent.id | Unique identifier of this agent. +| keyword -example: 8a4f500d - -| level: core - -type: keyword +(core) // =============================================================== | agent.ephemeral_id | Ephemeral identifier of this agent. +| keyword -example: 8a4f500f - -| level: extended - -type: keyword +(extended) // =============================================================== |===== +[[ecs-agent-details]] +==== Field details + +===== agent.version + +Level: core + +Datatype: keyword + +Version of the agent. + +example: `6.0.0-rc2` + +===== agent.name + +Level: core + +Datatype: keyword + +Custom name of the agent. +This is a name that can be given to an agent. This can be helpful if for example two Filebeat instances are running on the same host but a human readable separation is needed on which Filebeat instance data is coming from. +If no name is given, the name is often left empty. + +example: `foo` + +===== agent.type + +Level: core + +Datatype: keyword + +Type of the agent. +The agent type stays always the same and should be given by the agent used. In case of Filebeat the agent would always be Filebeat also if two Filebeat instances are run on the same machine. + +example: `filebeat` + +===== agent.id + +Level: core + +Datatype: keyword + +Unique identifier of this agent (if one exists). +Example: For Beats this would be beat.id. + +example: `8a4f500d` + +===== agent.ephemeral_id + +Level: extended + +Datatype: keyword + +Ephemeral identifier of this agent (if one exists). +This id normally changes across restarts, but `agent.id` does not. + +example: `8a4f500f` + [[ecs-client]] === Client fields @@ -132,3190 +217,5219 @@ A client is defined as the initiator of a network connection for events regardin For TCP events, the client is the initiator of the TCP connection that sends the SYN packet(s). For other protocols, the client is generally the initiator or requestor in the network transaction. Some systems use the term "originator" to refer the client in TCP connections. The client fields describe details about the system acting as the client in the network event. Client fields are usually populated in conjunction with server fields. Client fields are generally not populated for packet-level events. Client / server representations can add semantic context to an exchange, which is helpful to visualize the data in certain situations. If your context falls in that category, you should still ensure that source and destination are filled appropriately. +==== Fields summary + [options="header"] |===== -| Field | Description | Level / Type +| Field | Description | Type (Level) // =============================================================== | client.address | Client network address. +| keyword - - -| level: extended - -type: keyword +(extended) // =============================================================== | client.geo.location | Longitude and latitude. +| geo_point -example: { "lon": -73.614830, "lat": 45.505918 } - -| level: core - -type: geo_point +(core) // =============================================================== | client.user.id | One or multiple unique identifiers of the user. +| keyword - - -| level: core - -type: keyword +(core) // =============================================================== | client.user.group.id | Unique identifier for the group on the system/platform. +| keyword - - -| level: extended - -type: keyword +(extended) // =============================================================== | client.ip | IP address of the client. +| ip - - -| level: core - -type: ip +(core) // =============================================================== | client.user.group.name | Name of the group. +| keyword - - -| level: extended - -type: keyword +(extended) // =============================================================== | client.geo.continent_name | Name of the continent. +| keyword -example: North America - -| level: core - -type: keyword +(core) // =============================================================== | client.user.name | Short name or login of the user. +| keyword -example: albert - -| level: core - -type: keyword +(core) // =============================================================== | client.port | Port of the client. +| long - - -| level: core - -type: long +(core) // =============================================================== | client.geo.country_name | Country name. +| keyword -example: Canada - -| level: core - -type: keyword +(core) // =============================================================== | client.user.full_name | User's full name, if available. +| keyword -example: Albert Einstein - -| level: extended - -type: keyword +(extended) // =============================================================== | client.mac | MAC address of the client. +| keyword - - -| level: core - -type: keyword +(core) // =============================================================== | client.user.email | User email address. +| keyword - - -| level: extended - -type: keyword +(extended) // =============================================================== | client.geo.region_name | Region name. +| keyword -example: Quebec - -| level: core - -type: keyword +(core) // =============================================================== | client.domain | Client domain. +| keyword - - -| level: core - -type: keyword +(core) // =============================================================== | client.user.hash | Unique user hash to correlate information for a user in anonymized form. +| keyword - - -| level: extended - -type: keyword +(extended) // =============================================================== | client.geo.city_name | City name. +| keyword -example: Montreal - -| level: core - -type: keyword +(core) // =============================================================== | client.bytes | Bytes sent from the client to the server. +| long -example: 184 - -| level: core - -type: long +(core) // =============================================================== | client.geo.country_iso_code | Country ISO code. +| keyword -example: CA - -| level: core - -type: keyword +(core) // =============================================================== | client.packets | Packets sent from the client to the server. +| long -example: 12 - -| level: core - -type: long +(core) // =============================================================== | client.geo.region_iso_code | Region ISO code. +| keyword -example: CA-QC - -| level: core - -type: keyword +(core) // =============================================================== | client.geo.name | User-defined description of a location. +| keyword -example: boston-dc - -| level: extended - -type: keyword +(extended) // =============================================================== |===== -[[ecs-cloud]] -=== Cloud fields +[[ecs-client-details]] +==== Field details -Fields related to the cloud or infrastructure the events are coming from. +===== client.address -[options="header"] -|===== -| Field | Description | Level / Type +Level: extended -// =============================================================== +Datatype: keyword -| cloud.provider -| Name of the cloud provider. +Some event client addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. +Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. -example: aws -| level: extended -type: keyword +===== client.geo.location -// =============================================================== +Level: core -| cloud.availability_zone -| Availability zone in which this host is running. +Datatype: geo_point -example: us-east-1c +Longitude and latitude. -| level: extended +example: `{ "lon": -73.614830, "lat": 45.505918 }` -type: keyword +===== client.user.id -// =============================================================== +Level: core -| cloud.region -| Region in which this host is running. +Datatype: keyword -example: us-east-1 +One or multiple unique identifiers of the user. -| level: extended -type: keyword -// =============================================================== +===== client.user.group.id -| cloud.instance.id -| Instance ID of the host machine. +Level: extended -example: i-1234567890abcdef0 +Datatype: keyword -| level: extended +Unique identifier for the group on the system/platform. -type: keyword -// =============================================================== -| cloud.instance.name -| Instance name of the host machine. +===== client.ip +Level: core +Datatype: ip -| level: extended +IP address of the client. +Can be one or multiple IPv4 or IPv6 addresses. -type: keyword -// =============================================================== -| cloud.machine.type -| Machine type of the host machine. +===== client.user.group.name -example: t2.medium +Level: extended -| level: extended +Datatype: keyword -type: keyword +Name of the group. -// =============================================================== -| cloud.account.id -| The cloud account or organization id. -example: 666777888999 +===== client.geo.continent_name -| level: extended +Level: core -type: keyword +Datatype: keyword -// =============================================================== +Name of the continent. -|===== +example: `North America` -[[ecs-container]] -=== Container fields +===== client.user.name -Container fields are used for meta information about the specific container that is the source of information. -These fields help correlate data based containers from any runtime. +Level: core -[options="header"] -|===== -| Field | Description | Level / Type +Datatype: keyword -// =============================================================== +Short name or login of the user. -| container.runtime -| Runtime managing this container. +example: `albert` -example: docker +===== client.port -| level: extended +Level: core -type: keyword +Datatype: long -// =============================================================== +Port of the client. -| container.id -| Unique container id. +===== client.geo.country_name -| level: core +Level: core -type: keyword +Datatype: keyword -// =============================================================== +Country name. -| container.image.name -| Name of the image the container was built on. +example: `Canada` +===== client.user.full_name +Level: extended -| level: extended +Datatype: keyword -type: keyword +User's full name, if available. -// =============================================================== +example: `Albert Einstein` -| container.image.tag -| Container image tag. +===== client.mac +Level: core +Datatype: keyword -| level: extended +MAC address of the client. -type: keyword -// =============================================================== -| container.name -| Container name. +===== client.user.email +Level: extended +Datatype: keyword -| level: extended +User email address. -type: keyword -// =============================================================== -| container.labels -| Image labels. +===== client.geo.region_name +Level: core +Datatype: keyword -| level: extended +Region name. -type: object +example: `Quebec` -// =============================================================== +===== client.domain -|===== +Level: core -[[ecs-destination]] -=== Destination fields +Datatype: keyword -Destination fields describe details about the destination of a packet/event. -Destination fields are usually populated in conjunction with source fields. +Client domain. -[options="header"] -|===== -| Field | Description | Level / Type -// =============================================================== -| destination.address -| Destination network address. +===== client.user.hash +Level: extended +Datatype: keyword -| level: extended +Unique user hash to correlate information for a user in anonymized form. +Useful if `user.id` or `user.name` contain confidential information and cannot be used. -type: keyword -// =============================================================== -| destination.geo.location -| Longitude and latitude. +===== client.geo.city_name -example: { "lon": -73.614830, "lat": 45.505918 } +Level: core -| level: core +Datatype: keyword -type: geo_point +City name. -// =============================================================== +example: `Montreal` -| destination.user.id -| One or multiple unique identifiers of the user. +===== client.bytes +Level: core +Datatype: long -| level: core +Bytes sent from the client to the server. -type: keyword +example: `184` -// =============================================================== +===== client.geo.country_iso_code -| destination.user.group.id -| Unique identifier for the group on the system/platform. +Level: core +Datatype: keyword +Country ISO code. -| level: extended +example: `CA` -type: keyword +===== client.packets -// =============================================================== +Level: core -| destination.ip -| IP address of the destination. +Datatype: long +Packets sent from the client to the server. +example: `12` -| level: core +===== client.geo.region_iso_code -type: ip +Level: core -// =============================================================== +Datatype: keyword -| destination.user.group.name -| Name of the group. +Region ISO code. +example: `CA-QC` +===== client.geo.name -| level: extended +Level: extended -type: keyword +Datatype: keyword -// =============================================================== +User-defined description of a location, at the level of granularity they care about. +Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. +Not typically used in automated geolocation. -| destination.geo.continent_name -| Name of the continent. +example: `boston-dc` + +[[ecs-cloud]] +=== Cloud fields -example: North America +Fields related to the cloud or infrastructure the events are coming from. -| level: core +==== Fields summary -type: keyword +[options="header"] +|===== +| Field | Description | Type (Level) // =============================================================== -| destination.user.name -| Short name or login of the user. +| cloud.provider +| Name of the cloud provider. +| keyword + +(extended) -example: albert +// =============================================================== -| level: core +| cloud.availability_zone +| Availability zone in which this host is running. +| keyword -type: keyword +(extended) // =============================================================== -| destination.port -| Port of the destination. +| cloud.region +| Region in which this host is running. +| keyword +(extended) +// =============================================================== -| level: core +| cloud.instance.id +| Instance ID of the host machine. +| keyword -type: long +(extended) // =============================================================== -| destination.geo.country_name -| Country name. +| cloud.instance.name +| Instance name of the host machine. +| keyword -example: Canada +(extended) -| level: core +// =============================================================== + +| cloud.machine.type +| Machine type of the host machine. +| keyword -type: keyword +(extended) // =============================================================== -| destination.user.full_name -| User's full name, if available. +| cloud.account.id +| The cloud account or organization id. +| keyword -example: Albert Einstein +(extended) -| level: extended +// =============================================================== -type: keyword +|===== -// =============================================================== +[[ecs-cloud-details]] +==== Field details -| destination.mac -| MAC address of the destination. +===== cloud.provider +Level: extended +Datatype: keyword -| level: core +Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. -type: keyword +example: `aws` -// =============================================================== +===== cloud.availability_zone -| destination.user.email -| User email address. +Level: extended +Datatype: keyword +Availability zone in which this host is running. -| level: extended +example: `us-east-1c` -type: keyword +===== cloud.region -// =============================================================== +Level: extended -| destination.geo.region_name -| Region name. +Datatype: keyword -example: Quebec +Region in which this host is running. -| level: core +example: `us-east-1` -type: keyword +===== cloud.instance.id -// =============================================================== +Level: extended -| destination.domain -| Destination domain. +Datatype: keyword +Instance ID of the host machine. +example: `i-1234567890abcdef0` -| level: core +===== cloud.instance.name -type: keyword +Level: extended -// =============================================================== +Datatype: keyword -| destination.user.hash -| Unique user hash to correlate information for a user in anonymized form. +Instance name of the host machine. -| level: extended +===== cloud.machine.type -type: keyword +Level: extended -// =============================================================== +Datatype: keyword -| destination.geo.city_name -| City name. +Machine type of the host machine. -example: Montreal +example: `t2.medium` -| level: core +===== cloud.account.id -type: keyword +Level: extended -// =============================================================== +Datatype: keyword -| destination.bytes -| Bytes sent from the destination to the source. +The cloud account or organization id used to identify different entities in a multi-tenant environment. +Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. -example: 184 +example: `666777888999` -| level: core +[[ecs-container]] +=== Container fields -type: long +Container fields are used for meta information about the specific container that is the source of information. +These fields help correlate data based containers from any runtime. -// =============================================================== +==== Fields summary -| destination.geo.country_iso_code -| Country ISO code. +[options="header"] +|===== +| Field | Description | Type (Level) -example: CA +// =============================================================== -| level: core +| container.runtime +| Runtime managing this container. +| keyword -type: keyword +(extended) // =============================================================== -| destination.packets -| Packets sent from the destination to the source. - -example: 12 - -| level: core +| container.id +| Unique container id. +| keyword -type: long +(core) // =============================================================== -| destination.geo.region_iso_code -| Region ISO code. +| container.image.name +| Name of the image the container was built on. +| keyword + +(extended) -example: CA-QC +// =============================================================== -| level: core +| container.image.tag +| Container image tag. +| keyword -type: keyword +(extended) // =============================================================== -| destination.geo.name -| User-defined description of a location. +| container.name +| Container name. +| keyword + +(extended) -example: boston-dc +// =============================================================== -| level: extended +| container.labels +| Image labels. +| object -type: keyword +(extended) // =============================================================== |===== -[[ecs-ecs]] -=== ECS fields - -Meta-information specific to ECS. +[[ecs-container-details]] +==== Field details -[options="header"] -|===== -| Field | Description | Level / Type +===== container.runtime -// =============================================================== +Level: extended -| ecs.version -| ECS version this event conforms to. +Datatype: keyword -example: 1.0.0 +Runtime managing this container. -| level: core +example: `docker` -type: keyword +===== container.id -// =============================================================== +Level: core -|===== +Datatype: keyword -[[ecs-error]] -=== Error fields +Unique container id. -These fields can represent errors of any kind. -Use them for errors that happen while fetching events or in cases where the event itself contains an error. -[options="header"] -|===== -| Field | Description | Level / Type -// =============================================================== +===== container.image.name -| error.id -| Unique identifier for the error. +Level: extended +Datatype: keyword +Name of the image the container was built on. -| level: core -type: keyword -// =============================================================== +===== container.image.tag -| error.message -| Error message. +Level: extended +Datatype: keyword +Container image tag. -| level: core -type: text -// =============================================================== +===== container.name -| error.code -| Error code describing the error. +Level: extended +Datatype: keyword +Container name. -| level: core -type: keyword -// =============================================================== +===== container.labels -|===== +Level: extended -[[ecs-event]] -=== Event fields +Datatype: object -The event fields are used for context information about the log or metric event itself. -A log is defined as an event containing details of something that happened. Log events must include the time at which the thing happened. Examples of log events include a process starting on a host, a network packet being sent from a source to a destination, or a network connection between a client and a server being initiated or closed. A metric is defined as an event containing one or more numerical or categorical measurements and the time at which the measurement was taken. Examples of metric events include memory pressure measured on a host, or vulnerabilities measured on a scanned host. +Image labels. -[options="header"] -|===== -| Field | Description | Level / Type -// =============================================================== -| event.id -| Unique ID to describe the event. +[[ecs-destination]] +=== Destination fields -example: 8a4f500d +Destination fields describe details about the destination of a packet/event. +Destination fields are usually populated in conjunction with source fields. -| level: core +==== Fields summary -type: keyword +[options="header"] +|===== +| Field | Description | Type (Level) // =============================================================== -| event.kind -| The kind of the event. +| destination.address +| Destination network address. +| keyword -example: state +(extended) -| level: extended +// =============================================================== + +| destination.geo.location +| Longitude and latitude. +| geo_point -type: keyword +(core) // =============================================================== -| event.category -| Event category. +| destination.user.id +| One or multiple unique identifiers of the user. +| keyword -example: user-management +(core) -| level: core +// =============================================================== + +| destination.user.group.id +| Unique identifier for the group on the system/platform. +| keyword -type: keyword +(extended) // =============================================================== -| event.action -| The action captured by the event. +| destination.ip +| IP address of the destination. +| ip -example: user-password-change +(core) -| level: core +// =============================================================== + +| destination.user.group.name +| Name of the group. +| keyword -type: keyword +(extended) // =============================================================== -| event.outcome -| The outcome of the event. +| destination.geo.continent_name +| Name of the continent. +| keyword -example: success +(core) -| level: extended +// =============================================================== + +| destination.user.name +| Short name or login of the user. +| keyword -type: keyword +(core) // =============================================================== -| event.type -| Reserved for future usage. +| destination.port +| Port of the destination. +| long +(core) +// =============================================================== -| level: core +| destination.geo.country_name +| Country name. +| keyword -type: keyword +(core) // =============================================================== -| event.module -| Name of the module this data is coming from. +| destination.user.full_name +| User's full name, if available. +| keyword + +(extended) -example: mysql +// =============================================================== -| level: core +| destination.mac +| MAC address of the destination. +| keyword -type: keyword +(core) // =============================================================== -| event.dataset -| Name of the dataset. +| destination.user.email +| User email address. +| keyword + +(extended) -example: stats +// =============================================================== -| level: core +| destination.geo.region_name +| Region name. +| keyword -type: keyword +(core) // =============================================================== -| event.severity -| Original severity of the event. +| destination.domain +| Destination domain. +| keyword + +(core) -example: 7 +// =============================================================== -| level: core +| destination.user.hash +| Unique user hash to correlate information for a user in anonymized form. +| keyword -type: long +(extended) // =============================================================== -| event.original -| Raw text message of entire event. +| destination.geo.city_name +| City name. +| keyword + +(core) -example: Sep 19 08:26:10 host CEF:0|Security| threatmanager|1.0|100| worm successfully stopped|10|src=10.0.0.1 dst=2.1.2.2spt=1232 +// =============================================================== -| level: core +| destination.bytes +| Bytes sent from the destination to the source. +| long -type: keyword +(core) // =============================================================== -| event.hash -| Hash (perhaps logstash fingerprint) of raw field to be able to demonstrate log integrity. +| destination.geo.country_iso_code +| Country ISO code. +| keyword + +(core) -example: 123456789012345678901234567890ABCD +// =============================================================== -| level: extended +| destination.packets +| Packets sent from the destination to the source. +| long -type: keyword +(core) // =============================================================== -| event.duration -| Duration of the event in nanoseconds. +| destination.geo.region_iso_code +| Region ISO code. +| keyword +(core) +// =============================================================== -| level: core +| destination.geo.name +| User-defined description of a location. +| keyword -type: long +(extended) // =============================================================== -| event.timezone -| Event time zone. - +|===== +[[ecs-destination-details]] +==== Field details -| level: extended +===== destination.address -type: keyword +Level: extended -// =============================================================== +Datatype: keyword -| event.created -| Time when the event was first read by an agent or by your pipeline. +Some event destination addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. +Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. -| level: core +===== destination.geo.location -type: date +Level: core -// =============================================================== +Datatype: geo_point -| event.start -| event.start contains the date when the event started or when the activity was first observed. +Longitude and latitude. +example: `{ "lon": -73.614830, "lat": 45.505918 }` +===== destination.user.id -| level: extended +Level: core -type: date +Datatype: keyword -// =============================================================== +One or multiple unique identifiers of the user. -| event.end -| event.end contains the date when the event ended or when the activity was last observed. +===== destination.user.group.id -| level: extended +Level: extended -type: date +Datatype: keyword -// =============================================================== +Unique identifier for the group on the system/platform. -| event.risk_score -| Risk score or priority of the event (e.g. security solutions). Use your system's original value here. +===== destination.ip -| level: core +Level: core -type: float +Datatype: ip -// =============================================================== +IP address of the destination. +Can be one or multiple IPv4 or IPv6 addresses. -| event.risk_score_norm -| Normalized risk score or priority of the event (0-100). +===== destination.user.group.name -| level: extended +Level: extended -type: float +Datatype: keyword -// =============================================================== +Name of the group. -|===== -[[ecs-file]] -=== File fields -A file is defined as a set of information that has been created on, or has existed on a filesystem. -File objects can be associated with host events, network events, and/or file events (e.g., those produced by File Integrity Monitoring [FIM] products or services). File fields provide details about the affected file associated with the event or metric. +===== destination.geo.continent_name -[options="header"] -|===== -| Field | Description | Level / Type +Level: core -// =============================================================== +Datatype: keyword -| file.path -| Path to the file. +Name of the continent. +example: `North America` +===== destination.user.name -| level: extended +Level: core -type: keyword +Datatype: keyword -// =============================================================== +Short name or login of the user. -| file.target_path -| Target path for symlinks. +example: `albert` +===== destination.port +Level: core -| level: extended +Datatype: long -type: keyword +Port of the destination. -// =============================================================== -| file.extension -| File extension. -example: png +===== destination.geo.country_name -| level: extended +Level: core -type: keyword +Datatype: keyword -// =============================================================== +Country name. -| file.type -| File type (file, dir, or symlink). +example: `Canada` +===== destination.user.full_name +Level: extended -| level: extended +Datatype: keyword -type: keyword +User's full name, if available. -// =============================================================== +example: `Albert Einstein` -| file.device -| Device that is the source of the file. +===== destination.mac +Level: core +Datatype: keyword -| level: extended +MAC address of the destination. -type: keyword -// =============================================================== -| file.inode -| Inode representing the file in the filesystem. +===== destination.user.email +Level: extended +Datatype: keyword -| level: extended +User email address. -type: keyword -// =============================================================== -| file.uid -| The user ID (UID) or security identifier (SID) of the file owner. +===== destination.geo.region_name +Level: core +Datatype: keyword -| level: extended +Region name. -type: keyword +example: `Quebec` -// =============================================================== +===== destination.domain -| file.owner -| File owner's username. +Level: core +Datatype: keyword +Destination domain. -| level: extended -type: keyword -// =============================================================== +===== destination.user.hash -| file.gid -| Primary group ID (GID) of the file. +Level: extended +Datatype: keyword +Unique user hash to correlate information for a user in anonymized form. +Useful if `user.id` or `user.name` contain confidential information and cannot be used. -| level: extended -type: keyword -// =============================================================== +===== destination.geo.city_name -| file.group -| Primary group name of the file. +Level: core +Datatype: keyword +City name. -| level: extended +example: `Montreal` -type: keyword +===== destination.bytes -// =============================================================== +Level: core -| file.mode -| Mode of the file in octal representation. +Datatype: long -example: 416 +Bytes sent from the destination to the source. -| level: extended +example: `184` -type: keyword +===== destination.geo.country_iso_code -// =============================================================== +Level: core -| file.size -| File size in bytes (field is only added when `type` is `file`). +Datatype: keyword +Country ISO code. +example: `CA` -| level: extended +===== destination.packets -type: long +Level: core -// =============================================================== +Datatype: long -| file.mtime -| Last time file content was modified. +Packets sent from the destination to the source. +example: `12` +===== destination.geo.region_iso_code -| level: extended +Level: core -type: date +Datatype: keyword -// =============================================================== +Region ISO code. -| file.ctime -| Last time file metadata changed. +example: `CA-QC` +===== destination.geo.name +Level: extended -| level: extended +Datatype: keyword -type: date +User-defined description of a location, at the level of granularity they care about. +Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. +Not typically used in automated geolocation. -// =============================================================== +example: `boston-dc` -|===== +[[ecs-ecs]] +=== ECS fields -[[ecs-geo]] -=== Geo fields +Meta-information specific to ECS. -Geo fields can carry data about a specific location related to an event. -This geolocation information can be derived from techniques such as Geo IP, or be user-supplied. +==== Fields summary [options="header"] |===== -| Field | Description | Level / Type +| Field | Description | Type (Level) // =============================================================== -| geo.location -| Longitude and latitude. - -example: { "lon": -73.614830, "lat": 45.505918 } - -| level: core +| ecs.version +| ECS version this event conforms to. +| keyword -type: geo_point +(core) // =============================================================== -| geo.continent_name -| Name of the continent. +|===== -example: North America +[[ecs-ecs-details]] +==== Field details -| level: core +===== ecs.version -type: keyword +Level: core -// =============================================================== +Datatype: keyword -| geo.country_name -| Country name. +ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. +When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. -example: Canada +example: `1.0.0` -| level: core +[[ecs-error]] +=== Error fields -type: keyword +These fields can represent errors of any kind. +Use them for errors that happen while fetching events or in cases where the event itself contains an error. -// =============================================================== +==== Fields summary -| geo.region_name -| Region name. +[options="header"] +|===== +| Field | Description | Type (Level) -example: Quebec +// =============================================================== -| level: core +| error.id +| Unique identifier for the error. +| keyword -type: keyword +(core) // =============================================================== -| geo.city_name -| City name. +| error.message +| Error message. +| text -example: Montreal +(core) -| level: core +// =============================================================== -type: keyword +| error.code +| Error code describing the error. +| keyword -// =============================================================== +(core) -| geo.country_iso_code -| Country ISO code. +// =============================================================== -example: CA +|===== -| level: core +[[ecs-error-details]] +==== Field details -type: keyword +===== error.id -// =============================================================== +Level: core -| geo.region_iso_code -| Region ISO code. +Datatype: keyword -example: CA-QC +Unique identifier for the error. -| level: core -type: keyword -// =============================================================== +===== error.message -| geo.name -| User-defined description of a location. +Level: core -example: boston-dc +Datatype: text -| level: extended +Error message. -type: keyword -// =============================================================== -|===== +===== error.code -[[ecs-group]] -=== Group fields +Level: core -The group fields are meant to represent groups that are relevant to the event. +Datatype: keyword -[options="header"] -|===== -| Field | Description | Level / Type +Error code describing the error. -// =============================================================== -| group.id -| Unique identifier for the group on the system/platform. +[[ecs-event]] +=== Event fields +The event fields are used for context information about the log or metric event itself. +A log is defined as an event containing details of something that happened. Log events must include the time at which the thing happened. Examples of log events include a process starting on a host, a network packet being sent from a source to a destination, or a network connection between a client and a server being initiated or closed. A metric is defined as an event containing one or more numerical or categorical measurements and the time at which the measurement was taken. Examples of metric events include memory pressure measured on a host, or vulnerabilities measured on a scanned host. -| level: extended +==== Fields summary -type: keyword +[options="header"] +|===== +| Field | Description | Type (Level) // =============================================================== -| group.name -| Name of the group. +| event.id +| Unique ID to describe the event. +| keyword +(core) +// =============================================================== -| level: extended +| event.kind +| The kind of the event. +| keyword -type: keyword +(extended) // =============================================================== -|===== +| event.category +| Event category. +| keyword -[[ecs-host]] -=== Host fields +(core) -A host is defined as a general computing instance. -ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes. +// =============================================================== -[options="header"] -|===== -| Field | Description | Level / Type +| event.action +| The action captured by the event. +| keyword + +(core) // =============================================================== -| host.hostname -| Hostname of the host. +| event.outcome +| The outcome of the event. +| keyword +(extended) +// =============================================================== -| level: core +| event.type +| Reserved for future usage. +| keyword -type: keyword +(core) // =============================================================== -| host.geo.location -| Longitude and latitude. +| event.module +| Name of the module this data is coming from. +| keyword + +(core) -example: { "lon": -73.614830, "lat": 45.505918 } +// =============================================================== -| level: core +| event.dataset +| Name of the dataset. +| keyword -type: geo_point +(core) // =============================================================== -| host.user.id -| One or multiple unique identifiers of the user. +| event.severity +| Original severity of the event. +| long +(core) +// =============================================================== -| level: core +| event.original +| Raw text message of entire event. +| keyword -type: keyword +(core) // =============================================================== -| host.user.group.id -| Unique identifier for the group on the system/platform. +| event.hash +| Hash (perhaps logstash fingerprint) of raw field to be able to demonstrate log integrity. +| keyword +(extended) +// =============================================================== -| level: extended +| event.duration +| Duration of the event in nanoseconds. +| long -type: keyword +(core) // =============================================================== -| host.os.platform -| Operating system platform (such centos, ubuntu, windows). +| event.timezone +| Event time zone. +| keyword + +(extended) -example: darwin +// =============================================================== -| level: extended +| event.created +| Time when the event was first read by an agent or by your pipeline. +| date -type: keyword +(core) // =============================================================== -| host.name -| Name of the host. +| event.start +| event.start contains the date when the event started or when the activity was first observed. +| date +(extended) +// =============================================================== -| level: core +| event.end +| event.end contains the date when the event ended or when the activity was last observed. +| date -type: keyword +(extended) // =============================================================== -| host.user.group.name -| Name of the group. +| event.risk_score +| Risk score or priority of the event (e.g. security solutions). Use your system's original value here. +| float +(core) +// =============================================================== -| level: extended +| event.risk_score_norm +| Normalized risk score or priority of the event (0-100). +| float -type: keyword +(extended) // =============================================================== -| host.geo.continent_name -| Name of the continent. - -example: North America +|===== -| level: core +[[ecs-event-details]] +==== Field details -type: keyword +===== event.id -// =============================================================== +Level: core -| host.os.name -| Operating system name, without the version. +Datatype: keyword -example: Mac OS X +Unique ID to describe the event. -| level: extended +example: `8a4f500d` -type: keyword +===== event.kind -// =============================================================== +Level: extended -| host.user.name -| Short name or login of the user. +Datatype: keyword -example: albert +The kind of the event. +This gives information about what type of information the event contains, without being specific to the contents of the event. Examples are `event`, `state`, `alarm`. Warning: In future versions of ECS, we plan to provide a list of acceptable values for this field, please use with caution. -| level: core +example: `state` -type: keyword +===== event.category -// =============================================================== +Level: core -| host.id -| Unique host id. +Datatype: keyword +Event category. +This contains high-level information about the contents of the event. It is more generic than `event.action`, in the sense that typically a category contains multiple actions. Warning: In future versions of ECS, we plan to provide a list of acceptable values for this field, please use with caution. +example: `user-management` -| level: core +===== event.action -type: keyword +Level: core -// =============================================================== +Datatype: keyword -| host.geo.country_name -| Country name. +The action captured by the event. +This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. -example: Canada +example: `user-password-change` -| level: core +===== event.outcome -type: keyword +Level: extended -// =============================================================== +Datatype: keyword -| host.os.full -| Operating system name, including the version or code name. +The outcome of the event. +If the event describes an action, this fields contains the outcome of that action. Examples outcomes are `success` and `failure`. Warning: In future versions of ECS, we plan to provide a list of acceptable values for this field, please use with caution. -example: Mac OS Mojave +example: `success` -| level: extended +===== event.type -type: keyword +Level: core -// =============================================================== +Datatype: keyword -| host.user.full_name -| User's full name, if available. +Reserved for future usage. +Please avoid using this field for user data. -example: Albert Einstein -| level: extended -type: keyword +===== event.module -// =============================================================== +Level: core -| host.ip -| Host ip address. +Datatype: keyword +Name of the module this data is coming from. +This information is coming from the modules used in Beats or Logstash. +example: `mysql` -| level: core +===== event.dataset -type: ip +Level: core -// =============================================================== +Datatype: keyword -| host.user.email -| User email address. +Name of the dataset. +The concept of a `dataset` (fileset / metricset) is used in Beats as a subset of modules. It contains the information which is currently stored in metricset.name and metricset.module or fileset.name. +example: `stats` +===== event.severity -| level: extended +Level: core -type: keyword +Datatype: long -// =============================================================== +Severity describes the original severity of the event. What the different severity values mean can very different between use cases. It's up to the implementer to make sure severities are consistent across events. -| host.os.family -| OS family (such as redhat, debian, freebsd, windows). +example: `7` -example: debian +===== event.original -| level: extended +Level: core -type: keyword +Datatype: keyword -// =============================================================== +Raw text message of entire event. Used to demonstrate log integrity. +This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. -| host.geo.region_name -| Region name. +example: `Sep 19 08:26:10 host CEF:0|Security| threatmanager|1.0|100| worm successfully stopped|10|src=10.0.0.1 dst=2.1.2.2spt=1232` -example: Quebec +===== event.hash -| level: core +Level: extended -type: keyword +Datatype: keyword -// =============================================================== +Hash (perhaps logstash fingerprint) of raw field to be able to demonstrate log integrity. -| host.mac -| Host mac address. +example: `123456789012345678901234567890ABCD` +===== event.duration +Level: core -| level: core +Datatype: long -type: keyword +Duration of the event in nanoseconds. +If event.start and event.end are known this value should be the difference between the end and start time. -// =============================================================== -| host.user.hash -| Unique user hash to correlate information for a user in anonymized form. +===== event.timezone +Level: extended -| level: extended +Datatype: keyword -type: keyword +This field should be populated when the event's timestamp does not include timezone information already (e.g. default Syslog timestamps). It's optional otherwise. +Acceptable timezone formats are: a canonical ID (e.g. "Europe/Amsterdam"), abbreviated (e.g. "EST") or an HH:mm differential (e.g. "-05:00"). -// =============================================================== -| host.geo.city_name -| City name. -example: Montreal +===== event.created -| level: core +Level: core -type: keyword +Datatype: date -// =============================================================== +event.created contains the date/time when the event was first read by an agent, or by your pipeline. +This field is distinct from @timestamp in that @timestamp typically contain the time extracted from the original event. +In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. +In case the two timestamps are identical, @timestamp should be used. -| host.os.version -| Operating system version as a raw string. -example: 10.14.1 -| level: extended +===== event.start -type: keyword +Level: extended -// =============================================================== +Datatype: date -| host.type -| Type of host. +event.start contains the date when the event started or when the activity was first observed. -| level: core +===== event.end -type: keyword +Level: extended -// =============================================================== +Datatype: date -| host.geo.country_iso_code -| Country ISO code. +event.end contains the date when the event ended or when the activity was last observed. -example: CA -| level: core -type: keyword +===== event.risk_score -// =============================================================== +Level: core -| host.os.kernel -| Operating system kernel version as a raw string. +Datatype: float -example: 4.4.0-112-generic +Risk score or priority of the event (e.g. security solutions). Use your system's original value here. -| level: extended -type: keyword -// =============================================================== +===== event.risk_score_norm -| host.architecture -| Operating system architecture. +Level: extended -example: x86_64 +Datatype: float -| level: core +Normalized risk score or priority of the event, on a scale of 0 to 100. +This is mainly useful if you use more than one system that assigns risk scores, and you want to see a normalized value across all systems. -type: keyword -// =============================================================== -| host.geo.region_iso_code -| Region ISO code. +[[ecs-file]] +=== File fields -example: CA-QC +A file is defined as a set of information that has been created on, or has existed on a filesystem. +File objects can be associated with host events, network events, and/or file events (e.g., those produced by File Integrity Monitoring [FIM] products or services). File fields provide details about the affected file associated with the event or metric. -| level: core +==== Fields summary -type: keyword +[options="header"] +|===== +| Field | Description | Type (Level) // =============================================================== -| host.geo.name -| User-defined description of a location. +| file.path +| Path to the file. +| keyword -example: boston-dc +(extended) -| level: extended +// =============================================================== + +| file.target_path +| Target path for symlinks. +| keyword -type: keyword +(extended) // =============================================================== -|===== +| file.extension +| File extension. +| keyword -[[ecs-http]] -=== HTTP fields +(extended) -Fields related to HTTP activity. Use the `url` field set to store the url of the request. +// =============================================================== -[options="header"] -|===== -| Field | Description | Level / Type +| file.type +| File type (file, dir, or symlink). +| keyword + +(extended) // =============================================================== -| http.request.method -| HTTP request method. +| file.device +| Device that is the source of the file. +| keyword -example: get, post, put +(extended) -| level: extended +// =============================================================== + +| file.inode +| Inode representing the file in the filesystem. +| keyword -type: keyword +(extended) // =============================================================== -| http.request.body.content -| The full HTTP request body. +| file.uid +| The user ID (UID) or security identifier (SID) of the file owner. +| keyword -example: Hello world +(extended) -| level: extended +// =============================================================== + +| file.owner +| File owner's username. +| keyword -type: keyword +(extended) // =============================================================== -| http.request.referrer -| Referrer for this HTTP request. +| file.gid +| Primary group ID (GID) of the file. +| keyword -example: https://blog.example.com/ +(extended) -| level: extended +// =============================================================== + +| file.group +| Primary group name of the file. +| keyword -type: keyword +(extended) // =============================================================== -| http.response.status_code -| HTTP response status code. +| file.mode +| Mode of the file in octal representation. +| keyword -example: 404 +(extended) -| level: extended +// =============================================================== + +| file.size +| File size in bytes (field is only added when `type` is `file`). +| long -type: long +(extended) // =============================================================== -| http.response.body.content -| The full HTTP response body. +| file.mtime +| Last time file content was modified. +| date -example: Hello world +(extended) -| level: extended +// =============================================================== + +| file.ctime +| Last time file metadata changed. +| date -type: keyword +(extended) // =============================================================== -| http.version -| HTTP version. +|===== -example: 1.1 +[[ecs-file-details]] +==== Field details -| level: extended +===== file.path -type: keyword +Level: extended -// =============================================================== +Datatype: keyword -| http.request.bytes -| Total size in bytes of the request (body and headers). +Path to the file. -example: 1437 -| level: extended -type: long +===== file.target_path -// =============================================================== +Level: extended -| http.request.body.bytes -| Size in bytes of the request body. +Datatype: keyword -example: 887 +Target path for symlinks. -| level: extended -type: long -// =============================================================== +===== file.extension -| http.response.bytes -| Total size in bytes of the response (body and headers). +Level: extended -example: 1437 +Datatype: keyword -| level: extended +File extension. +This should allow easy filtering by file extensions. -type: long +example: `png` + +===== file.type + +Level: extended + +Datatype: keyword + +File type (file, dir, or symlink). + + + +===== file.device + +Level: extended + +Datatype: keyword + +Device that is the source of the file. + + + +===== file.inode + +Level: extended + +Datatype: keyword + +Inode representing the file in the filesystem. + + + +===== file.uid + +Level: extended + +Datatype: keyword + +The user ID (UID) or security identifier (SID) of the file owner. + + + +===== file.owner + +Level: extended + +Datatype: keyword + +File owner's username. + + + +===== file.gid + +Level: extended + +Datatype: keyword + +Primary group ID (GID) of the file. + + + +===== file.group + +Level: extended + +Datatype: keyword + +Primary group name of the file. + + + +===== file.mode + +Level: extended + +Datatype: keyword + +Mode of the file in octal representation. + +example: `416` + +===== file.size + +Level: extended + +Datatype: long + +File size in bytes (field is only added when `type` is `file`). + + + +===== file.mtime + +Level: extended + +Datatype: date + +Last time file content was modified. + + + +===== file.ctime + +Level: extended + +Datatype: date + +Last time file metadata changed. + + + +[[ecs-geo]] +=== Geo fields + +Geo fields can carry data about a specific location related to an event. +This geolocation information can be derived from techniques such as Geo IP, or be user-supplied. + +==== Fields summary + +[options="header"] +|===== +| Field | Description | Type (Level) + +// =============================================================== + +| geo.location +| Longitude and latitude. +| geo_point + +(core) + +// =============================================================== + +| geo.continent_name +| Name of the continent. +| keyword + +(core) + +// =============================================================== + +| geo.country_name +| Country name. +| keyword + +(core) + +// =============================================================== + +| geo.region_name +| Region name. +| keyword + +(core) + +// =============================================================== + +| geo.city_name +| City name. +| keyword + +(core) + +// =============================================================== + +| geo.country_iso_code +| Country ISO code. +| keyword + +(core) + +// =============================================================== + +| geo.region_iso_code +| Region ISO code. +| keyword + +(core) + +// =============================================================== + +| geo.name +| User-defined description of a location. +| keyword + +(extended) + +// =============================================================== + +|===== + +[[ecs-geo-details]] +==== Field details + +===== geo.location + +Level: core + +Datatype: geo_point + +Longitude and latitude. + +example: `{ "lon": -73.614830, "lat": 45.505918 }` + +===== geo.continent_name + +Level: core + +Datatype: keyword + +Name of the continent. + +example: `North America` + +===== geo.country_name + +Level: core + +Datatype: keyword + +Country name. + +example: `Canada` + +===== geo.region_name + +Level: core + +Datatype: keyword + +Region name. + +example: `Quebec` + +===== geo.city_name + +Level: core + +Datatype: keyword + +City name. + +example: `Montreal` + +===== geo.country_iso_code + +Level: core + +Datatype: keyword + +Country ISO code. + +example: `CA` + +===== geo.region_iso_code + +Level: core + +Datatype: keyword + +Region ISO code. + +example: `CA-QC` + +===== geo.name + +Level: extended + +Datatype: keyword + +User-defined description of a location, at the level of granularity they care about. +Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. +Not typically used in automated geolocation. + +example: `boston-dc` + +[[ecs-group]] +=== Group fields + +The group fields are meant to represent groups that are relevant to the event. + +==== Fields summary + +[options="header"] +|===== +| Field | Description | Type (Level) + +// =============================================================== + +| group.id +| Unique identifier for the group on the system/platform. +| keyword + +(extended) + +// =============================================================== + +| group.name +| Name of the group. +| keyword + +(extended) + +// =============================================================== + +|===== + +[[ecs-group-details]] +==== Field details + +===== group.id + +Level: extended + +Datatype: keyword + +Unique identifier for the group on the system/platform. + + + +===== group.name + +Level: extended + +Datatype: keyword + +Name of the group. + + + +[[ecs-host]] +=== Host fields + +A host is defined as a general computing instance. +ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes. + +==== Fields summary + +[options="header"] +|===== +| Field | Description | Type (Level) + +// =============================================================== + +| host.hostname +| Hostname of the host. +| keyword + +(core) + +// =============================================================== + +| host.geo.location +| Longitude and latitude. +| geo_point + +(core) + +// =============================================================== + +| host.user.id +| One or multiple unique identifiers of the user. +| keyword + +(core) + +// =============================================================== + +| host.user.group.id +| Unique identifier for the group on the system/platform. +| keyword + +(extended) + +// =============================================================== + +| host.os.platform +| Operating system platform (such centos, ubuntu, windows). +| keyword + +(extended) + +// =============================================================== + +| host.name +| Name of the host. +| keyword + +(core) + +// =============================================================== + +| host.user.group.name +| Name of the group. +| keyword + +(extended) + +// =============================================================== + +| host.geo.continent_name +| Name of the continent. +| keyword + +(core) + +// =============================================================== + +| host.os.name +| Operating system name, without the version. +| keyword + +(extended) + +// =============================================================== + +| host.user.name +| Short name or login of the user. +| keyword + +(core) + +// =============================================================== + +| host.id +| Unique host id. +| keyword + +(core) + +// =============================================================== + +| host.geo.country_name +| Country name. +| keyword + +(core) + +// =============================================================== + +| host.os.full +| Operating system name, including the version or code name. +| keyword + +(extended) + +// =============================================================== + +| host.user.full_name +| User's full name, if available. +| keyword + +(extended) + +// =============================================================== + +| host.ip +| Host ip address. +| ip + +(core) + +// =============================================================== + +| host.user.email +| User email address. +| keyword + +(extended) + +// =============================================================== + +| host.os.family +| OS family (such as redhat, debian, freebsd, windows). +| keyword + +(extended) + +// =============================================================== + +| host.geo.region_name +| Region name. +| keyword + +(core) + +// =============================================================== + +| host.mac +| Host mac address. +| keyword + +(core) + +// =============================================================== + +| host.user.hash +| Unique user hash to correlate information for a user in anonymized form. +| keyword + +(extended) + +// =============================================================== + +| host.geo.city_name +| City name. +| keyword + +(core) + +// =============================================================== + +| host.os.version +| Operating system version as a raw string. +| keyword + +(extended) + +// =============================================================== + +| host.type +| Type of host. +| keyword + +(core) + +// =============================================================== + +| host.geo.country_iso_code +| Country ISO code. +| keyword + +(core) + +// =============================================================== + +| host.os.kernel +| Operating system kernel version as a raw string. +| keyword + +(extended) + +// =============================================================== + +| host.architecture +| Operating system architecture. +| keyword + +(core) + +// =============================================================== + +| host.geo.region_iso_code +| Region ISO code. +| keyword + +(core) + +// =============================================================== + +| host.geo.name +| User-defined description of a location. +| keyword + +(extended) + +// =============================================================== + +|===== + +[[ecs-host-details]] +==== Field details + +===== host.hostname + +Level: core + +Datatype: keyword + +Hostname of the host. +It normally contains what the `hostname` command returns on the host machine. + + + +===== host.geo.location + +Level: core + +Datatype: geo_point + +Longitude and latitude. + +example: `{ "lon": -73.614830, "lat": 45.505918 }` + +===== host.user.id + +Level: core + +Datatype: keyword + +One or multiple unique identifiers of the user. + + + +===== host.user.group.id + +Level: extended + +Datatype: keyword + +Unique identifier for the group on the system/platform. + + + +===== host.os.platform + +Level: extended + +Datatype: keyword + +Operating system platform (such centos, ubuntu, windows). + +example: `darwin` + +===== host.name + +Level: core + +Datatype: keyword + +Name of the host. +It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. + + + +===== host.user.group.name + +Level: extended + +Datatype: keyword + +Name of the group. + + + +===== host.geo.continent_name + +Level: core + +Datatype: keyword + +Name of the continent. + +example: `North America` + +===== host.os.name + +Level: extended + +Datatype: keyword + +Operating system name, without the version. + +example: `Mac OS X` + +===== host.user.name + +Level: core + +Datatype: keyword + +Short name or login of the user. + +example: `albert` + +===== host.id + +Level: core + +Datatype: keyword + +Unique host id. +As hostname is not always unique, use values that are meaningful in your environment. +Example: The current usage of `beat.name`. + + + +===== host.geo.country_name + +Level: core + +Datatype: keyword + +Country name. + +example: `Canada` + +===== host.os.full + +Level: extended + +Datatype: keyword + +Operating system name, including the version or code name. + +example: `Mac OS Mojave` + +===== host.user.full_name + +Level: extended + +Datatype: keyword + +User's full name, if available. + +example: `Albert Einstein` + +===== host.ip + +Level: core + +Datatype: ip + +Host ip address. + + + +===== host.user.email + +Level: extended + +Datatype: keyword + +User email address. + + + +===== host.os.family + +Level: extended + +Datatype: keyword + +OS family (such as redhat, debian, freebsd, windows). + +example: `debian` + +===== host.geo.region_name + +Level: core + +Datatype: keyword + +Region name. + +example: `Quebec` + +===== host.mac + +Level: core + +Datatype: keyword + +Host mac address. + + + +===== host.user.hash + +Level: extended + +Datatype: keyword + +Unique user hash to correlate information for a user in anonymized form. +Useful if `user.id` or `user.name` contain confidential information and cannot be used. + + + +===== host.geo.city_name + +Level: core + +Datatype: keyword + +City name. + +example: `Montreal` + +===== host.os.version + +Level: extended + +Datatype: keyword + +Operating system version as a raw string. + +example: `10.14.1` + +===== host.type + +Level: core + +Datatype: keyword + +Type of host. +For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. + + + +===== host.geo.country_iso_code + +Level: core + +Datatype: keyword + +Country ISO code. + +example: `CA` + +===== host.os.kernel + +Level: extended + +Datatype: keyword + +Operating system kernel version as a raw string. + +example: `4.4.0-112-generic` + +===== host.architecture + +Level: core + +Datatype: keyword + +Operating system architecture. + +example: `x86_64` + +===== host.geo.region_iso_code + +Level: core + +Datatype: keyword + +Region ISO code. + +example: `CA-QC` + +===== host.geo.name + +Level: extended + +Datatype: keyword + +User-defined description of a location, at the level of granularity they care about. +Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. +Not typically used in automated geolocation. + +example: `boston-dc` + +[[ecs-http]] +=== HTTP fields + +Fields related to HTTP activity. Use the `url` field set to store the url of the request. + +==== Fields summary + +[options="header"] +|===== +| Field | Description | Type (Level) + +// =============================================================== + +| http.request.method +| HTTP request method. +| keyword + +(extended) + +// =============================================================== + +| http.request.body.content +| The full HTTP request body. +| keyword + +(extended) + +// =============================================================== + +| http.request.referrer +| Referrer for this HTTP request. +| keyword + +(extended) + +// =============================================================== + +| http.response.status_code +| HTTP response status code. +| long + +(extended) + +// =============================================================== + +| http.response.body.content +| The full HTTP response body. +| keyword + +(extended) + +// =============================================================== + +| http.version +| HTTP version. +| keyword + +(extended) + +// =============================================================== + +| http.request.bytes +| Total size in bytes of the request (body and headers). +| long + +(extended) + +// =============================================================== + +| http.request.body.bytes +| Size in bytes of the request body. +| long + +(extended) + +// =============================================================== + +| http.response.bytes +| Total size in bytes of the response (body and headers). +| long + +(extended) + +// =============================================================== + +| http.response.body.bytes +| Size in bytes of the response body. +| long + +(extended) + +// =============================================================== + +|===== + +[[ecs-http-details]] +==== Field details + +===== http.request.method + +Level: extended + +Datatype: keyword + +HTTP request method. +The field value must be normalized to lowercase for querying. See the documentation section "Implementing ECS". + +example: `get, post, put` + +===== http.request.body.content + +Level: extended + +Datatype: keyword + +The full HTTP request body. + +example: `Hello world` + +===== http.request.referrer + +Level: extended + +Datatype: keyword + +Referrer for this HTTP request. + +example: `https://blog.example.com/` + +===== http.response.status_code + +Level: extended + +Datatype: long + +HTTP response status code. + +example: `404` + +===== http.response.body.content + +Level: extended + +Datatype: keyword + +The full HTTP response body. + +example: `Hello world` + +===== http.version + +Level: extended + +Datatype: keyword + +HTTP version. + +example: `1.1` + +===== http.request.bytes + +Level: extended + +Datatype: long + +Total size in bytes of the request (body and headers). + +example: `1437` + +===== http.request.body.bytes + +Level: extended + +Datatype: long + +Size in bytes of the request body. + +example: `887` + +===== http.response.bytes + +Level: extended + +Datatype: long + +Total size in bytes of the response (body and headers). + +example: `1437` + +===== http.response.body.bytes + +Level: extended + +Datatype: long + +Size in bytes of the response body. + +example: `887` + +[[ecs-log]] +=== Log fields + +Fields which are specific to log events. + +==== Fields summary + +[options="header"] +|===== +| Field | Description | Type (Level) + +// =============================================================== + +| log.level +| Log level of the log event. +| keyword + +(core) + +// =============================================================== + +| log.original +| Original log message with light interpretation only (encoding, newlines). +| keyword + +(core) + +// =============================================================== + +|===== + +[[ecs-log-details]] +==== Field details + +===== log.level + +Level: core + +Datatype: keyword + +Original log level of the log event. +Some examples are `warn`, `error`, `i`. + +example: `err` + +===== log.original + +Level: core + +Datatype: keyword + +This is the original log message and contains the full log message before splitting it up in multiple parts. +In contrast to the `message` field which can contain an extracted part of the log message, this field contains the original, full log message. It can have already some modifications applied like encoding or new lines removed to clean up the log message. +This field is not indexed and doc_values are disabled so it can't be queried but the value can be retrieved from `_source`. + +example: `Sep 19 08:26:10 localhost My log` + +[[ecs-network]] +=== Network fields + +The network is defined as the communication path over which a host or network event happens. +The network.* fields should be populated with details about the network activity associated with an event. + +==== Fields summary + +[options="header"] +|===== +| Field | Description | Type (Level) + +// =============================================================== + +| network.name +| Name given by operators to sections of their network. +| keyword + +(extended) + +// =============================================================== + +| network.type +| In the OSI Model this would be the Network Layer. ipv4, ipv6, ipsec, pim, etc +| keyword + +(core) + +// =============================================================== + +| network.iana_number +| IANA Protocol Number. +| keyword + +(extended) + +// =============================================================== + +| network.transport +| Protocol Name corresponding to the field `iana_number`. +| keyword + +(core) + +// =============================================================== + +| network.application +| Application level protocol name. +| keyword + +(extended) + +// =============================================================== + +| network.protocol +| L7 Network protocol name. +| keyword + +(core) + +// =============================================================== + +| network.direction +| Direction of the network traffic. +| keyword + +(core) + +// =============================================================== + +| network.forwarded_ip +| Host IP address when the source IP address is the proxy. +| ip + +(core) + +// =============================================================== + +| network.community_id +| A hash of source and destination IPs and ports. +| keyword + +(extended) + +// =============================================================== + +| network.bytes +| Total bytes transferred in both directions. +| long + +(core) + +// =============================================================== + +| network.packets +| Total packets transferred in both directions. +| long + +(core) + +// =============================================================== + +|===== + +[[ecs-network-details]] +==== Field details + +===== network.name + +Level: extended + +Datatype: keyword + +Name given by operators to sections of their network. + +example: `Guest Wifi` + +===== network.type + +Level: core + +Datatype: keyword + +In the OSI Model this would be the Network Layer. ipv4, ipv6, ipsec, pim, etc +The field value must be normalized to lowercase for querying. See the documentation section "Implementing ECS". + +example: `ipv4` + +===== network.iana_number + +Level: extended + +Datatype: keyword + +IANA Protocol Number (https://www.iana.org/assignments/protocol-numbers/protocol-numbers.xhtml). Standardized list of protocols. This aligns well with NetFlow and sFlow related logs which use the IANA Protocol Number. + +example: `6` + +===== network.transport + +Level: core + +Datatype: keyword + +Same as network.iana_number, but instead using the Keyword name of the transport layer (udp, tcp, ipv6-icmp, etc.) +The field value must be normalized to lowercase for querying. See the documentation section "Implementing ECS". + +example: `tcp` + +===== network.application + +Level: extended + +Datatype: keyword + +A name given to an application level protocol. This can be arbitrarily assigned for things like microservices, but also apply to things like skype, icq, facebook, twitter. This would be used in situations where the vendor or service can be decoded such as from the source/dest IP owners, ports, or wire format. +The field value must be normalized to lowercase for querying. See the documentation section "Implementing ECS". + +example: `aim` + +===== network.protocol + +Level: core + +Datatype: keyword + +L7 Network protocol name. ex. http, lumberjack, transport protocol. +The field value must be normalized to lowercase for querying. See the documentation section "Implementing ECS". + +example: `http` + +===== network.direction + +Level: core + +Datatype: keyword + +Direction of the network traffic. +Recommended values are: + * inbound + * outbound + * internal + * external + * unknown + +When mapping events from a host-based monitoring context, populate this field from the host's point of view. +When mapping events from a network or perimeter-based monitoring context, populate this field from the point of view of your network perimeter. + +example: `inbound` + +===== network.forwarded_ip + +Level: core + +Datatype: ip + +Host IP address when the source IP address is the proxy. + +example: `192.1.1.2` + +===== network.community_id + +Level: extended + +Datatype: keyword + +A hash of source and destination IPs and ports, as well as the protocol used in a communication. This is a tool-agnostic standard to identify flows. +Learn more at https://github.com/corelight/community-id-spec. + +example: `1:hO+sN4H+MG5MY/8hIrXPqc4ZQz0=` + +===== network.bytes + +Level: core + +Datatype: long + +Total bytes transferred in both directions. +If `source.bytes` and `destination.bytes` are known, `network.bytes` is their sum. + +example: `368` + +===== network.packets + +Level: core + +Datatype: long + +Total packets transferred in both directions. +If `source.packets` and `destination.packets` are known, `network.packets` is their sum. + +example: `24` + +[[ecs-observer]] +=== Observer fields + +An observer is defined as a special network, security, or application device used to detect, observe, or create network, security, or application-related events and metrics. +This could be a custom hardware appliance or a server that has been configured to run special network, security, or application software. Examples include firewalls, intrusion detection/prevention systems, network monitoring sensors, web application firewalls, data loss prevention systems, and APM servers. The observer.* fields shall be populated with details of the system, if any, that detects, observes and/or creates a network, security, or application event or metric. Message queues and ETL components used in processing events or metrics are not considered observers in ECS. + +==== Fields summary + +[options="header"] +|===== +| Field | Description | Type (Level) + +// =============================================================== + +| observer.mac +| MAC address of the observer +| keyword + +(core) + +// =============================================================== + +| observer.geo.location +| Longitude and latitude. +| geo_point + +(core) + +// =============================================================== + +| observer.os.platform +| Operating system platform (such centos, ubuntu, windows). +| keyword + +(extended) + +// =============================================================== + +| observer.ip +| IP address of the observer. +| ip + +(core) + +// =============================================================== + +| observer.geo.continent_name +| Name of the continent. +| keyword + +(core) + +// =============================================================== + +| observer.os.name +| Operating system name, without the version. +| keyword + +(extended) + +// =============================================================== + +| observer.hostname +| Hostname of the observer. +| keyword + +(core) + +// =============================================================== + +| observer.geo.country_name +| Country name. +| keyword + +(core) + +// =============================================================== + +| observer.os.full +| Operating system name, including the version or code name. +| keyword + +(extended) + +// =============================================================== + +| observer.vendor +| observer vendor information. +| keyword + +(core) + +// =============================================================== + +| observer.os.family +| OS family (such as redhat, debian, freebsd, windows). +| keyword + +(extended) + +// =============================================================== + +| observer.geo.region_name +| Region name. +| keyword + +(core) + +// =============================================================== + +| observer.version +| Observer version. +| keyword + +(core) + +// =============================================================== + +| observer.geo.city_name +| City name. +| keyword + +(core) + +// =============================================================== + +| observer.os.version +| Operating system version as a raw string. +| keyword + +(extended) + +// =============================================================== + +| observer.serial_number +| Observer serial number. +| keyword + +(extended) + +// =============================================================== + +| observer.geo.country_iso_code +| Country ISO code. +| keyword + +(core) + +// =============================================================== + +| observer.os.kernel +| Operating system kernel version as a raw string. +| keyword + +(extended) + +// =============================================================== + +| observer.type +| The type of the observer the data is coming from. +| keyword + +(core) + +// =============================================================== + +| observer.geo.region_iso_code +| Region ISO code. +| keyword + +(core) + +// =============================================================== + +| observer.geo.name +| User-defined description of a location. +| keyword + +(extended) + +// =============================================================== + +|===== + +[[ecs-observer-details]] +==== Field details + +===== observer.mac + +Level: core + +Datatype: keyword + +MAC address of the observer + + + +===== observer.geo.location + +Level: core + +Datatype: geo_point + +Longitude and latitude. + +example: `{ "lon": -73.614830, "lat": 45.505918 }` + +===== observer.os.platform + +Level: extended + +Datatype: keyword + +Operating system platform (such centos, ubuntu, windows). + +example: `darwin` + +===== observer.ip + +Level: core + +Datatype: ip + +IP address of the observer. + + + +===== observer.geo.continent_name + +Level: core + +Datatype: keyword + +Name of the continent. + +example: `North America` + +===== observer.os.name + +Level: extended + +Datatype: keyword + +Operating system name, without the version. + +example: `Mac OS X` + +===== observer.hostname + +Level: core + +Datatype: keyword + +Hostname of the observer. + + + +===== observer.geo.country_name + +Level: core + +Datatype: keyword + +Country name. + +example: `Canada` + +===== observer.os.full + +Level: extended + +Datatype: keyword + +Operating system name, including the version or code name. + +example: `Mac OS Mojave` + +===== observer.vendor + +Level: core + +Datatype: keyword + +observer vendor information. + + + +===== observer.os.family + +Level: extended + +Datatype: keyword + +OS family (such as redhat, debian, freebsd, windows). + +example: `debian` + +===== observer.geo.region_name + +Level: core + +Datatype: keyword + +Region name. + +example: `Quebec` + +===== observer.version + +Level: core + +Datatype: keyword + +Observer version. + + + +===== observer.geo.city_name + +Level: core + +Datatype: keyword + +City name. + +example: `Montreal` + +===== observer.os.version + +Level: extended + +Datatype: keyword + +Operating system version as a raw string. + +example: `10.14.1` + +===== observer.serial_number + +Level: extended + +Datatype: keyword + +Observer serial number. + + + +===== observer.geo.country_iso_code + +Level: core + +Datatype: keyword + +Country ISO code. + +example: `CA` + +===== observer.os.kernel + +Level: extended + +Datatype: keyword + +Operating system kernel version as a raw string. + +example: `4.4.0-112-generic` + +===== observer.type + +Level: core + +Datatype: keyword + +The type of the observer the data is coming from. +There is no predefined list of observer types. Some examples are `forwarder`, `firewall`, `ids`, `ips`, `proxy`, `poller`, `sensor`, `APM server`. + +example: `firewall` + +===== observer.geo.region_iso_code + +Level: core + +Datatype: keyword + +Region ISO code. + +example: `CA-QC` + +===== observer.geo.name + +Level: extended + +Datatype: keyword + +User-defined description of a location, at the level of granularity they care about. +Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. +Not typically used in automated geolocation. + +example: `boston-dc` + +[[ecs-organization]] +=== Organization fields + +The organization fields enrich data with information about the company or entity the data is associated with. +These fields help you arrange or filter data stored in an index by one or multiple organizations. + +==== Fields summary + +[options="header"] +|===== +| Field | Description | Type (Level) + +// =============================================================== + +| organization.name +| Organization name. +| keyword + +(extended) + +// =============================================================== + +| organization.id +| Unique identifier for the organization. +| keyword + +(extended) + +// =============================================================== + +|===== + +[[ecs-organization-details]] +==== Field details + +===== organization.name + +Level: extended + +Datatype: keyword + +Organization name. + + + +===== organization.id + +Level: extended + +Datatype: keyword + +Unique identifier for the organization. + + + +[[ecs-os]] +=== Operating System fields + +The OS fields contain information about the operating system. + +==== Fields summary + +[options="header"] +|===== +| Field | Description | Type (Level) + +// =============================================================== + +| os.platform +| Operating system platform (such centos, ubuntu, windows). +| keyword + +(extended) + +// =============================================================== + +| os.name +| Operating system name, without the version. +| keyword + +(extended) + +// =============================================================== + +| os.full +| Operating system name, including the version or code name. +| keyword + +(extended) + +// =============================================================== + +| os.family +| OS family (such as redhat, debian, freebsd, windows). +| keyword + +(extended) // =============================================================== -| http.response.body.bytes -| Size in bytes of the response body. +| os.version +| Operating system version as a raw string. +| keyword + +(extended) -example: 887 +// =============================================================== -| level: extended +| os.kernel +| Operating system kernel version as a raw string. +| keyword -type: long +(extended) // =============================================================== |===== -[[ecs-log]] -=== Log fields +[[ecs-os-details]] +==== Field details -Fields which are specific to log events. +===== os.platform -[options="header"] -|===== -| Field | Description | Level / Type +Level: extended -// =============================================================== +Datatype: keyword -| log.level -| Log level of the log event. +Operating system platform (such centos, ubuntu, windows). -example: err +example: `darwin` -| level: core +===== os.name -type: keyword +Level: extended -// =============================================================== +Datatype: keyword -| log.original -| Original log message with light interpretation only (encoding, newlines). +Operating system name, without the version. -example: Sep 19 08:26:10 localhost My log +example: `Mac OS X` -| level: core +===== os.full -type: keyword +Level: extended -// =============================================================== +Datatype: keyword -|===== +Operating system name, including the version or code name. -[[ecs-network]] -=== Network fields +example: `Mac OS Mojave` -The network is defined as the communication path over which a host or network event happens. -The network.* fields should be populated with details about the network activity associated with an event. +===== os.family -[options="header"] -|===== -| Field | Description | Level / Type +Level: extended -// =============================================================== +Datatype: keyword -| network.name -| Name given by operators to sections of their network. +OS family (such as redhat, debian, freebsd, windows). -example: Guest Wifi +example: `debian` -| level: extended +===== os.version -type: keyword +Level: extended -// =============================================================== +Datatype: keyword -| network.type -| In the OSI Model this would be the Network Layer. ipv4, ipv6, ipsec, pim, etc +Operating system version as a raw string. -example: ipv4 +example: `10.14.1` -| level: core +===== os.kernel -type: keyword +Level: extended -// =============================================================== +Datatype: keyword -| network.iana_number -| IANA Protocol Number. +Operating system kernel version as a raw string. -example: 6 +example: `4.4.0-112-generic` -| level: extended +[[ecs-process]] +=== Process fields -type: keyword +These fields contain information about a process. +These fields can help you correlate metrics information with a process id/name from a log message. The `process.pid` often stays in the metric itself and is copied to the global field for correlation. -// =============================================================== +==== Fields summary -| network.transport -| Protocol Name corresponding to the field `iana_number`. +[options="header"] +|===== +| Field | Description | Type (Level) -example: tcp +// =============================================================== -| level: core +| process.pid +| Process id. +| long -type: keyword +(core) // =============================================================== -| network.application -| Application level protocol name. +| process.name +| Process name. +| keyword + +(extended) -example: aim +// =============================================================== -| level: extended +| process.ppid +| Process parent id. +| long -type: keyword +(extended) // =============================================================== -| network.protocol -| L7 Network protocol name. +| process.args +| Array of process arguments. +| keyword + +(extended) -example: http +// =============================================================== -| level: core +| process.executable +| Absolute path to the process executable. +| keyword -type: keyword +(extended) // =============================================================== -| network.direction -| Direction of the network traffic. +| process.title +| Process title. +| keyword + +(extended) -example: inbound +// =============================================================== -| level: core +| process.thread.id +| Thread ID. +| long -type: keyword +(extended) // =============================================================== -| network.forwarded_ip -| Host IP address when the source IP address is the proxy. +| process.start +| The time the process started. +| date + +(extended) -example: 192.1.1.2 +// =============================================================== -| level: core +| process.working_directory +| The working directory of the process. +| keyword -type: ip +(extended) // =============================================================== -| network.community_id -| A hash of source and destination IPs and ports. +|===== -example: 1:hO+sN4H+MG5MY/8hIrXPqc4ZQz0= +[[ecs-process-details]] +==== Field details -| level: extended +===== process.pid -type: keyword +Level: core -// =============================================================== +Datatype: long -| network.bytes -| Total bytes transferred in both directions. +Process id. -example: 368 -| level: core -type: long +===== process.name -// =============================================================== +Level: extended -| network.packets -| Total packets transferred in both directions. +Datatype: keyword -example: 24 +Process name. +Sometimes called program name or similar. -| level: core +example: `ssh` -type: long +===== process.ppid -// =============================================================== +Level: extended -|===== +Datatype: long -[[ecs-observer]] -=== Observer fields +Process parent id. -An observer is defined as a special network, security, or application device used to detect, observe, or create network, security, or application-related events and metrics. -This could be a custom hardware appliance or a server that has been configured to run special network, security, or application software. Examples include firewalls, intrusion detection/prevention systems, network monitoring sensors, web application firewalls, data loss prevention systems, and APM servers. The observer.* fields shall be populated with details of the system, if any, that detects, observes and/or creates a network, security, or application event or metric. Message queues and ETL components used in processing events or metrics are not considered observers in ECS. -[options="header"] -|===== -| Field | Description | Level / Type -// =============================================================== +===== process.args -| observer.mac -| MAC address of the observer +Level: extended +Datatype: keyword +Array of process arguments. +May be filtered to protect sensitive information. -| level: core +example: `['ssh', '-l', 'user', '10.0.0.16']` -type: keyword +===== process.executable -// =============================================================== +Level: extended -| observer.geo.location -| Longitude and latitude. +Datatype: keyword -example: { "lon": -73.614830, "lat": 45.505918 } +Absolute path to the process executable. -| level: core +example: `/usr/bin/ssh` -type: geo_point +===== process.title -// =============================================================== +Level: extended -| observer.os.platform -| Operating system platform (such centos, ubuntu, windows). +Datatype: keyword -example: darwin +Process title. +The proctitle, some times the same as process name. Can also be different: for example a browser setting its title to the web page currently opened. -| level: extended -type: keyword -// =============================================================== +===== process.thread.id -| observer.ip -| IP address of the observer. +Level: extended +Datatype: long +Thread ID. -| level: core +example: `4242` -type: ip +===== process.start -// =============================================================== +Level: extended -| observer.geo.continent_name -| Name of the continent. +Datatype: date -example: North America +The time the process started. -| level: core +example: `2016-05-23T08:05:34.853Z` -type: keyword +===== process.working_directory -// =============================================================== +Level: extended -| observer.os.name -| Operating system name, without the version. +Datatype: keyword -example: Mac OS X +The working directory of the process. -| level: extended +example: `/home/alice` -type: keyword +[[ecs-related]] +=== Related fields -// =============================================================== +This field set is meant to facilitate pivoting around a piece of data. +Some pieces of information can be seen in many places in an ECS event. To facilitate searching for them, store an array of all seen values to their corresponding field in `related.`. +A concrete example is IP addresses, which can be under host, observer, source, destination, client, server, and network.forwarded_ip. If you append all IPs to `related.ip`, you can then search for a given IP trivially, no matter where it appeared, by querying `related.ip:a.b.c.d`. -| observer.hostname -| Hostname of the observer. +==== Fields summary +[options="header"] +|===== +| Field | Description | Type (Level) +// =============================================================== -| level: core +| related.ip +| All of the IPs seen on your event. +| ip -type: keyword +(extended) // =============================================================== -| observer.geo.country_name -| Country name. +|===== -example: Canada +[[ecs-related-details]] +==== Field details -| level: core +===== related.ip -type: keyword +Level: extended -// =============================================================== +Datatype: ip -| observer.os.full -| Operating system name, including the version or code name. +All of the IPs seen on your event. + + + +[[ecs-server]] +=== Server fields -example: Mac OS Mojave +A Server is defined as the responder in a network connection for events regarding sessions, connections, or bidirectional flow records. +For TCP events, the server is the receiver of the initial SYN packet(s) of the TCP connection. For other protocols, the server is generally the responder in the network transaction. Some systems actually use the term "responder" to refer the server in TCP connections. The server fields describe details about the system acting as the server in the network event. Server fields are usually populated in conjunction with client fields. Server fields are generally not populated for packet-level events. +Client / server representations can add semantic context to an exchange, which is helpful to visualize the data in certain situations. If your context falls in that category, you should still ensure that source and destination are filled appropriately. -| level: extended +==== Fields summary -type: keyword +[options="header"] +|===== +| Field | Description | Type (Level) // =============================================================== -| observer.vendor -| observer vendor information. +| server.address +| Server network address. +| keyword +(extended) +// =============================================================== -| level: core +| server.geo.location +| Longitude and latitude. +| geo_point -type: keyword +(core) // =============================================================== -| observer.os.family -| OS family (such as redhat, debian, freebsd, windows). +| server.user.id +| One or multiple unique identifiers of the user. +| keyword -example: debian +(core) -| level: extended +// =============================================================== + +| server.user.group.id +| Unique identifier for the group on the system/platform. +| keyword -type: keyword +(extended) // =============================================================== -| observer.geo.region_name -| Region name. +| server.ip +| IP address of the server. +| ip -example: Quebec +(core) -| level: core +// =============================================================== + +| server.user.group.name +| Name of the group. +| keyword -type: keyword +(extended) // =============================================================== -| observer.version -| Observer version. +| server.geo.continent_name +| Name of the continent. +| keyword +(core) +// =============================================================== -| level: core +| server.user.name +| Short name or login of the user. +| keyword -type: keyword +(core) // =============================================================== -| observer.geo.city_name -| City name. +| server.port +| Port of the server. +| long + +(core) -example: Montreal +// =============================================================== -| level: core +| server.geo.country_name +| Country name. +| keyword -type: keyword +(core) // =============================================================== -| observer.os.version -| Operating system version as a raw string. +| server.user.full_name +| User's full name, if available. +| keyword + +(extended) -example: 10.14.1 +// =============================================================== -| level: extended +| server.mac +| MAC address of the server. +| keyword -type: keyword +(core) // =============================================================== -| observer.serial_number -| Observer serial number. +| server.user.email +| User email address. +| keyword +(extended) +// =============================================================== -| level: extended +| server.geo.region_name +| Region name. +| keyword -type: keyword +(core) // =============================================================== -| observer.geo.country_iso_code -| Country ISO code. +| server.domain +| Server domain. +| keyword -example: CA +(core) + +// =============================================================== -| level: core +| server.user.hash +| Unique user hash to correlate information for a user in anonymized form. +| keyword -type: keyword +(extended) // =============================================================== -| observer.os.kernel -| Operating system kernel version as a raw string. +| server.geo.city_name +| City name. +| keyword -example: 4.4.0-112-generic +(core) + +// =============================================================== -| level: extended +| server.bytes +| Bytes sent from the server to the client. +| long -type: keyword +(core) // =============================================================== -| observer.type -| The type of the observer the data is coming from. +| server.geo.country_iso_code +| Country ISO code. +| keyword -example: firewall +(core) + +// =============================================================== -| level: core +| server.packets +| Packets sent from the server to the client. +| long -type: keyword +(core) // =============================================================== -| observer.geo.region_iso_code +| server.geo.region_iso_code | Region ISO code. +| keyword -example: CA-QC +(core) -| level: core +// =============================================================== + +| server.geo.name +| User-defined description of a location. +| keyword -type: keyword +(extended) // =============================================================== -| observer.geo.name -| User-defined description of a location. +|===== -example: boston-dc +[[ecs-server-details]] +==== Field details -| level: extended +===== server.address -type: keyword +Level: extended -// =============================================================== +Datatype: keyword -|===== +Some event server addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. +Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. -[[ecs-organization]] -=== Organization fields -The organization fields enrich data with information about the company or entity the data is associated with. -These fields help you arrange or filter data stored in an index by one or multiple organizations. -[options="header"] -|===== -| Field | Description | Level / Type +===== server.geo.location -// =============================================================== +Level: core + +Datatype: geo_point + +Longitude and latitude. + +example: `{ "lon": -73.614830, "lat": 45.505918 }` + +===== server.user.id + +Level: core + +Datatype: keyword + +One or multiple unique identifiers of the user. + + + +===== server.user.group.id + +Level: extended + +Datatype: keyword -| organization.name -| Organization name. +Unique identifier for the group on the system/platform. -| level: extended +===== server.ip -type: keyword +Level: core -// =============================================================== +Datatype: ip -| organization.id -| Unique identifier for the organization. +IP address of the server. +Can be one or multiple IPv4 or IPv6 addresses. -| level: extended +===== server.user.group.name -type: keyword +Level: extended -// =============================================================== +Datatype: keyword -|===== +Name of the group. -[[ecs-os]] -=== Operating System fields -The OS fields contain information about the operating system. -[options="header"] -|===== -| Field | Description | Level / Type +===== server.geo.continent_name -// =============================================================== +Level: core -| os.platform -| Operating system platform (such centos, ubuntu, windows). +Datatype: keyword -example: darwin +Name of the continent. -| level: extended +example: `North America` -type: keyword +===== server.user.name -// =============================================================== +Level: core -| os.name -| Operating system name, without the version. +Datatype: keyword -example: Mac OS X +Short name or login of the user. -| level: extended +example: `albert` -type: keyword +===== server.port -// =============================================================== +Level: core -| os.full -| Operating system name, including the version or code name. +Datatype: long -example: Mac OS Mojave +Port of the server. -| level: extended -type: keyword -// =============================================================== +===== server.geo.country_name -| os.family -| OS family (such as redhat, debian, freebsd, windows). +Level: core -example: debian +Datatype: keyword -| level: extended +Country name. -type: keyword +example: `Canada` -// =============================================================== +===== server.user.full_name -| os.version -| Operating system version as a raw string. +Level: extended -example: 10.14.1 +Datatype: keyword -| level: extended +User's full name, if available. -type: keyword +example: `Albert Einstein` -// =============================================================== +===== server.mac -| os.kernel -| Operating system kernel version as a raw string. +Level: core -example: 4.4.0-112-generic +Datatype: keyword -| level: extended +MAC address of the server. -type: keyword -// =============================================================== -|===== +===== server.user.email -[[ecs-process]] -=== Process fields +Level: extended -These fields contain information about a process. -These fields can help you correlate metrics information with a process id/name from a log message. The `process.pid` often stays in the metric itself and is copied to the global field for correlation. +Datatype: keyword -[options="header"] -|===== -| Field | Description | Level / Type +User email address. -// =============================================================== -| process.pid -| Process id. +===== server.geo.region_name +Level: core -| level: core +Datatype: keyword -type: long +Region name. -// =============================================================== +example: `Quebec` -| process.name -| Process name. +===== server.domain -example: ssh +Level: core -| level: extended +Datatype: keyword -type: keyword +Server domain. -// =============================================================== -| process.ppid -| Process parent id. +===== server.user.hash +Level: extended -| level: extended +Datatype: keyword -type: long +Unique user hash to correlate information for a user in anonymized form. +Useful if `user.id` or `user.name` contain confidential information and cannot be used. -// =============================================================== -| process.args -| Array of process arguments. -example: ['ssh', '-l', 'user', '10.0.0.16'] +===== server.geo.city_name -| level: extended +Level: core -type: keyword +Datatype: keyword -// =============================================================== +City name. -| process.executable -| Absolute path to the process executable. +example: `Montreal` -example: /usr/bin/ssh +===== server.bytes -| level: extended +Level: core -type: keyword +Datatype: long -// =============================================================== +Bytes sent from the server to the client. -| process.title -| Process title. +example: `184` +===== server.geo.country_iso_code +Level: core -| level: extended +Datatype: keyword -type: keyword +Country ISO code. -// =============================================================== +example: `CA` -| process.thread.id -| Thread ID. +===== server.packets -example: 4242 +Level: core -| level: extended +Datatype: long -type: long +Packets sent from the server to the client. -// =============================================================== +example: `12` -| process.start -| The time the process started. +===== server.geo.region_iso_code -example: 2016-05-23T08:05:34.853Z +Level: core -| level: extended +Datatype: keyword -type: date +Region ISO code. -// =============================================================== +example: `CA-QC` -| process.working_directory -| The working directory of the process. +===== server.geo.name -example: /home/alice +Level: extended -| level: extended +Datatype: keyword -type: keyword +User-defined description of a location, at the level of granularity they care about. +Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. +Not typically used in automated geolocation. -// =============================================================== +example: `boston-dc` -|===== +[[ecs-service]] +=== Service fields -[[ecs-related]] -=== Related fields +The service fields describe the service for or from which the data was collected. +These fields help you find and correlate logs for a specific service and version. -This field set is meant to facilitate pivoting around a piece of data. -Some pieces of information can be seen in many places in an ECS event. To facilitate searching for them, store an array of all seen values to their corresponding field in `related.`. -A concrete example is IP addresses, which can be under host, observer, source, destination, client, server, and network.forwarded_ip. If you append all IPs to `related.ip`, you can then search for a given IP trivially, no matter where it appeared, by querying `related.ip:a.b.c.d`. +==== Fields summary [options="header"] |===== -| Field | Description | Level / Type +| Field | Description | Type (Level) // =============================================================== -| related.ip -| All of the IPs seen on your event. +| service.id +| Unique identifier of the running service. +| keyword +(core) +// =============================================================== -| level: extended +| service.name +| Name of the service. +| keyword -type: ip +(core) // =============================================================== -|===== +| service.type +| The type of the service. +| keyword -[[ecs-server]] -=== Server fields +(core) -A Server is defined as the responder in a network connection for events regarding sessions, connections, or bidirectional flow records. -For TCP events, the server is the receiver of the initial SYN packet(s) of the TCP connection. For other protocols, the server is generally the responder in the network transaction. Some systems actually use the term "responder" to refer the server in TCP connections. The server fields describe details about the system acting as the server in the network event. Server fields are usually populated in conjunction with client fields. Server fields are generally not populated for packet-level events. -Client / server representations can add semantic context to an exchange, which is helpful to visualize the data in certain situations. If your context falls in that category, you should still ensure that source and destination are filled appropriately. +// =============================================================== -[options="header"] -|===== -| Field | Description | Level / Type +| service.state +| Current state of the service. +| keyword + +(core) // =============================================================== -| server.address -| Server network address. +| service.version +| Version of the service. +| keyword +(core) +// =============================================================== -| level: extended +| service.ephemeral_id +| Ephemeral identifier of this service. +| keyword -type: keyword +(extended) // =============================================================== -| server.geo.location -| Longitude and latitude. - -example: { "lon": -73.614830, "lat": 45.505918 } +|===== -| level: core +[[ecs-service-details]] +==== Field details -type: geo_point +===== service.id -// =============================================================== +Level: core -| server.user.id -| One or multiple unique identifiers of the user. +Datatype: keyword +Unique identifier of the running service. +This id should uniquely identify this service. This makes it possible to correlate logs and metrics for one specific service. +Example: If you are experiencing issues with one redis instance, you can filter on that id to see metrics and logs for that single instance. +example: `d37e5ebfe0ae6c4972dbe9f0174a1637bb8247f6` -| level: core +===== service.name -type: keyword +Level: core -// =============================================================== +Datatype: keyword -| server.user.group.id -| Unique identifier for the group on the system/platform. +Name of the service data is collected from. +The name of the service is normally user given. This allows if two instances of the same service are running on the same machine they can be differentiated by the `service.name`. +Also it allows for distributed services that run on multiple hosts to correlate the related instances based on the name. +In the case of Elasticsearch the service.name could contain the cluster name. For Beats the service.name is by default a copy of the `service.type` field if no name is specified. +example: `elasticsearch-metrics` +===== service.type -| level: extended +Level: core -type: keyword +Datatype: keyword -// =============================================================== +The type of the service data is collected from. +The type can be used to group and correlate logs and metrics from one service type. +Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. -| server.ip -| IP address of the server. +example: `elasticsearch` +===== service.state +Level: core -| level: core +Datatype: keyword -type: ip +Current state of the service. -// =============================================================== -| server.user.group.name -| Name of the group. +===== service.version +Level: core -| level: extended +Datatype: keyword -type: keyword +Version of the service the data was collected from. +This allows to look at a data set only for a specific version of a service. -// =============================================================== +example: `3.2.4` -| server.geo.continent_name -| Name of the continent. +===== service.ephemeral_id -example: North America +Level: extended -| level: core +Datatype: keyword -type: keyword +Ephemeral identifier of this service (if one exists). +This id normally changes across restarts, but `service.id` does not. -// =============================================================== +example: `8a4f500f` -| server.user.name -| Short name or login of the user. +[[ecs-source]] +=== Source fields -example: albert +Source fields describe details about the source of a packet/event. +Source fields are usually populated in conjunction with destination fields. -| level: core +==== Fields summary -type: keyword +[options="header"] +|===== +| Field | Description | Type (Level) // =============================================================== -| server.port -| Port of the server. +| source.address +| Source network address. +| keyword +(extended) +// =============================================================== -| level: core +| source.geo.location +| Longitude and latitude. +| geo_point -type: long +(core) // =============================================================== -| server.geo.country_name -| Country name. +| source.user.id +| One or multiple unique identifiers of the user. +| keyword + +(core) -example: Canada +// =============================================================== -| level: core +| source.user.group.id +| Unique identifier for the group on the system/platform. +| keyword -type: keyword +(extended) // =============================================================== -| server.user.full_name -| User's full name, if available. +| source.ip +| IP address of the source. +| ip + +(core) -example: Albert Einstein +// =============================================================== -| level: extended +| source.user.group.name +| Name of the group. +| keyword -type: keyword +(extended) // =============================================================== -| server.mac -| MAC address of the server. +| source.geo.continent_name +| Name of the continent. +| keyword +(core) +// =============================================================== -| level: core +| source.user.name +| Short name or login of the user. +| keyword -type: keyword +(core) // =============================================================== -| server.user.email -| User email address. +| source.port +| Port of the source. +| long +(core) +// =============================================================== -| level: extended +| source.geo.country_name +| Country name. +| keyword -type: keyword +(core) // =============================================================== -| server.geo.region_name -| Region name. +| source.user.full_name +| User's full name, if available. +| keyword + +(extended) -example: Quebec +// =============================================================== -| level: core +| source.mac +| MAC address of the source. +| keyword -type: keyword +(core) // =============================================================== -| server.domain -| Server domain. +| source.user.email +| User email address. +| keyword +(extended) +// =============================================================== -| level: core +| source.geo.region_name +| Region name. +| keyword -type: keyword +(core) // =============================================================== -| server.user.hash -| Unique user hash to correlate information for a user in anonymized form. +| source.domain +| Source domain. +| keyword +(core) +// =============================================================== -| level: extended +| source.user.hash +| Unique user hash to correlate information for a user in anonymized form. +| keyword -type: keyword +(extended) // =============================================================== -| server.geo.city_name +| source.geo.city_name | City name. +| keyword -example: Montreal - -| level: core - -type: keyword +(core) // =============================================================== -| server.bytes -| Bytes sent from the server to the client. - -example: 184 - -| level: core +| source.bytes +| Bytes sent from the source to the destination. +| long -type: long +(core) // =============================================================== -| server.geo.country_iso_code +| source.geo.country_iso_code | Country ISO code. +| keyword + +(core) -example: CA +// =============================================================== -| level: core +| source.packets +| Packets sent from the source to the destination. +| long -type: keyword +(core) // =============================================================== -| server.packets -| Packets sent from the server to the client. +| source.geo.region_iso_code +| Region ISO code. +| keyword + +(core) -example: 12 +// =============================================================== -| level: core +| source.geo.name +| User-defined description of a location. +| keyword -type: long +(extended) // =============================================================== -| server.geo.region_iso_code -| Region ISO code. +|===== -example: CA-QC +[[ecs-source-details]] +==== Field details -| level: core +===== source.address -type: keyword +Level: extended -// =============================================================== +Datatype: keyword -| server.geo.name -| User-defined description of a location. +Some event source addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. +Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. -example: boston-dc -| level: extended -type: keyword +===== source.geo.location -// =============================================================== +Level: core -|===== +Datatype: geo_point -[[ecs-service]] -=== Service fields +Longitude and latitude. -The service fields describe the service for or from which the data was collected. -These fields help you find and correlate logs for a specific service and version. +example: `{ "lon": -73.614830, "lat": 45.505918 }` -[options="header"] -|===== -| Field | Description | Level / Type +===== source.user.id -// =============================================================== +Level: core -| service.id -| Unique identifier of the running service. +Datatype: keyword -example: d37e5ebfe0ae6c4972dbe9f0174a1637bb8247f6 +One or multiple unique identifiers of the user. -| level: core -type: keyword -// =============================================================== +===== source.user.group.id -| service.name -| Name of the service. +Level: extended -example: elasticsearch-metrics +Datatype: keyword -| level: core +Unique identifier for the group on the system/platform. -type: keyword -// =============================================================== -| service.type -| The type of the service. +===== source.ip -example: elasticsearch +Level: core -| level: core +Datatype: ip -type: keyword +IP address of the source. +Can be one or multiple IPv4 or IPv6 addresses. -// =============================================================== -| service.state -| Current state of the service. +===== source.user.group.name +Level: extended -| level: core +Datatype: keyword -type: keyword +Name of the group. -// =============================================================== -| service.version -| Version of the service. -example: 3.2.4 +===== source.geo.continent_name -| level: core +Level: core -type: keyword +Datatype: keyword -// =============================================================== +Name of the continent. -| service.ephemeral_id -| Ephemeral identifier of this service. +example: `North America` -example: 8a4f500f +===== source.user.name -| level: extended +Level: core -type: keyword +Datatype: keyword -// =============================================================== +Short name or login of the user. -|===== +example: `albert` -[[ecs-source]] -=== Source fields +===== source.port + +Level: core + +Datatype: long + +Port of the source. -Source fields describe details about the source of a packet/event. -Source fields are usually populated in conjunction with destination fields. -[options="header"] -|===== -| Field | Description | Level / Type -// =============================================================== +===== source.geo.country_name -| source.address -| Source network address. +Level: core +Datatype: keyword +Country name. -| level: extended +example: `Canada` -type: keyword +===== source.user.full_name -// =============================================================== +Level: extended -| source.geo.location -| Longitude and latitude. +Datatype: keyword -example: { "lon": -73.614830, "lat": 45.505918 } +User's full name, if available. -| level: core +example: `Albert Einstein` -type: geo_point +===== source.mac -// =============================================================== +Level: core -| source.user.id -| One or multiple unique identifiers of the user. +Datatype: keyword +MAC address of the source. -| level: core -type: keyword +===== source.user.email -// =============================================================== +Level: extended -| source.user.group.id -| Unique identifier for the group on the system/platform. +Datatype: keyword +User email address. -| level: extended -type: keyword +===== source.geo.region_name -// =============================================================== +Level: core -| source.ip -| IP address of the source. +Datatype: keyword +Region name. +example: `Quebec` -| level: core +===== source.domain -type: ip +Level: core -// =============================================================== +Datatype: keyword -| source.user.group.name -| Name of the group. +Source domain. -| level: extended +===== source.user.hash -type: keyword +Level: extended -// =============================================================== +Datatype: keyword -| source.geo.continent_name -| Name of the continent. +Unique user hash to correlate information for a user in anonymized form. +Useful if `user.id` or `user.name` contain confidential information and cannot be used. -example: North America -| level: core -type: keyword +===== source.geo.city_name -// =============================================================== +Level: core -| source.user.name -| Short name or login of the user. +Datatype: keyword -example: albert +City name. -| level: core +example: `Montreal` -type: keyword +===== source.bytes -// =============================================================== +Level: core -| source.port -| Port of the source. +Datatype: long +Bytes sent from the source to the destination. +example: `184` -| level: core +===== source.geo.country_iso_code -type: long +Level: core -// =============================================================== +Datatype: keyword -| source.geo.country_name -| Country name. +Country ISO code. -example: Canada +example: `CA` -| level: core +===== source.packets -type: keyword +Level: core -// =============================================================== +Datatype: long -| source.user.full_name -| User's full name, if available. +Packets sent from the source to the destination. -example: Albert Einstein +example: `12` -| level: extended +===== source.geo.region_iso_code -type: keyword +Level: core -// =============================================================== +Datatype: keyword -| source.mac -| MAC address of the source. +Region ISO code. +example: `CA-QC` +===== source.geo.name -| level: core +Level: extended -type: keyword +Datatype: keyword -// =============================================================== +User-defined description of a location, at the level of granularity they care about. +Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. +Not typically used in automated geolocation. -| source.user.email -| User email address. +example: `boston-dc` +[[ecs-url]] +=== URL fields +URL fields provide support for complete or partial URLs, and supports the breaking down into scheme, domain, path, and so on. -| level: extended +==== Fields summary -type: keyword +[options="header"] +|===== +| Field | Description | Type (Level) // =============================================================== -| source.geo.region_name -| Region name. +| url.original +| Unmodified original url as seen in the event source. +| keyword -example: Quebec +(extended) + +// =============================================================== -| level: core +| url.full +| Full unparsed URL. +| keyword -type: keyword +(extended) // =============================================================== -| source.domain -| Source domain. +| url.scheme +| Scheme of the url. +| keyword +(extended) +// =============================================================== -| level: core +| url.domain +| Domain of the url. +| keyword -type: keyword +(extended) // =============================================================== -| source.user.hash -| Unique user hash to correlate information for a user in anonymized form. +| url.port +| Port of the request, such as 443. +| long +(extended) +// =============================================================== -| level: extended +| url.path +| Path of the request, such as "/search". +| keyword -type: keyword +(extended) // =============================================================== -| source.geo.city_name -| City name. +| url.query +| Query string of the request. +| keyword -example: Montreal +(extended) -| level: core +// =============================================================== + +| url.fragment +| Portion of the url after the `#`. +| keyword -type: keyword +(extended) // =============================================================== -| source.bytes -| Bytes sent from the source to the destination. +| url.username +| Username of the request. +| keyword -example: 184 +(extended) -| level: core +// =============================================================== + +| url.password +| Password of the request. +| keyword -type: long +(extended) // =============================================================== -| source.geo.country_iso_code -| Country ISO code. +|===== -example: CA +[[ecs-url-details]] +==== Field details -| level: core +===== url.original -type: keyword +Level: extended -// =============================================================== +Datatype: keyword -| source.packets -| Packets sent from the source to the destination. +Unmodified original url as seen in the event source. +Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. +This field is meant to represent the URL as it was observed, complete or not. -example: 12 +example: `https://www.elastic.co:443/search?q=elasticsearch#top or /search?q=elasticsearch` -| level: core +===== url.full -type: long +Level: extended -// =============================================================== +Datatype: keyword -| source.geo.region_iso_code -| Region ISO code. +If full URLs are important to your use case, they should be stored in `url.full`, whether this field is reconstructed or present in the event source. -example: CA-QC +example: `https://www.elastic.co:443/search?q=elasticsearch#top` -| level: core +===== url.scheme -type: keyword +Level: extended -// =============================================================== +Datatype: keyword -| source.geo.name -| User-defined description of a location. +Scheme of the request, such as "https". +Note: The `:` is not part of the scheme. -example: boston-dc +example: `https` -| level: extended +===== url.domain -type: keyword +Level: extended -// =============================================================== +Datatype: keyword -|===== +Domain of the url, such as "www.elastic.co". +In some cases a URL may refer to an IP and/or port directly, without a domain name. In this case, the IP address would go to the `domain` field. -[[ecs-url]] -=== URL fields +example: `www.elastic.co` -URL fields provide support for complete or partial URLs, and supports the breaking down into scheme, domain, path, and so on. +===== url.port -[options="header"] -|===== -| Field | Description | Level / Type +Level: extended -// =============================================================== +Datatype: long -| url.original -| Unmodified original url as seen in the event source. +Port of the request, such as 443. -example: https://www.elastic.co:443/search?q=elasticsearch#top or /search?q=elasticsearch +example: `443` -| level: extended +===== url.path -type: keyword +Level: extended -// =============================================================== +Datatype: keyword -| url.full -| Full unparsed URL. +Path of the request, such as "/search". -example: https://www.elastic.co:443/search?q=elasticsearch#top -| level: extended -type: keyword +===== url.query -// =============================================================== +Level: extended -| url.scheme -| Scheme of the url. +Datatype: keyword -example: https +The query field describes the query string of the request, such as "q=elasticsearch". +The `?` is excluded from the query string. If a URL contains no `?`, there is no query field. If there is a `?` but no query, the query field exists with an empty string. The `exists` query can be used to differentiate between the two cases. -| level: extended -type: keyword -// =============================================================== +===== url.fragment -| url.domain -| Domain of the url. +Level: extended -example: www.elastic.co +Datatype: keyword -| level: extended +Portion of the url after the `#`, such as "top". +The `#` is not part of the fragment. -type: keyword -// =============================================================== -| url.port -| Port of the request, such as 443. +===== url.username -example: 443 +Level: extended -| level: extended +Datatype: keyword -type: long +Username of the request. -// =============================================================== -| url.path -| Path of the request, such as "/search". +===== url.password +Level: extended -| level: extended +Datatype: keyword -type: keyword +Password of the request. -// =============================================================== -| url.query -| Query string of the request. +[[ecs-user]] +=== User fields +The user fields describe information about the user that is relevant to the event. +Fields can have one entry or multiple entries. If a user has more than one id, provide an array that includes all of them. -| level: extended +==== Fields summary -type: keyword +[options="header"] +|===== +| Field | Description | Type (Level) // =============================================================== -| url.fragment -| Portion of the url after the `#`. +| user.id +| One or multiple unique identifiers of the user. +| keyword +(core) +// =============================================================== -| level: extended +| user.group.id +| Unique identifier for the group on the system/platform. +| keyword -type: keyword +(extended) // =============================================================== -| url.username -| Username of the request. +| user.group.name +| Name of the group. +| keyword +(extended) +// =============================================================== -| level: extended +| user.name +| Short name or login of the user. +| keyword -type: keyword +(core) // =============================================================== -| url.password -| Password of the request. +| user.full_name +| User's full name, if available. +| keyword +(extended) +// =============================================================== -| level: extended +| user.email +| User email address. +| keyword -type: keyword +(extended) // =============================================================== -|===== +| user.hash +| Unique user hash to correlate information for a user in anonymized form. +| keyword -[[ecs-user]] -=== User fields +(extended) -The user fields describe information about the user that is relevant to the event. -Fields can have one entry or multiple entries. If a user has more than one id, provide an array that includes all of them. +// =============================================================== -[options="header"] |===== -| Field | Description | Level / Type -// =============================================================== - -| user.id -| One or multiple unique identifiers of the user. +[[ecs-user-details]] +==== Field details +===== user.id +Level: core -| level: core +Datatype: keyword -type: keyword +One or multiple unique identifiers of the user. -// =============================================================== -| user.group.id -| Unique identifier for the group on the system/platform. +===== user.group.id +Level: extended -| level: extended +Datatype: keyword -type: keyword +Unique identifier for the group on the system/platform. -// =============================================================== -| user.group.name -| Name of the group. +===== user.group.name +Level: extended -| level: extended +Datatype: keyword -type: keyword +Name of the group. -// =============================================================== -| user.name -| Short name or login of the user. -example: albert +===== user.name -| level: core +Level: core -type: keyword +Datatype: keyword -// =============================================================== +Short name or login of the user. -| user.full_name -| User's full name, if available. +example: `albert` -example: Albert Einstein +===== user.full_name -| level: extended +Level: extended -type: keyword +Datatype: keyword -// =============================================================== +User's full name, if available. -| user.email -| User email address. +example: `Albert Einstein` +===== user.email +Level: extended -| level: extended +Datatype: keyword -type: keyword +User email address. -// =============================================================== -| user.hash -| Unique user hash to correlate information for a user in anonymized form. +===== user.hash +Level: extended -| level: extended +Datatype: keyword -type: keyword +Unique user hash to correlate information for a user in anonymized form. +Useful if `user.id` or `user.name` contain confidential information and cannot be used. -// =============================================================== -|===== [[ecs-user_agent]] === User agent fields @@ -3323,120 +5437,195 @@ type: keyword The user_agent fields normally come from a browser request. They often show up in web service logs coming from the parsed user agent string. +==== Fields summary + [options="header"] |===== -| Field | Description | Level / Type +| Field | Description | Type (Level) // =============================================================== | user_agent.original | Unparsed version of the user_agent. +| keyword -example: Mozilla/5.0 (iPhone; CPU iPhone OS 12_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.0 Mobile/15E148 Safari/604.1 - -| level: extended - -type: keyword +(extended) // =============================================================== | user_agent.os.platform | Operating system platform (such centos, ubuntu, windows). +| keyword -example: darwin - -| level: extended - -type: keyword +(extended) // =============================================================== | user_agent.name | Name of the user agent. +| keyword -example: Safari - -| level: extended - -type: keyword +(extended) // =============================================================== | user_agent.os.name | Operating system name, without the version. +| keyword -example: Mac OS X - -| level: extended - -type: keyword +(extended) // =============================================================== | user_agent.version | Version of the user agent. +| keyword -example: 12.0 - -| level: extended - -type: keyword +(extended) // =============================================================== | user_agent.os.full | Operating system name, including the version or code name. +| keyword -example: Mac OS Mojave - -| level: extended - -type: keyword +(extended) // =============================================================== | user_agent.device.name | Name of the device. +| keyword -example: iPhone - -| level: extended - -type: keyword +(extended) // =============================================================== | user_agent.os.family | OS family (such as redhat, debian, freebsd, windows). +| keyword -example: debian - -| level: extended - -type: keyword +(extended) // =============================================================== | user_agent.os.version | Operating system version as a raw string. +| keyword -example: 10.14.1 - -| level: extended - -type: keyword +(extended) // =============================================================== | user_agent.os.kernel | Operating system kernel version as a raw string. +| keyword -example: 4.4.0-112-generic - -| level: extended - -type: keyword +(extended) // =============================================================== |===== + +[[ecs-user_agent-details]] +==== Field details + +===== user_agent.original + +Level: extended + +Datatype: keyword + +Unparsed version of the user_agent. + +example: `Mozilla/5.0 (iPhone; CPU iPhone OS 12_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.0 Mobile/15E148 Safari/604.1` + +===== user_agent.os.platform + +Level: extended + +Datatype: keyword + +Operating system platform (such centos, ubuntu, windows). + +example: `darwin` + +===== user_agent.name + +Level: extended + +Datatype: keyword + +Name of the user agent. + +example: `Safari` + +===== user_agent.os.name + +Level: extended + +Datatype: keyword + +Operating system name, without the version. + +example: `Mac OS X` + +===== user_agent.version + +Level: extended + +Datatype: keyword + +Version of the user agent. + +example: `12.0` + +===== user_agent.os.full + +Level: extended + +Datatype: keyword + +Operating system name, including the version or code name. + +example: `Mac OS Mojave` + +===== user_agent.device.name + +Level: extended + +Datatype: keyword + +Name of the device. + +example: `iPhone` + +===== user_agent.os.family + +Level: extended + +Datatype: keyword + +OS family (such as redhat, debian, freebsd, windows). + +example: `debian` + +===== user_agent.os.version + +Level: extended + +Datatype: keyword + +Operating system version as a raw string. + +example: `10.14.1` + +===== user_agent.os.kernel + +Level: extended + +Datatype: keyword + +Operating system kernel version as a raw string. + +example: `4.4.0-112-generic` diff --git a/scripts/generators/asciidoc_fields.py b/scripts/generators/asciidoc_fields.py index 32524d9a1d..4491de053f 100644 --- a/scripts/generators/asciidoc_fields.py +++ b/scripts/generators/asciidoc_fields.py @@ -2,8 +2,8 @@ def generate(ecs_nested, ecs_version): - save_asciidoc('docs/fields.asciidoc', render_field_index(ecs_nested)) - save_asciidoc('docs/field-details.asciidoc', render_field_details(ecs_nested)) + save_asciidoc('docs/fields.asciidoc', page_field_index(ecs_nested)) + save_asciidoc('docs/field-details.asciidoc', page_field_details(ecs_nested)) # Helpers @@ -29,16 +29,16 @@ def sorted_by_keys(dict, sort_keys): # Field Index -def render_field_index(ecs_nested): +def page_field_index(ecs_nested): page_text = index_header() for fieldset in sorted_by_keys(ecs_nested, ['group', 'name']): - page_text += render_index_row(fieldset) + page_text += render_field_index_row(fieldset) page_text += table_footer() page_text += index_footer() return page_text -def render_index_row(fieldset): +def render_field_index_row(fieldset): return index_row().format( fieldset_id='ecs-' + fieldset['name'], fieldset_title=fieldset['title'], @@ -46,9 +46,9 @@ def render_index_row(fieldset): ) -# Field Details +# Field Details Page -def render_field_details(ecs_nested): +def page_field_details(ecs_nested): page_text = '' for fieldset in sorted_by_keys(ecs_nested, ['group', 'name']): page_text += render_fieldset(fieldset) @@ -56,24 +56,46 @@ def render_field_details(ecs_nested): def render_fieldset(fieldset): - fieldset_text = fieldset_header().format( - fieldset_id='ecs-' + fieldset['name'], + summary_text = fieldset_summary_header().format( + fieldset_name=fieldset['name'], fieldset_description=fieldset['description'], fieldset_title=fieldset['title'] ) + detail_text = fieldset_details_header().format( + fieldset_name=fieldset['name'], + fieldset_title=fieldset['title'] + ) + for field in sorted_by_keys(fieldset['fields'], 'order'): - fieldset_text += render_field(field) - fieldset_text += table_footer() - return fieldset_text + summary_text += render_field_summary_row(field) + detail_text += render_field_details(field) + summary_text += table_footer() + return summary_text + detail_text -def render_field(field): - example = '' - if 'example' in field: - example = 'example: ' + str(field['example']) + +def render_field_summary_row(field): + # example = '' + # if 'example' in field: + # example = "example: `{}`".format(str(field['example'])) field_text = field_row().format( - field_name=field['flat_name'], + field_flat_name=field['flat_name'], field_short=field['short'], + # field_example=example, + field_level=field['level'], + field_type=field['type'], + ) + return field_text + + +def render_field_details(field): + example = '' + if 'example' in field: + example = "example: `{}`".format(str(field['example'])) + field_text = field_details().format( + field_name=field['name'], + field_flat_name=field['flat_name'], + field_description=field['description'], field_example=example, field_level=field['level'], field_type=field['type'], @@ -121,16 +143,18 @@ def index_footer(): # Field Details -def fieldset_header(): +def fieldset_summary_header(): return ''' -[[{fieldset_id}]] +[[ecs-{fieldset_name}]] === {fieldset_title} fields {fieldset_description} +==== Fields summary + [options="header"] |===== -| Field | Description | Level / Type +| Field | Description | Type (Level) // =============================================================== ''' @@ -138,18 +162,37 @@ def fieldset_header(): def field_row(): return ''' -| {field_name} +| {field_flat_name} | {field_short} +| {field_type} -{field_example} +({field_level}) + +// =============================================================== +''' -| level: {field_level} -type: {field_type} +def fieldset_details_header(): + return ''' +[[ecs-{fieldset_name}-details]] +==== Field details +''' -// =============================================================== + +def field_details(): + return ''' +===== {field_flat_name} + +Level: {field_level} + +Datatype: {field_type} + +{field_description} + +{field_example} ''' + # File From 1f5b225c575e644fb99c7c893aec2efa4b7f83a8 Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Fri, 1 Mar 2019 15:14:00 -0500 Subject: [PATCH 14/28] Get back to the initial 3 column design --- docs/field-details.asciidoc | 5642 ++++++++----------------- scripts/generators/asciidoc_fields.py | 40 +- 2 files changed, 1826 insertions(+), 3856 deletions(-) diff --git a/docs/field-details.asciidoc b/docs/field-details.asciidoc index 053c6f740d..d975fd724f 100644 --- a/docs/field-details.asciidoc +++ b/docs/field-details.asciidoc @@ -8,94 +8,63 @@ The `base` field set contains all fields which are on the top level. These field [options="header"] |===== -| Field | Description | Type (Level) +| Field | Description | Level // =============================================================== | @timestamp | Date/time when the event originated. -| date - -(core) - -// =============================================================== - -| tags -| List of keywords used to tag each event. -| keyword - -(core) - -// =============================================================== - -| labels -| Custom key/value pairs. -| object - -(core) - -// =============================================================== - -| message -| Log message optimized for viewing in a log viewer. -| text - -(core) - -// =============================================================== - -|===== - -[[ecs-base-details]] -==== Field details - -===== @timestamp - -Level: core - -Datatype: date - -Date/time when the event originated. This is the date/time extracted from the event, typically representing when the event was generated by the source. If the event source has no original timestamp, this value is typically populated by the first time the event was received by the pipeline. Required field for all events. +type: date + example: `2016-05-23T08:05:34.853Z` -===== tags +| core -Level: core +// =============================================================== -Datatype: keyword +| tags +| List of keywords used to tag each event. -List of keywords used to tag each event. +type: keyword example: `["production", "env2"]` -===== labels - -Level: core +| core -Datatype: object +// =============================================================== -Custom key/value pairs. +| labels +| Custom key/value pairs. Can be used to add meta information to events. Should not contain nested objects. All values are stored as keyword. Example: `docker` and `k8s` labels. -example: `{'application': 'foo-bar', 'env': 'production'}` +type: object -===== message +example: `{'application': 'foo-bar', 'env': 'production'}` -Level: core +| core -Datatype: text +// =============================================================== -For log events the message field contains the log message, optimized for viewing in a log viewer. +| message +| For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. +type: text + example: `Hello World` +| core + +// =============================================================== + +|===== + [[ecs-agent]] === Agent fields @@ -106,110 +75,72 @@ Examples include Beats. Agents may also run on observers. ECS agent.* fields sha [options="header"] |===== -| Field | Description | Type (Level) +| Field | Description | Level // =============================================================== | agent.version | Version of the agent. -| keyword - -(core) - -// =============================================================== - -| agent.name -| Custom name of the agent. -| keyword - -(core) - -// =============================================================== - -| agent.type -| Type of the agent. -| keyword - -(core) - -// =============================================================== - -| agent.id -| Unique identifier of this agent. -| keyword - -(core) - -// =============================================================== - -| agent.ephemeral_id -| Ephemeral identifier of this agent. -| keyword - -(extended) - -// =============================================================== - -|===== -[[ecs-agent-details]] -==== Field details - -===== agent.version - -Level: core - -Datatype: keyword - -Version of the agent. +type: keyword example: `6.0.0-rc2` -===== agent.name +| core -Level: core - -Datatype: keyword +// =============================================================== -Custom name of the agent. +| agent.name +| Custom name of the agent. This is a name that can be given to an agent. This can be helpful if for example two Filebeat instances are running on the same host but a human readable separation is needed on which Filebeat instance data is coming from. If no name is given, the name is often left empty. -example: `foo` +type: keyword -===== agent.type +example: `foo` -Level: core +| core -Datatype: keyword +// =============================================================== -Type of the agent. +| agent.type +| Type of the agent. The agent type stays always the same and should be given by the agent used. In case of Filebeat the agent would always be Filebeat also if two Filebeat instances are run on the same machine. -example: `filebeat` +type: keyword -===== agent.id +example: `filebeat` -Level: core +| core -Datatype: keyword +// =============================================================== -Unique identifier of this agent (if one exists). +| agent.id +| Unique identifier of this agent (if one exists). Example: For Beats this would be beat.id. -example: `8a4f500d` +type: keyword -===== agent.ephemeral_id +example: `8a4f500d` -Level: extended +| core -Datatype: keyword +// =============================================================== -Ephemeral identifier of this agent (if one exists). +| agent.ephemeral_id +| Ephemeral identifier of this agent (if one exists). This id normally changes across restarts, but `agent.id` does not. +type: keyword + example: `8a4f500f` +| extended + +// =============================================================== + +|===== + [[ecs-client]] === Client fields @@ -221,5411 +152,3446 @@ Client / server representations can add semantic context to an exchange, which i [options="header"] |===== -| Field | Description | Type (Level) +| Field | Description | Level // =============================================================== | client.address -| Client network address. -| keyword +| Some event client addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. +Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. -(extended) +type: keyword + + + +| extended // =============================================================== | client.geo.location | Longitude and latitude. -| geo_point -(core) +type: geo_point + +example: `{ "lon": -73.614830, "lat": 45.505918 }` + +| core // =============================================================== | client.user.id | One or multiple unique identifiers of the user. -| keyword -(core) +type: keyword + + + +| core // =============================================================== | client.user.group.id | Unique identifier for the group on the system/platform. -| keyword -(extended) +type: keyword + + + +| extended // =============================================================== | client.ip | IP address of the client. -| ip +Can be one or multiple IPv4 or IPv6 addresses. + +type: ip + + -(core) +| core // =============================================================== | client.user.group.name | Name of the group. -| keyword -(extended) +type: keyword + + + +| extended // =============================================================== | client.geo.continent_name | Name of the continent. -| keyword -(core) +type: keyword + +example: `North America` + +| core // =============================================================== | client.user.name | Short name or login of the user. -| keyword -(core) +type: keyword + +example: `albert` + +| core // =============================================================== | client.port | Port of the client. -| long -(core) +type: long + + + +| core // =============================================================== | client.geo.country_name | Country name. -| keyword -(core) +type: keyword + +example: `Canada` + +| core // =============================================================== | client.user.full_name | User's full name, if available. -| keyword -(extended) +type: keyword + +example: `Albert Einstein` + +| extended // =============================================================== | client.mac | MAC address of the client. -| keyword -(core) +type: keyword + + + +| core // =============================================================== | client.user.email | User email address. -| keyword -(extended) +type: keyword + + + +| extended // =============================================================== | client.geo.region_name | Region name. -| keyword -(core) +type: keyword + +example: `Quebec` + +| core // =============================================================== | client.domain | Client domain. -| keyword -(core) +type: keyword + + + +| core // =============================================================== | client.user.hash | Unique user hash to correlate information for a user in anonymized form. -| keyword +Useful if `user.id` or `user.name` contain confidential information and cannot be used. + +type: keyword -(extended) + + +| extended // =============================================================== | client.geo.city_name | City name. -| keyword -(core) +type: keyword + +example: `Montreal` + +| core // =============================================================== | client.bytes | Bytes sent from the client to the server. -| long -(core) +type: long + +example: `184` + +| core // =============================================================== | client.geo.country_iso_code | Country ISO code. -| keyword -(core) +type: keyword + +example: `CA` + +| core // =============================================================== | client.packets | Packets sent from the client to the server. -| long -(core) +type: long + +example: `12` + +| core // =============================================================== | client.geo.region_iso_code | Region ISO code. -| keyword -(core) +type: keyword + +example: `CA-QC` + +| core // =============================================================== | client.geo.name -| User-defined description of a location. -| keyword +| User-defined description of a location, at the level of granularity they care about. +Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. +Not typically used in automated geolocation. + +type: keyword -(extended) +example: `boston-dc` + +| extended // =============================================================== |===== -[[ecs-client-details]] -==== Field details +[[ecs-cloud]] +=== Cloud fields -===== client.address +Fields related to the cloud or infrastructure the events are coming from. -Level: extended +==== Fields summary -Datatype: keyword +[options="header"] +|===== +| Field | Description | Level -Some event client addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. -Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. +// =============================================================== +| cloud.provider +| Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. +type: keyword -===== client.geo.location +example: `aws` -Level: core +| extended -Datatype: geo_point +// =============================================================== -Longitude and latitude. +| cloud.availability_zone +| Availability zone in which this host is running. -example: `{ "lon": -73.614830, "lat": 45.505918 }` +type: keyword -===== client.user.id +example: `us-east-1c` -Level: core +| extended -Datatype: keyword +// =============================================================== -One or multiple unique identifiers of the user. +| cloud.region +| Region in which this host is running. +type: keyword +example: `us-east-1` -===== client.user.group.id +| extended -Level: extended +// =============================================================== -Datatype: keyword +| cloud.instance.id +| Instance ID of the host machine. -Unique identifier for the group on the system/platform. +type: keyword +example: `i-1234567890abcdef0` +| extended -===== client.ip +// =============================================================== -Level: core +| cloud.instance.name +| Instance name of the host machine. -Datatype: ip +type: keyword -IP address of the client. -Can be one or multiple IPv4 or IPv6 addresses. +| extended -===== client.user.group.name +// =============================================================== -Level: extended +| cloud.machine.type +| Machine type of the host machine. -Datatype: keyword +type: keyword -Name of the group. +example: `t2.medium` +| extended +// =============================================================== -===== client.geo.continent_name +| cloud.account.id +| The cloud account or organization id used to identify different entities in a multi-tenant environment. +Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. -Level: core +type: keyword -Datatype: keyword +example: `666777888999` -Name of the continent. +| extended -example: `North America` +// =============================================================== -===== client.user.name +|===== -Level: core +[[ecs-container]] +=== Container fields -Datatype: keyword +Container fields are used for meta information about the specific container that is the source of information. +These fields help correlate data based containers from any runtime. -Short name or login of the user. +==== Fields summary -example: `albert` +[options="header"] +|===== +| Field | Description | Level -===== client.port +// =============================================================== -Level: core +| container.runtime +| Runtime managing this container. -Datatype: long +type: keyword -Port of the client. +example: `docker` +| extended +// =============================================================== -===== client.geo.country_name +| container.id +| Unique container id. -Level: core +type: keyword -Datatype: keyword -Country name. -example: `Canada` +| core -===== client.user.full_name +// =============================================================== -Level: extended +| container.image.name +| Name of the image the container was built on. -Datatype: keyword +type: keyword -User's full name, if available. -example: `Albert Einstein` -===== client.mac +| extended -Level: core +// =============================================================== -Datatype: keyword +| container.image.tag +| Container image tag. -MAC address of the client. +type: keyword -===== client.user.email +| extended -Level: extended +// =============================================================== -Datatype: keyword +| container.name +| Container name. -User email address. +type: keyword -===== client.geo.region_name +| extended -Level: core +// =============================================================== -Datatype: keyword +| container.labels +| Image labels. -Region name. +type: object -example: `Quebec` -===== client.domain -Level: core +| extended -Datatype: keyword +// =============================================================== -Client domain. +|===== +[[ecs-destination]] +=== Destination fields +Destination fields describe details about the destination of a packet/event. +Destination fields are usually populated in conjunction with source fields. -===== client.user.hash +==== Fields summary -Level: extended +[options="header"] +|===== +| Field | Description | Level -Datatype: keyword +// =============================================================== -Unique user hash to correlate information for a user in anonymized form. -Useful if `user.id` or `user.name` contain confidential information and cannot be used. +| destination.address +| Some event destination addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. +Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. +type: keyword -===== client.geo.city_name -Level: core +| extended -Datatype: keyword +// =============================================================== -City name. +| destination.geo.location +| Longitude and latitude. -example: `Montreal` +type: geo_point -===== client.bytes +example: `{ "lon": -73.614830, "lat": 45.505918 }` -Level: core +| core -Datatype: long - -Bytes sent from the client to the server. +// =============================================================== -example: `184` +| destination.user.id +| One or multiple unique identifiers of the user. -===== client.geo.country_iso_code +type: keyword -Level: core -Datatype: keyword -Country ISO code. +| core -example: `CA` +// =============================================================== -===== client.packets +| destination.user.group.id +| Unique identifier for the group on the system/platform. -Level: core +type: keyword -Datatype: long -Packets sent from the client to the server. -example: `12` +| extended -===== client.geo.region_iso_code +// =============================================================== -Level: core +| destination.ip +| IP address of the destination. +Can be one or multiple IPv4 or IPv6 addresses. -Datatype: keyword +type: ip -Region ISO code. -example: `CA-QC` -===== client.geo.name +| core -Level: extended +// =============================================================== -Datatype: keyword +| destination.user.group.name +| Name of the group. -User-defined description of a location, at the level of granularity they care about. -Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. -Not typically used in automated geolocation. +type: keyword -example: `boston-dc` -[[ecs-cloud]] -=== Cloud fields -Fields related to the cloud or infrastructure the events are coming from. +| extended -==== Fields summary +// =============================================================== -[options="header"] -|===== -| Field | Description | Type (Level) +| destination.geo.continent_name +| Name of the continent. -// =============================================================== +type: keyword -| cloud.provider -| Name of the cloud provider. -| keyword +example: `North America` -(extended) +| core // =============================================================== -| cloud.availability_zone -| Availability zone in which this host is running. -| keyword - -(extended) +| destination.user.name +| Short name or login of the user. -// =============================================================== +type: keyword -| cloud.region -| Region in which this host is running. -| keyword +example: `albert` -(extended) +| core // =============================================================== -| cloud.instance.id -| Instance ID of the host machine. -| keyword +| destination.port +| Port of the destination. -(extended) +type: long -// =============================================================== -| cloud.instance.name -| Instance name of the host machine. -| keyword -(extended) +| core // =============================================================== -| cloud.machine.type -| Machine type of the host machine. -| keyword - -(extended) +| destination.geo.country_name +| Country name. -// =============================================================== +type: keyword -| cloud.account.id -| The cloud account or organization id. -| keyword +example: `Canada` -(extended) +| core // =============================================================== -|===== +| destination.user.full_name +| User's full name, if available. -[[ecs-cloud-details]] -==== Field details +type: keyword -===== cloud.provider +example: `Albert Einstein` + +| extended -Level: extended +// =============================================================== -Datatype: keyword +| destination.mac +| MAC address of the destination. -Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. +type: keyword -example: `aws` -===== cloud.availability_zone -Level: extended +| core -Datatype: keyword +// =============================================================== -Availability zone in which this host is running. +| destination.user.email +| User email address. -example: `us-east-1c` +type: keyword -===== cloud.region -Level: extended -Datatype: keyword +| extended -Region in which this host is running. +// =============================================================== -example: `us-east-1` +| destination.geo.region_name +| Region name. -===== cloud.instance.id +type: keyword -Level: extended +example: `Quebec` -Datatype: keyword +| core -Instance ID of the host machine. +// =============================================================== -example: `i-1234567890abcdef0` +| destination.domain +| Destination domain. -===== cloud.instance.name +type: keyword -Level: extended -Datatype: keyword -Instance name of the host machine. +| core +// =============================================================== +| destination.user.hash +| Unique user hash to correlate information for a user in anonymized form. +Useful if `user.id` or `user.name` contain confidential information and cannot be used. -===== cloud.machine.type +type: keyword -Level: extended -Datatype: keyword -Machine type of the host machine. +| extended -example: `t2.medium` +// =============================================================== -===== cloud.account.id +| destination.geo.city_name +| City name. -Level: extended +type: keyword -Datatype: keyword +example: `Montreal` -The cloud account or organization id used to identify different entities in a multi-tenant environment. -Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. +| core -example: `666777888999` +// =============================================================== -[[ecs-container]] -=== Container fields +| destination.bytes +| Bytes sent from the destination to the source. -Container fields are used for meta information about the specific container that is the source of information. -These fields help correlate data based containers from any runtime. +type: long -==== Fields summary +example: `184` -[options="header"] -|===== -| Field | Description | Type (Level) +| core // =============================================================== -| container.runtime -| Runtime managing this container. -| keyword - -(extended) +| destination.geo.country_iso_code +| Country ISO code. -// =============================================================== +type: keyword -| container.id -| Unique container id. -| keyword +example: `CA` -(core) +| core // =============================================================== -| container.image.name -| Name of the image the container was built on. -| keyword - -(extended) +| destination.packets +| Packets sent from the destination to the source. -// =============================================================== +type: long -| container.image.tag -| Container image tag. -| keyword +example: `12` -(extended) +| core // =============================================================== -| container.name -| Container name. -| keyword +| destination.geo.region_iso_code +| Region ISO code. + +type: keyword -(extended) +example: `CA-QC` + +| core // =============================================================== -| container.labels -| Image labels. -| object +| destination.geo.name +| User-defined description of a location, at the level of granularity they care about. +Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. +Not typically used in automated geolocation. + +type: keyword -(extended) +example: `boston-dc` + +| extended // =============================================================== |===== -[[ecs-container-details]] -==== Field details - -===== container.runtime +[[ecs-ecs]] +=== ECS fields -Level: extended +Meta-information specific to ECS. -Datatype: keyword +==== Fields summary -Runtime managing this container. +[options="header"] +|===== +| Field | Description | Level -example: `docker` +// =============================================================== -===== container.id +| ecs.version +| ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. +When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. -Level: core +type: keyword -Datatype: keyword +example: `1.0.0` -Unique container id. +| core +// =============================================================== +|===== -===== container.image.name +[[ecs-error]] +=== Error fields -Level: extended +These fields can represent errors of any kind. +Use them for errors that happen while fetching events or in cases where the event itself contains an error. -Datatype: keyword +==== Fields summary -Name of the image the container was built on. +[options="header"] +|===== +| Field | Description | Level +// =============================================================== +| error.id +| Unique identifier for the error. -===== container.image.tag +type: keyword -Level: extended -Datatype: keyword -Container image tag. +| core +// =============================================================== +| error.message +| Error message. -===== container.name +type: text -Level: extended -Datatype: keyword -Container name. +| core +// =============================================================== +| error.code +| Error code describing the error. -===== container.labels +type: keyword -Level: extended -Datatype: object -Image labels. +| core +// =============================================================== +|===== -[[ecs-destination]] -=== Destination fields +[[ecs-event]] +=== Event fields -Destination fields describe details about the destination of a packet/event. -Destination fields are usually populated in conjunction with source fields. +The event fields are used for context information about the log or metric event itself. +A log is defined as an event containing details of something that happened. Log events must include the time at which the thing happened. Examples of log events include a process starting on a host, a network packet being sent from a source to a destination, or a network connection between a client and a server being initiated or closed. A metric is defined as an event containing one or more numerical or categorical measurements and the time at which the measurement was taken. Examples of metric events include memory pressure measured on a host, or vulnerabilities measured on a scanned host. ==== Fields summary [options="header"] |===== -| Field | Description | Type (Level) +| Field | Description | Level // =============================================================== -| destination.address -| Destination network address. -| keyword - -(extended) +| event.id +| Unique ID to describe the event. -// =============================================================== +type: keyword -| destination.geo.location -| Longitude and latitude. -| geo_point +example: `8a4f500d` -(core) +| core // =============================================================== -| destination.user.id -| One or multiple unique identifiers of the user. -| keyword - -(core) +| event.kind +| The kind of the event. +This gives information about what type of information the event contains, without being specific to the contents of the event. Examples are `event`, `state`, `alarm`. Warning: In future versions of ECS, we plan to provide a list of acceptable values for this field, please use with caution. -// =============================================================== +type: keyword -| destination.user.group.id -| Unique identifier for the group on the system/platform. -| keyword +example: `state` -(extended) +| extended // =============================================================== -| destination.ip -| IP address of the destination. -| ip - -(core) +| event.category +| Event category. +This contains high-level information about the contents of the event. It is more generic than `event.action`, in the sense that typically a category contains multiple actions. Warning: In future versions of ECS, we plan to provide a list of acceptable values for this field, please use with caution. -// =============================================================== +type: keyword -| destination.user.group.name -| Name of the group. -| keyword +example: `user-management` -(extended) +| core // =============================================================== -| destination.geo.continent_name -| Name of the continent. -| keyword - -(core) +| event.action +| The action captured by the event. +This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. -// =============================================================== +type: keyword -| destination.user.name -| Short name or login of the user. -| keyword +example: `user-password-change` -(core) +| core // =============================================================== -| destination.port -| Port of the destination. -| long - -(core) +| event.outcome +| The outcome of the event. +If the event describes an action, this fields contains the outcome of that action. Examples outcomes are `success` and `failure`. Warning: In future versions of ECS, we plan to provide a list of acceptable values for this field, please use with caution. -// =============================================================== +type: keyword -| destination.geo.country_name -| Country name. -| keyword +example: `success` -(core) +| extended // =============================================================== -| destination.user.full_name -| User's full name, if available. -| keyword +| event.type +| Reserved for future usage. +Please avoid using this field for user data. -(extended) +type: keyword -// =============================================================== -| destination.mac -| MAC address of the destination. -| keyword -(core) +| core // =============================================================== -| destination.user.email -| User email address. -| keyword - -(extended) +| event.module +| Name of the module this data is coming from. +This information is coming from the modules used in Beats or Logstash. -// =============================================================== +type: keyword -| destination.geo.region_name -| Region name. -| keyword +example: `mysql` -(core) +| core // =============================================================== -| destination.domain -| Destination domain. -| keyword - -(core) +| event.dataset +| Name of the dataset. +The concept of a `dataset` (fileset / metricset) is used in Beats as a subset of modules. It contains the information which is currently stored in metricset.name and metricset.module or fileset.name. -// =============================================================== +type: keyword -| destination.user.hash -| Unique user hash to correlate information for a user in anonymized form. -| keyword +example: `stats` -(extended) +| core // =============================================================== -| destination.geo.city_name -| City name. -| keyword +| event.severity +| Severity describes the original severity of the event. What the different severity values mean can very different between use cases. It's up to the implementer to make sure severities are consistent across events. + +type: long + +example: `7` -(core) +| core // =============================================================== -| destination.bytes -| Bytes sent from the destination to the source. -| long +| event.original +| Raw text message of entire event. Used to demonstrate log integrity. +This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. -(core) +type: keyword + +example: `Sep 19 08:26:10 host CEF:0|Security| threatmanager|1.0|100| worm successfully stopped|10|src=10.0.0.1 dst=2.1.2.2spt=1232` + +| core // =============================================================== -| destination.geo.country_iso_code -| Country ISO code. -| keyword +| event.hash +| Hash (perhaps logstash fingerprint) of raw field to be able to demonstrate log integrity. + +type: keyword -(core) +example: `123456789012345678901234567890ABCD` + +| extended // =============================================================== -| destination.packets -| Packets sent from the destination to the source. -| long +| event.duration +| Duration of the event in nanoseconds. +If event.start and event.end are known this value should be the difference between the end and start time. -(core) +type: long -// =============================================================== -| destination.geo.region_iso_code -| Region ISO code. -| keyword -(core) +| core // =============================================================== -| destination.geo.name -| User-defined description of a location. -| keyword +| event.timezone +| This field should be populated when the event's timestamp does not include timezone information already (e.g. default Syslog timestamps). It's optional otherwise. +Acceptable timezone formats are: a canonical ID (e.g. "Europe/Amsterdam"), abbreviated (e.g. "EST") or an HH:mm differential (e.g. "-05:00"). -(extended) +type: keyword -// =============================================================== -|===== -[[ecs-destination-details]] -==== Field details +| extended -===== destination.address +// =============================================================== -Level: extended +| event.created +| event.created contains the date/time when the event was first read by an agent, or by your pipeline. +This field is distinct from @timestamp in that @timestamp typically contain the time extracted from the original event. +In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. +In case the two timestamps are identical, @timestamp should be used. -Datatype: keyword +type: date -Some event destination addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. -Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. +| core -===== destination.geo.location +// =============================================================== -Level: core +| event.start +| event.start contains the date when the event started or when the activity was first observed. -Datatype: geo_point +type: date -Longitude and latitude. -example: `{ "lon": -73.614830, "lat": 45.505918 }` -===== destination.user.id +| extended -Level: core +// =============================================================== -Datatype: keyword +| event.end +| event.end contains the date when the event ended or when the activity was last observed. -One or multiple unique identifiers of the user. +type: date -===== destination.user.group.id +| extended -Level: extended +// =============================================================== -Datatype: keyword +| event.risk_score +| Risk score or priority of the event (e.g. security solutions). Use your system's original value here. -Unique identifier for the group on the system/platform. +type: float -===== destination.ip +| core -Level: core +// =============================================================== -Datatype: ip +| event.risk_score_norm +| Normalized risk score or priority of the event, on a scale of 0 to 100. +This is mainly useful if you use more than one system that assigns risk scores, and you want to see a normalized value across all systems. -IP address of the destination. -Can be one or multiple IPv4 or IPv6 addresses. +type: float -===== destination.user.group.name +| extended -Level: extended +// =============================================================== -Datatype: keyword +|===== -Name of the group. +[[ecs-file]] +=== File fields +A file is defined as a set of information that has been created on, or has existed on a filesystem. +File objects can be associated with host events, network events, and/or file events (e.g., those produced by File Integrity Monitoring [FIM] products or services). File fields provide details about the affected file associated with the event or metric. +==== Fields summary -===== destination.geo.continent_name +[options="header"] +|===== +| Field | Description | Level -Level: core +// =============================================================== -Datatype: keyword +| file.path +| Path to the file. -Name of the continent. +type: keyword -example: `North America` -===== destination.user.name -Level: core +| extended -Datatype: keyword +// =============================================================== -Short name or login of the user. +| file.target_path +| Target path for symlinks. -example: `albert` +type: keyword -===== destination.port -Level: core -Datatype: long +| extended -Port of the destination. +// =============================================================== +| file.extension +| File extension. +This should allow easy filtering by file extensions. +type: keyword -===== destination.geo.country_name +example: `png` -Level: core +| extended -Datatype: keyword +// =============================================================== -Country name. +| file.type +| File type (file, dir, or symlink). -example: `Canada` +type: keyword -===== destination.user.full_name -Level: extended -Datatype: keyword +| extended -User's full name, if available. +// =============================================================== -example: `Albert Einstein` +| file.device +| Device that is the source of the file. -===== destination.mac +type: keyword -Level: core -Datatype: keyword -MAC address of the destination. +| extended +// =============================================================== +| file.inode +| Inode representing the file in the filesystem. -===== destination.user.email +type: keyword -Level: extended -Datatype: keyword -User email address. +| extended +// =============================================================== +| file.uid +| The user ID (UID) or security identifier (SID) of the file owner. -===== destination.geo.region_name +type: keyword -Level: core -Datatype: keyword -Region name. +| extended -example: `Quebec` +// =============================================================== -===== destination.domain +| file.owner +| File owner's username. -Level: core +type: keyword -Datatype: keyword -Destination domain. +| extended +// =============================================================== -===== destination.user.hash +| file.gid +| Primary group ID (GID) of the file. -Level: extended +type: keyword -Datatype: keyword -Unique user hash to correlate information for a user in anonymized form. -Useful if `user.id` or `user.name` contain confidential information and cannot be used. +| extended +// =============================================================== -===== destination.geo.city_name +| file.group +| Primary group name of the file. -Level: core +type: keyword -Datatype: keyword -City name. -example: `Montreal` +| extended -===== destination.bytes +// =============================================================== -Level: core +| file.mode +| Mode of the file in octal representation. -Datatype: long +type: keyword -Bytes sent from the destination to the source. +example: `416` -example: `184` +| extended -===== destination.geo.country_iso_code +// =============================================================== -Level: core +| file.size +| File size in bytes (field is only added when `type` is `file`). -Datatype: keyword +type: long -Country ISO code. -example: `CA` -===== destination.packets +| extended -Level: core +// =============================================================== -Datatype: long +| file.mtime +| Last time file content was modified. -Packets sent from the destination to the source. +type: date -example: `12` -===== destination.geo.region_iso_code -Level: core +| extended -Datatype: keyword +// =============================================================== -Region ISO code. +| file.ctime +| Last time file metadata changed. -example: `CA-QC` +type: date -===== destination.geo.name -Level: extended -Datatype: keyword +| extended -User-defined description of a location, at the level of granularity they care about. -Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. -Not typically used in automated geolocation. +// =============================================================== -example: `boston-dc` +|===== -[[ecs-ecs]] -=== ECS fields +[[ecs-geo]] +=== Geo fields -Meta-information specific to ECS. +Geo fields can carry data about a specific location related to an event. +This geolocation information can be derived from techniques such as Geo IP, or be user-supplied. ==== Fields summary [options="header"] |===== -| Field | Description | Type (Level) +| Field | Description | Level // =============================================================== -| ecs.version -| ECS version this event conforms to. -| keyword +| geo.location +| Longitude and latitude. -(core) +type: geo_point -// =============================================================== +example: `{ "lon": -73.614830, "lat": 45.505918 }` -|===== +| core -[[ecs-ecs-details]] -==== Field details +// =============================================================== -===== ecs.version +| geo.continent_name +| Name of the continent. -Level: core +type: keyword -Datatype: keyword +example: `North America` -ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. -When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. +| core -example: `1.0.0` +// =============================================================== -[[ecs-error]] -=== Error fields +| geo.country_name +| Country name. -These fields can represent errors of any kind. -Use them for errors that happen while fetching events or in cases where the event itself contains an error. +type: keyword -==== Fields summary +example: `Canada` -[options="header"] -|===== -| Field | Description | Type (Level) +| core // =============================================================== -| error.id -| Unique identifier for the error. -| keyword - -(core) +| geo.region_name +| Region name. -// =============================================================== +type: keyword -| error.message -| Error message. -| text +example: `Quebec` -(core) +| core // =============================================================== -| error.code -| Error code describing the error. -| keyword - -(core) - -// =============================================================== +| geo.city_name +| City name. -|===== +type: keyword -[[ecs-error-details]] -==== Field details +example: `Montreal` -===== error.id +| core -Level: core +// =============================================================== -Datatype: keyword +| geo.country_iso_code +| Country ISO code. -Unique identifier for the error. +type: keyword +example: `CA` +| core -===== error.message +// =============================================================== -Level: core +| geo.region_iso_code +| Region ISO code. -Datatype: text +type: keyword -Error message. +example: `CA-QC` +| core +// =============================================================== -===== error.code +| geo.name +| User-defined description of a location, at the level of granularity they care about. +Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. +Not typically used in automated geolocation. -Level: core +type: keyword -Datatype: keyword +example: `boston-dc` -Error code describing the error. +| extended +// =============================================================== +|===== -[[ecs-event]] -=== Event fields +[[ecs-group]] +=== Group fields -The event fields are used for context information about the log or metric event itself. -A log is defined as an event containing details of something that happened. Log events must include the time at which the thing happened. Examples of log events include a process starting on a host, a network packet being sent from a source to a destination, or a network connection between a client and a server being initiated or closed. A metric is defined as an event containing one or more numerical or categorical measurements and the time at which the measurement was taken. Examples of metric events include memory pressure measured on a host, or vulnerabilities measured on a scanned host. +The group fields are meant to represent groups that are relevant to the event. ==== Fields summary [options="header"] |===== -| Field | Description | Type (Level) +| Field | Description | Level // =============================================================== -| event.id -| Unique ID to describe the event. -| keyword +| group.id +| Unique identifier for the group on the system/platform. -(core) +type: keyword -// =============================================================== -| event.kind -| The kind of the event. -| keyword -(extended) +| extended // =============================================================== -| event.category -| Event category. -| keyword +| group.name +| Name of the group. -(core) +type: keyword -// =============================================================== -| event.action -| The action captured by the event. -| keyword -(core) +| extended // =============================================================== -| event.outcome -| The outcome of the event. -| keyword +|===== -(extended) +[[ecs-host]] +=== Host fields -// =============================================================== +A host is defined as a general computing instance. +ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes. -| event.type -| Reserved for future usage. -| keyword +==== Fields summary -(core) +[options="header"] +|===== +| Field | Description | Level // =============================================================== -| event.module -| Name of the module this data is coming from. -| keyword +| host.hostname +| Hostname of the host. +It normally contains what the `hostname` command returns on the host machine. -(core) +type: keyword -// =============================================================== -| event.dataset -| Name of the dataset. -| keyword -(core) +| core // =============================================================== -| event.severity -| Original severity of the event. -| long - -(core) +| host.geo.location +| Longitude and latitude. -// =============================================================== +type: geo_point -| event.original -| Raw text message of entire event. -| keyword +example: `{ "lon": -73.614830, "lat": 45.505918 }` -(core) +| core // =============================================================== -| event.hash -| Hash (perhaps logstash fingerprint) of raw field to be able to demonstrate log integrity. -| keyword +| host.user.id +| One or multiple unique identifiers of the user. -(extended) +type: keyword -// =============================================================== -| event.duration -| Duration of the event in nanoseconds. -| long -(core) +| core // =============================================================== -| event.timezone -| Event time zone. -| keyword +| host.user.group.id +| Unique identifier for the group on the system/platform. -(extended) +type: keyword -// =============================================================== -| event.created -| Time when the event was first read by an agent or by your pipeline. -| date -(core) +| extended // =============================================================== -| event.start -| event.start contains the date when the event started or when the activity was first observed. -| date +| host.os.platform +| Operating system platform (such centos, ubuntu, windows). + +type: keyword + +example: `darwin` -(extended) +| extended // =============================================================== -| event.end -| event.end contains the date when the event ended or when the activity was last observed. -| date +| host.name +| Name of the host. +It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. -(extended) +type: keyword -// =============================================================== -| event.risk_score -| Risk score or priority of the event (e.g. security solutions). Use your system's original value here. -| float -(core) +| core // =============================================================== -| event.risk_score_norm -| Normalized risk score or priority of the event (0-100). -| float - -(extended) +| host.user.group.name +| Name of the group. -// =============================================================== +type: keyword -|===== -[[ecs-event-details]] -==== Field details -===== event.id +| extended -Level: core +// =============================================================== -Datatype: keyword +| host.geo.continent_name +| Name of the continent. -Unique ID to describe the event. +type: keyword -example: `8a4f500d` +example: `North America` -===== event.kind +| core -Level: extended +// =============================================================== -Datatype: keyword +| host.os.name +| Operating system name, without the version. -The kind of the event. -This gives information about what type of information the event contains, without being specific to the contents of the event. Examples are `event`, `state`, `alarm`. Warning: In future versions of ECS, we plan to provide a list of acceptable values for this field, please use with caution. +type: keyword -example: `state` +example: `Mac OS X` -===== event.category +| extended -Level: core +// =============================================================== -Datatype: keyword +| host.user.name +| Short name or login of the user. -Event category. -This contains high-level information about the contents of the event. It is more generic than `event.action`, in the sense that typically a category contains multiple actions. Warning: In future versions of ECS, we plan to provide a list of acceptable values for this field, please use with caution. +type: keyword -example: `user-management` +example: `albert` -===== event.action +| core -Level: core +// =============================================================== -Datatype: keyword +| host.id +| Unique host id. +As hostname is not always unique, use values that are meaningful in your environment. +Example: The current usage of `beat.name`. -The action captured by the event. -This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. +type: keyword -example: `user-password-change` -===== event.outcome -Level: extended +| core -Datatype: keyword +// =============================================================== -The outcome of the event. -If the event describes an action, this fields contains the outcome of that action. Examples outcomes are `success` and `failure`. Warning: In future versions of ECS, we plan to provide a list of acceptable values for this field, please use with caution. +| host.geo.country_name +| Country name. -example: `success` +type: keyword -===== event.type +example: `Canada` -Level: core +| core -Datatype: keyword +// =============================================================== -Reserved for future usage. -Please avoid using this field for user data. +| host.os.full +| Operating system name, including the version or code name. +type: keyword +example: `Mac OS Mojave` -===== event.module +| extended -Level: core +// =============================================================== -Datatype: keyword +| host.user.full_name +| User's full name, if available. -Name of the module this data is coming from. -This information is coming from the modules used in Beats or Logstash. +type: keyword -example: `mysql` +example: `Albert Einstein` -===== event.dataset +| extended -Level: core +// =============================================================== -Datatype: keyword +| host.ip +| Host ip address. -Name of the dataset. -The concept of a `dataset` (fileset / metricset) is used in Beats as a subset of modules. It contains the information which is currently stored in metricset.name and metricset.module or fileset.name. +type: ip -example: `stats` -===== event.severity -Level: core +| core -Datatype: long +// =============================================================== -Severity describes the original severity of the event. What the different severity values mean can very different between use cases. It's up to the implementer to make sure severities are consistent across events. +| host.user.email +| User email address. -example: `7` +type: keyword -===== event.original -Level: core -Datatype: keyword +| extended -Raw text message of entire event. Used to demonstrate log integrity. -This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. +// =============================================================== -example: `Sep 19 08:26:10 host CEF:0|Security| threatmanager|1.0|100| worm successfully stopped|10|src=10.0.0.1 dst=2.1.2.2spt=1232` +| host.os.family +| OS family (such as redhat, debian, freebsd, windows). -===== event.hash +type: keyword -Level: extended +example: `debian` -Datatype: keyword +| extended -Hash (perhaps logstash fingerprint) of raw field to be able to demonstrate log integrity. +// =============================================================== -example: `123456789012345678901234567890ABCD` +| host.geo.region_name +| Region name. -===== event.duration +type: keyword -Level: core +example: `Quebec` -Datatype: long +| core -Duration of the event in nanoseconds. -If event.start and event.end are known this value should be the difference between the end and start time. +// =============================================================== +| host.mac +| Host mac address. +type: keyword -===== event.timezone -Level: extended -Datatype: keyword +| core -This field should be populated when the event's timestamp does not include timezone information already (e.g. default Syslog timestamps). It's optional otherwise. -Acceptable timezone formats are: a canonical ID (e.g. "Europe/Amsterdam"), abbreviated (e.g. "EST") or an HH:mm differential (e.g. "-05:00"). +// =============================================================== +| host.user.hash +| Unique user hash to correlate information for a user in anonymized form. +Useful if `user.id` or `user.name` contain confidential information and cannot be used. +type: keyword -===== event.created -Level: core -Datatype: date +| extended -event.created contains the date/time when the event was first read by an agent, or by your pipeline. -This field is distinct from @timestamp in that @timestamp typically contain the time extracted from the original event. -In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. -In case the two timestamps are identical, @timestamp should be used. +// =============================================================== +| host.geo.city_name +| City name. +type: keyword -===== event.start +example: `Montreal` -Level: extended +| core -Datatype: date +// =============================================================== -event.start contains the date when the event started or when the activity was first observed. +| host.os.version +| Operating system version as a raw string. +type: keyword +example: `10.14.1` -===== event.end +| extended -Level: extended +// =============================================================== -Datatype: date +| host.type +| Type of host. +For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. -event.end contains the date when the event ended or when the activity was last observed. +type: keyword -===== event.risk_score +| core -Level: core +// =============================================================== -Datatype: float +| host.geo.country_iso_code +| Country ISO code. -Risk score or priority of the event (e.g. security solutions). Use your system's original value here. +type: keyword +example: `CA` +| core -===== event.risk_score_norm +// =============================================================== -Level: extended +| host.os.kernel +| Operating system kernel version as a raw string. -Datatype: float +type: keyword -Normalized risk score or priority of the event, on a scale of 0 to 100. -This is mainly useful if you use more than one system that assigns risk scores, and you want to see a normalized value across all systems. +example: `4.4.0-112-generic` +| extended +// =============================================================== -[[ecs-file]] -=== File fields +| host.architecture +| Operating system architecture. -A file is defined as a set of information that has been created on, or has existed on a filesystem. -File objects can be associated with host events, network events, and/or file events (e.g., those produced by File Integrity Monitoring [FIM] products or services). File fields provide details about the affected file associated with the event or metric. +type: keyword -==== Fields summary +example: `x86_64` -[options="header"] -|===== -| Field | Description | Type (Level) +| core // =============================================================== -| file.path -| Path to the file. -| keyword - -(extended) +| host.geo.region_iso_code +| Region ISO code. -// =============================================================== +type: keyword -| file.target_path -| Target path for symlinks. -| keyword +example: `CA-QC` -(extended) +| core // =============================================================== -| file.extension -| File extension. -| keyword - -(extended) +| host.geo.name +| User-defined description of a location, at the level of granularity they care about. +Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. +Not typically used in automated geolocation. -// =============================================================== +type: keyword -| file.type -| File type (file, dir, or symlink). -| keyword +example: `boston-dc` -(extended) +| extended // =============================================================== -| file.device -| Device that is the source of the file. -| keyword +|===== -(extended) +[[ecs-http]] +=== HTTP fields -// =============================================================== +Fields related to HTTP activity. Use the `url` field set to store the url of the request. -| file.inode -| Inode representing the file in the filesystem. -| keyword +==== Fields summary -(extended) +[options="header"] +|===== +| Field | Description | Level // =============================================================== -| file.uid -| The user ID (UID) or security identifier (SID) of the file owner. -| keyword - -(extended) +| http.request.method +| HTTP request method. +The field value must be normalized to lowercase for querying. See the documentation section "Implementing ECS". -// =============================================================== +type: keyword -| file.owner -| File owner's username. -| keyword +example: `get, post, put` -(extended) +| extended // =============================================================== -| file.gid -| Primary group ID (GID) of the file. -| keyword - -(extended) +| http.request.body.content +| The full HTTP request body. -// =============================================================== +type: keyword -| file.group -| Primary group name of the file. -| keyword +example: `Hello world` -(extended) +| extended // =============================================================== -| file.mode -| Mode of the file in octal representation. -| keyword - -(extended) +| http.request.referrer +| Referrer for this HTTP request. -// =============================================================== +type: keyword -| file.size -| File size in bytes (field is only added when `type` is `file`). -| long +example: `https://blog.example.com/` -(extended) +| extended // =============================================================== -| file.mtime -| Last time file content was modified. -| date - -(extended) +| http.response.status_code +| HTTP response status code. -// =============================================================== +type: long -| file.ctime -| Last time file metadata changed. -| date +example: `404` -(extended) +| extended // =============================================================== -|===== +| http.response.body.content +| The full HTTP response body. -[[ecs-file-details]] -==== Field details +type: keyword -===== file.path +example: `Hello world` -Level: extended +| extended -Datatype: keyword +// =============================================================== -Path to the file. +| http.version +| HTTP version. +type: keyword +example: `1.1` -===== file.target_path +| extended -Level: extended +// =============================================================== -Datatype: keyword +| http.request.bytes +| Total size in bytes of the request (body and headers). -Target path for symlinks. +type: long +example: `1437` +| extended -===== file.extension +// =============================================================== -Level: extended +| http.request.body.bytes +| Size in bytes of the request body. -Datatype: keyword +type: long -File extension. -This should allow easy filtering by file extensions. +example: `887` -example: `png` +| extended -===== file.type +// =============================================================== -Level: extended +| http.response.bytes +| Total size in bytes of the response (body and headers). -Datatype: keyword +type: long -File type (file, dir, or symlink). +example: `1437` +| extended +// =============================================================== -===== file.device +| http.response.body.bytes +| Size in bytes of the response body. -Level: extended +type: long -Datatype: keyword +example: `887` -Device that is the source of the file. +| extended +// =============================================================== +|===== -===== file.inode +[[ecs-log]] +=== Log fields -Level: extended +Fields which are specific to log events. -Datatype: keyword +==== Fields summary -Inode representing the file in the filesystem. +[options="header"] +|===== +| Field | Description | Level +// =============================================================== +| log.level +| Original log level of the log event. +Some examples are `warn`, `error`, `i`. -===== file.uid +type: keyword -Level: extended +example: `err` -Datatype: keyword +| core -The user ID (UID) or security identifier (SID) of the file owner. +// =============================================================== +| log.original +| This is the original log message and contains the full log message before splitting it up in multiple parts. +In contrast to the `message` field which can contain an extracted part of the log message, this field contains the original, full log message. It can have already some modifications applied like encoding or new lines removed to clean up the log message. +This field is not indexed and doc_values are disabled so it can't be queried but the value can be retrieved from `_source`. +type: keyword -===== file.owner +example: `Sep 19 08:26:10 localhost My log` -Level: extended +| core -Datatype: keyword +// =============================================================== -File owner's username. +|===== +[[ecs-network]] +=== Network fields +The network is defined as the communication path over which a host or network event happens. +The network.* fields should be populated with details about the network activity associated with an event. -===== file.gid +==== Fields summary -Level: extended +[options="header"] +|===== +| Field | Description | Level -Datatype: keyword +// =============================================================== -Primary group ID (GID) of the file. +| network.name +| Name given by operators to sections of their network. +type: keyword +example: `Guest Wifi` -===== file.group +| extended -Level: extended +// =============================================================== -Datatype: keyword +| network.type +| In the OSI Model this would be the Network Layer. ipv4, ipv6, ipsec, pim, etc +The field value must be normalized to lowercase for querying. See the documentation section "Implementing ECS". -Primary group name of the file. +type: keyword +example: `ipv4` +| core -===== file.mode +// =============================================================== -Level: extended +| network.iana_number +| IANA Protocol Number (https://www.iana.org/assignments/protocol-numbers/protocol-numbers.xhtml). Standardized list of protocols. This aligns well with NetFlow and sFlow related logs which use the IANA Protocol Number. -Datatype: keyword +type: keyword -Mode of the file in octal representation. +example: `6` -example: `416` +| extended -===== file.size +// =============================================================== -Level: extended +| network.transport +| Same as network.iana_number, but instead using the Keyword name of the transport layer (udp, tcp, ipv6-icmp, etc.) +The field value must be normalized to lowercase for querying. See the documentation section "Implementing ECS". -Datatype: long +type: keyword -File size in bytes (field is only added when `type` is `file`). +example: `tcp` +| core +// =============================================================== -===== file.mtime +| network.application +| A name given to an application level protocol. This can be arbitrarily assigned for things like microservices, but also apply to things like skype, icq, facebook, twitter. This would be used in situations where the vendor or service can be decoded such as from the source/dest IP owners, ports, or wire format. +The field value must be normalized to lowercase for querying. See the documentation section "Implementing ECS". -Level: extended +type: keyword -Datatype: date +example: `aim` -Last time file content was modified. +| extended +// =============================================================== +| network.protocol +| L7 Network protocol name. ex. http, lumberjack, transport protocol. +The field value must be normalized to lowercase for querying. See the documentation section "Implementing ECS". -===== file.ctime +type: keyword -Level: extended +example: `http` -Datatype: date - -Last time file metadata changed. - - - -[[ecs-geo]] -=== Geo fields - -Geo fields can carry data about a specific location related to an event. -This geolocation information can be derived from techniques such as Geo IP, or be user-supplied. - -==== Fields summary - -[options="header"] -|===== -| Field | Description | Type (Level) - -// =============================================================== - -| geo.location -| Longitude and latitude. -| geo_point - -(core) - -// =============================================================== - -| geo.continent_name -| Name of the continent. -| keyword - -(core) - -// =============================================================== - -| geo.country_name -| Country name. -| keyword - -(core) - -// =============================================================== - -| geo.region_name -| Region name. -| keyword - -(core) - -// =============================================================== - -| geo.city_name -| City name. -| keyword - -(core) - -// =============================================================== - -| geo.country_iso_code -| Country ISO code. -| keyword - -(core) - -// =============================================================== - -| geo.region_iso_code -| Region ISO code. -| keyword - -(core) - -// =============================================================== - -| geo.name -| User-defined description of a location. -| keyword - -(extended) - -// =============================================================== - -|===== - -[[ecs-geo-details]] -==== Field details - -===== geo.location - -Level: core - -Datatype: geo_point - -Longitude and latitude. - -example: `{ "lon": -73.614830, "lat": 45.505918 }` - -===== geo.continent_name - -Level: core - -Datatype: keyword - -Name of the continent. - -example: `North America` - -===== geo.country_name - -Level: core - -Datatype: keyword - -Country name. - -example: `Canada` - -===== geo.region_name - -Level: core - -Datatype: keyword - -Region name. - -example: `Quebec` - -===== geo.city_name - -Level: core - -Datatype: keyword - -City name. - -example: `Montreal` - -===== geo.country_iso_code - -Level: core - -Datatype: keyword - -Country ISO code. - -example: `CA` - -===== geo.region_iso_code - -Level: core - -Datatype: keyword - -Region ISO code. - -example: `CA-QC` - -===== geo.name - -Level: extended - -Datatype: keyword - -User-defined description of a location, at the level of granularity they care about. -Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. -Not typically used in automated geolocation. - -example: `boston-dc` - -[[ecs-group]] -=== Group fields - -The group fields are meant to represent groups that are relevant to the event. - -==== Fields summary - -[options="header"] -|===== -| Field | Description | Type (Level) - -// =============================================================== - -| group.id -| Unique identifier for the group on the system/platform. -| keyword - -(extended) - -// =============================================================== - -| group.name -| Name of the group. -| keyword - -(extended) - -// =============================================================== - -|===== - -[[ecs-group-details]] -==== Field details - -===== group.id - -Level: extended - -Datatype: keyword - -Unique identifier for the group on the system/platform. - - - -===== group.name - -Level: extended - -Datatype: keyword - -Name of the group. - - - -[[ecs-host]] -=== Host fields - -A host is defined as a general computing instance. -ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes. - -==== Fields summary - -[options="header"] -|===== -| Field | Description | Type (Level) - -// =============================================================== - -| host.hostname -| Hostname of the host. -| keyword - -(core) - -// =============================================================== - -| host.geo.location -| Longitude and latitude. -| geo_point - -(core) - -// =============================================================== - -| host.user.id -| One or multiple unique identifiers of the user. -| keyword - -(core) - -// =============================================================== - -| host.user.group.id -| Unique identifier for the group on the system/platform. -| keyword - -(extended) - -// =============================================================== - -| host.os.platform -| Operating system platform (such centos, ubuntu, windows). -| keyword - -(extended) - -// =============================================================== - -| host.name -| Name of the host. -| keyword - -(core) - -// =============================================================== - -| host.user.group.name -| Name of the group. -| keyword - -(extended) - -// =============================================================== - -| host.geo.continent_name -| Name of the continent. -| keyword - -(core) - -// =============================================================== - -| host.os.name -| Operating system name, without the version. -| keyword - -(extended) - -// =============================================================== - -| host.user.name -| Short name or login of the user. -| keyword - -(core) - -// =============================================================== - -| host.id -| Unique host id. -| keyword - -(core) - -// =============================================================== - -| host.geo.country_name -| Country name. -| keyword - -(core) - -// =============================================================== - -| host.os.full -| Operating system name, including the version or code name. -| keyword - -(extended) - -// =============================================================== - -| host.user.full_name -| User's full name, if available. -| keyword - -(extended) - -// =============================================================== - -| host.ip -| Host ip address. -| ip - -(core) - -// =============================================================== - -| host.user.email -| User email address. -| keyword - -(extended) - -// =============================================================== - -| host.os.family -| OS family (such as redhat, debian, freebsd, windows). -| keyword - -(extended) - -// =============================================================== - -| host.geo.region_name -| Region name. -| keyword - -(core) - -// =============================================================== - -| host.mac -| Host mac address. -| keyword - -(core) - -// =============================================================== - -| host.user.hash -| Unique user hash to correlate information for a user in anonymized form. -| keyword - -(extended) - -// =============================================================== - -| host.geo.city_name -| City name. -| keyword - -(core) - -// =============================================================== - -| host.os.version -| Operating system version as a raw string. -| keyword - -(extended) - -// =============================================================== - -| host.type -| Type of host. -| keyword - -(core) - -// =============================================================== - -| host.geo.country_iso_code -| Country ISO code. -| keyword - -(core) - -// =============================================================== - -| host.os.kernel -| Operating system kernel version as a raw string. -| keyword - -(extended) - -// =============================================================== - -| host.architecture -| Operating system architecture. -| keyword - -(core) - -// =============================================================== - -| host.geo.region_iso_code -| Region ISO code. -| keyword - -(core) - -// =============================================================== - -| host.geo.name -| User-defined description of a location. -| keyword - -(extended) - -// =============================================================== - -|===== - -[[ecs-host-details]] -==== Field details - -===== host.hostname - -Level: core - -Datatype: keyword - -Hostname of the host. -It normally contains what the `hostname` command returns on the host machine. - - - -===== host.geo.location - -Level: core - -Datatype: geo_point - -Longitude and latitude. - -example: `{ "lon": -73.614830, "lat": 45.505918 }` - -===== host.user.id - -Level: core - -Datatype: keyword - -One or multiple unique identifiers of the user. - - - -===== host.user.group.id - -Level: extended - -Datatype: keyword - -Unique identifier for the group on the system/platform. - - - -===== host.os.platform - -Level: extended - -Datatype: keyword - -Operating system platform (such centos, ubuntu, windows). - -example: `darwin` - -===== host.name - -Level: core - -Datatype: keyword - -Name of the host. -It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. - - - -===== host.user.group.name - -Level: extended - -Datatype: keyword - -Name of the group. - - - -===== host.geo.continent_name - -Level: core - -Datatype: keyword - -Name of the continent. - -example: `North America` - -===== host.os.name - -Level: extended - -Datatype: keyword - -Operating system name, without the version. - -example: `Mac OS X` - -===== host.user.name - -Level: core - -Datatype: keyword - -Short name or login of the user. - -example: `albert` - -===== host.id - -Level: core - -Datatype: keyword - -Unique host id. -As hostname is not always unique, use values that are meaningful in your environment. -Example: The current usage of `beat.name`. - - - -===== host.geo.country_name - -Level: core - -Datatype: keyword - -Country name. - -example: `Canada` - -===== host.os.full - -Level: extended - -Datatype: keyword - -Operating system name, including the version or code name. - -example: `Mac OS Mojave` - -===== host.user.full_name - -Level: extended - -Datatype: keyword - -User's full name, if available. - -example: `Albert Einstein` - -===== host.ip - -Level: core - -Datatype: ip - -Host ip address. - - - -===== host.user.email - -Level: extended - -Datatype: keyword - -User email address. - - - -===== host.os.family - -Level: extended - -Datatype: keyword - -OS family (such as redhat, debian, freebsd, windows). - -example: `debian` - -===== host.geo.region_name - -Level: core - -Datatype: keyword - -Region name. - -example: `Quebec` - -===== host.mac - -Level: core - -Datatype: keyword - -Host mac address. - - - -===== host.user.hash - -Level: extended - -Datatype: keyword - -Unique user hash to correlate information for a user in anonymized form. -Useful if `user.id` or `user.name` contain confidential information and cannot be used. - - - -===== host.geo.city_name - -Level: core - -Datatype: keyword - -City name. - -example: `Montreal` - -===== host.os.version - -Level: extended - -Datatype: keyword - -Operating system version as a raw string. - -example: `10.14.1` - -===== host.type - -Level: core - -Datatype: keyword - -Type of host. -For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. - - - -===== host.geo.country_iso_code - -Level: core - -Datatype: keyword - -Country ISO code. - -example: `CA` - -===== host.os.kernel - -Level: extended - -Datatype: keyword - -Operating system kernel version as a raw string. - -example: `4.4.0-112-generic` - -===== host.architecture - -Level: core - -Datatype: keyword - -Operating system architecture. - -example: `x86_64` - -===== host.geo.region_iso_code - -Level: core - -Datatype: keyword - -Region ISO code. - -example: `CA-QC` - -===== host.geo.name - -Level: extended - -Datatype: keyword - -User-defined description of a location, at the level of granularity they care about. -Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. -Not typically used in automated geolocation. - -example: `boston-dc` - -[[ecs-http]] -=== HTTP fields - -Fields related to HTTP activity. Use the `url` field set to store the url of the request. - -==== Fields summary - -[options="header"] -|===== -| Field | Description | Type (Level) - -// =============================================================== - -| http.request.method -| HTTP request method. -| keyword - -(extended) - -// =============================================================== - -| http.request.body.content -| The full HTTP request body. -| keyword - -(extended) - -// =============================================================== - -| http.request.referrer -| Referrer for this HTTP request. -| keyword - -(extended) - -// =============================================================== - -| http.response.status_code -| HTTP response status code. -| long - -(extended) - -// =============================================================== - -| http.response.body.content -| The full HTTP response body. -| keyword - -(extended) - -// =============================================================== - -| http.version -| HTTP version. -| keyword - -(extended) - -// =============================================================== - -| http.request.bytes -| Total size in bytes of the request (body and headers). -| long - -(extended) - -// =============================================================== - -| http.request.body.bytes -| Size in bytes of the request body. -| long - -(extended) - -// =============================================================== - -| http.response.bytes -| Total size in bytes of the response (body and headers). -| long - -(extended) - -// =============================================================== - -| http.response.body.bytes -| Size in bytes of the response body. -| long - -(extended) - -// =============================================================== - -|===== - -[[ecs-http-details]] -==== Field details - -===== http.request.method - -Level: extended - -Datatype: keyword - -HTTP request method. -The field value must be normalized to lowercase for querying. See the documentation section "Implementing ECS". - -example: `get, post, put` - -===== http.request.body.content - -Level: extended - -Datatype: keyword - -The full HTTP request body. - -example: `Hello world` - -===== http.request.referrer - -Level: extended - -Datatype: keyword - -Referrer for this HTTP request. - -example: `https://blog.example.com/` - -===== http.response.status_code - -Level: extended - -Datatype: long - -HTTP response status code. - -example: `404` - -===== http.response.body.content - -Level: extended - -Datatype: keyword - -The full HTTP response body. - -example: `Hello world` - -===== http.version - -Level: extended - -Datatype: keyword - -HTTP version. - -example: `1.1` - -===== http.request.bytes - -Level: extended - -Datatype: long - -Total size in bytes of the request (body and headers). - -example: `1437` - -===== http.request.body.bytes - -Level: extended - -Datatype: long - -Size in bytes of the request body. - -example: `887` - -===== http.response.bytes - -Level: extended - -Datatype: long - -Total size in bytes of the response (body and headers). - -example: `1437` - -===== http.response.body.bytes - -Level: extended - -Datatype: long - -Size in bytes of the response body. - -example: `887` - -[[ecs-log]] -=== Log fields - -Fields which are specific to log events. - -==== Fields summary - -[options="header"] -|===== -| Field | Description | Type (Level) - -// =============================================================== - -| log.level -| Log level of the log event. -| keyword - -(core) - -// =============================================================== - -| log.original -| Original log message with light interpretation only (encoding, newlines). -| keyword - -(core) - -// =============================================================== - -|===== - -[[ecs-log-details]] -==== Field details - -===== log.level - -Level: core - -Datatype: keyword - -Original log level of the log event. -Some examples are `warn`, `error`, `i`. - -example: `err` - -===== log.original - -Level: core - -Datatype: keyword - -This is the original log message and contains the full log message before splitting it up in multiple parts. -In contrast to the `message` field which can contain an extracted part of the log message, this field contains the original, full log message. It can have already some modifications applied like encoding or new lines removed to clean up the log message. -This field is not indexed and doc_values are disabled so it can't be queried but the value can be retrieved from `_source`. - -example: `Sep 19 08:26:10 localhost My log` - -[[ecs-network]] -=== Network fields - -The network is defined as the communication path over which a host or network event happens. -The network.* fields should be populated with details about the network activity associated with an event. - -==== Fields summary - -[options="header"] -|===== -| Field | Description | Type (Level) - -// =============================================================== - -| network.name -| Name given by operators to sections of their network. -| keyword - -(extended) - -// =============================================================== - -| network.type -| In the OSI Model this would be the Network Layer. ipv4, ipv6, ipsec, pim, etc -| keyword - -(core) - -// =============================================================== - -| network.iana_number -| IANA Protocol Number. -| keyword - -(extended) - -// =============================================================== - -| network.transport -| Protocol Name corresponding to the field `iana_number`. -| keyword - -(core) - -// =============================================================== - -| network.application -| Application level protocol name. -| keyword - -(extended) - -// =============================================================== - -| network.protocol -| L7 Network protocol name. -| keyword - -(core) - -// =============================================================== - -| network.direction -| Direction of the network traffic. -| keyword - -(core) - -// =============================================================== - -| network.forwarded_ip -| Host IP address when the source IP address is the proxy. -| ip - -(core) - -// =============================================================== - -| network.community_id -| A hash of source and destination IPs and ports. -| keyword - -(extended) - -// =============================================================== - -| network.bytes -| Total bytes transferred in both directions. -| long - -(core) - -// =============================================================== - -| network.packets -| Total packets transferred in both directions. -| long - -(core) - -// =============================================================== - -|===== - -[[ecs-network-details]] -==== Field details - -===== network.name - -Level: extended - -Datatype: keyword - -Name given by operators to sections of their network. - -example: `Guest Wifi` - -===== network.type - -Level: core - -Datatype: keyword - -In the OSI Model this would be the Network Layer. ipv4, ipv6, ipsec, pim, etc -The field value must be normalized to lowercase for querying. See the documentation section "Implementing ECS". - -example: `ipv4` - -===== network.iana_number - -Level: extended - -Datatype: keyword - -IANA Protocol Number (https://www.iana.org/assignments/protocol-numbers/protocol-numbers.xhtml). Standardized list of protocols. This aligns well with NetFlow and sFlow related logs which use the IANA Protocol Number. - -example: `6` - -===== network.transport - -Level: core - -Datatype: keyword - -Same as network.iana_number, but instead using the Keyword name of the transport layer (udp, tcp, ipv6-icmp, etc.) -The field value must be normalized to lowercase for querying. See the documentation section "Implementing ECS". - -example: `tcp` - -===== network.application - -Level: extended - -Datatype: keyword - -A name given to an application level protocol. This can be arbitrarily assigned for things like microservices, but also apply to things like skype, icq, facebook, twitter. This would be used in situations where the vendor or service can be decoded such as from the source/dest IP owners, ports, or wire format. -The field value must be normalized to lowercase for querying. See the documentation section "Implementing ECS". - -example: `aim` - -===== network.protocol - -Level: core - -Datatype: keyword - -L7 Network protocol name. ex. http, lumberjack, transport protocol. -The field value must be normalized to lowercase for querying. See the documentation section "Implementing ECS". - -example: `http` - -===== network.direction - -Level: core - -Datatype: keyword - -Direction of the network traffic. -Recommended values are: - * inbound - * outbound - * internal - * external - * unknown - -When mapping events from a host-based monitoring context, populate this field from the host's point of view. -When mapping events from a network or perimeter-based monitoring context, populate this field from the point of view of your network perimeter. - -example: `inbound` - -===== network.forwarded_ip - -Level: core - -Datatype: ip - -Host IP address when the source IP address is the proxy. - -example: `192.1.1.2` - -===== network.community_id - -Level: extended - -Datatype: keyword - -A hash of source and destination IPs and ports, as well as the protocol used in a communication. This is a tool-agnostic standard to identify flows. -Learn more at https://github.com/corelight/community-id-spec. - -example: `1:hO+sN4H+MG5MY/8hIrXPqc4ZQz0=` - -===== network.bytes - -Level: core - -Datatype: long - -Total bytes transferred in both directions. -If `source.bytes` and `destination.bytes` are known, `network.bytes` is their sum. - -example: `368` - -===== network.packets - -Level: core - -Datatype: long - -Total packets transferred in both directions. -If `source.packets` and `destination.packets` are known, `network.packets` is their sum. - -example: `24` - -[[ecs-observer]] -=== Observer fields - -An observer is defined as a special network, security, or application device used to detect, observe, or create network, security, or application-related events and metrics. -This could be a custom hardware appliance or a server that has been configured to run special network, security, or application software. Examples include firewalls, intrusion detection/prevention systems, network monitoring sensors, web application firewalls, data loss prevention systems, and APM servers. The observer.* fields shall be populated with details of the system, if any, that detects, observes and/or creates a network, security, or application event or metric. Message queues and ETL components used in processing events or metrics are not considered observers in ECS. - -==== Fields summary - -[options="header"] -|===== -| Field | Description | Type (Level) - -// =============================================================== - -| observer.mac -| MAC address of the observer -| keyword - -(core) - -// =============================================================== - -| observer.geo.location -| Longitude and latitude. -| geo_point - -(core) - -// =============================================================== - -| observer.os.platform -| Operating system platform (such centos, ubuntu, windows). -| keyword - -(extended) - -// =============================================================== - -| observer.ip -| IP address of the observer. -| ip - -(core) - -// =============================================================== - -| observer.geo.continent_name -| Name of the continent. -| keyword - -(core) - -// =============================================================== - -| observer.os.name -| Operating system name, without the version. -| keyword - -(extended) - -// =============================================================== - -| observer.hostname -| Hostname of the observer. -| keyword - -(core) - -// =============================================================== - -| observer.geo.country_name -| Country name. -| keyword - -(core) - -// =============================================================== - -| observer.os.full -| Operating system name, including the version or code name. -| keyword - -(extended) - -// =============================================================== - -| observer.vendor -| observer vendor information. -| keyword - -(core) - -// =============================================================== - -| observer.os.family -| OS family (such as redhat, debian, freebsd, windows). -| keyword - -(extended) - -// =============================================================== - -| observer.geo.region_name -| Region name. -| keyword - -(core) - -// =============================================================== - -| observer.version -| Observer version. -| keyword - -(core) - -// =============================================================== - -| observer.geo.city_name -| City name. -| keyword - -(core) - -// =============================================================== - -| observer.os.version -| Operating system version as a raw string. -| keyword - -(extended) - -// =============================================================== - -| observer.serial_number -| Observer serial number. -| keyword - -(extended) - -// =============================================================== - -| observer.geo.country_iso_code -| Country ISO code. -| keyword - -(core) - -// =============================================================== - -| observer.os.kernel -| Operating system kernel version as a raw string. -| keyword - -(extended) - -// =============================================================== - -| observer.type -| The type of the observer the data is coming from. -| keyword - -(core) - -// =============================================================== - -| observer.geo.region_iso_code -| Region ISO code. -| keyword - -(core) - -// =============================================================== - -| observer.geo.name -| User-defined description of a location. -| keyword - -(extended) - -// =============================================================== - -|===== - -[[ecs-observer-details]] -==== Field details - -===== observer.mac - -Level: core - -Datatype: keyword - -MAC address of the observer - - - -===== observer.geo.location - -Level: core - -Datatype: geo_point - -Longitude and latitude. - -example: `{ "lon": -73.614830, "lat": 45.505918 }` - -===== observer.os.platform - -Level: extended - -Datatype: keyword - -Operating system platform (such centos, ubuntu, windows). - -example: `darwin` - -===== observer.ip - -Level: core - -Datatype: ip - -IP address of the observer. - - - -===== observer.geo.continent_name - -Level: core - -Datatype: keyword - -Name of the continent. - -example: `North America` - -===== observer.os.name - -Level: extended - -Datatype: keyword - -Operating system name, without the version. - -example: `Mac OS X` - -===== observer.hostname - -Level: core - -Datatype: keyword - -Hostname of the observer. - - - -===== observer.geo.country_name - -Level: core - -Datatype: keyword - -Country name. - -example: `Canada` - -===== observer.os.full - -Level: extended - -Datatype: keyword - -Operating system name, including the version or code name. - -example: `Mac OS Mojave` - -===== observer.vendor - -Level: core - -Datatype: keyword - -observer vendor information. - - - -===== observer.os.family - -Level: extended - -Datatype: keyword - -OS family (such as redhat, debian, freebsd, windows). - -example: `debian` - -===== observer.geo.region_name - -Level: core - -Datatype: keyword - -Region name. - -example: `Quebec` - -===== observer.version - -Level: core - -Datatype: keyword - -Observer version. - - - -===== observer.geo.city_name - -Level: core - -Datatype: keyword - -City name. - -example: `Montreal` - -===== observer.os.version - -Level: extended - -Datatype: keyword - -Operating system version as a raw string. - -example: `10.14.1` - -===== observer.serial_number - -Level: extended - -Datatype: keyword - -Observer serial number. - - - -===== observer.geo.country_iso_code - -Level: core - -Datatype: keyword - -Country ISO code. - -example: `CA` - -===== observer.os.kernel - -Level: extended - -Datatype: keyword - -Operating system kernel version as a raw string. - -example: `4.4.0-112-generic` - -===== observer.type - -Level: core - -Datatype: keyword - -The type of the observer the data is coming from. -There is no predefined list of observer types. Some examples are `forwarder`, `firewall`, `ids`, `ips`, `proxy`, `poller`, `sensor`, `APM server`. - -example: `firewall` - -===== observer.geo.region_iso_code - -Level: core - -Datatype: keyword - -Region ISO code. - -example: `CA-QC` - -===== observer.geo.name - -Level: extended - -Datatype: keyword - -User-defined description of a location, at the level of granularity they care about. -Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. -Not typically used in automated geolocation. - -example: `boston-dc` - -[[ecs-organization]] -=== Organization fields - -The organization fields enrich data with information about the company or entity the data is associated with. -These fields help you arrange or filter data stored in an index by one or multiple organizations. - -==== Fields summary - -[options="header"] -|===== -| Field | Description | Type (Level) - -// =============================================================== - -| organization.name -| Organization name. -| keyword - -(extended) - -// =============================================================== - -| organization.id -| Unique identifier for the organization. -| keyword - -(extended) - -// =============================================================== - -|===== - -[[ecs-organization-details]] -==== Field details - -===== organization.name - -Level: extended - -Datatype: keyword - -Organization name. - - - -===== organization.id - -Level: extended - -Datatype: keyword - -Unique identifier for the organization. - - - -[[ecs-os]] -=== Operating System fields - -The OS fields contain information about the operating system. - -==== Fields summary - -[options="header"] -|===== -| Field | Description | Type (Level) - -// =============================================================== - -| os.platform -| Operating system platform (such centos, ubuntu, windows). -| keyword - -(extended) - -// =============================================================== - -| os.name -| Operating system name, without the version. -| keyword - -(extended) - -// =============================================================== - -| os.full -| Operating system name, including the version or code name. -| keyword - -(extended) - -// =============================================================== - -| os.family -| OS family (such as redhat, debian, freebsd, windows). -| keyword - -(extended) - -// =============================================================== - -| os.version -| Operating system version as a raw string. -| keyword - -(extended) - -// =============================================================== - -| os.kernel -| Operating system kernel version as a raw string. -| keyword - -(extended) - -// =============================================================== - -|===== - -[[ecs-os-details]] -==== Field details - -===== os.platform - -Level: extended - -Datatype: keyword - -Operating system platform (such centos, ubuntu, windows). - -example: `darwin` - -===== os.name - -Level: extended - -Datatype: keyword - -Operating system name, without the version. - -example: `Mac OS X` - -===== os.full - -Level: extended - -Datatype: keyword - -Operating system name, including the version or code name. - -example: `Mac OS Mojave` - -===== os.family - -Level: extended - -Datatype: keyword - -OS family (such as redhat, debian, freebsd, windows). - -example: `debian` - -===== os.version - -Level: extended - -Datatype: keyword - -Operating system version as a raw string. - -example: `10.14.1` - -===== os.kernel - -Level: extended - -Datatype: keyword - -Operating system kernel version as a raw string. - -example: `4.4.0-112-generic` - -[[ecs-process]] -=== Process fields - -These fields contain information about a process. -These fields can help you correlate metrics information with a process id/name from a log message. The `process.pid` often stays in the metric itself and is copied to the global field for correlation. - -==== Fields summary - -[options="header"] -|===== -| Field | Description | Type (Level) - -// =============================================================== - -| process.pid -| Process id. -| long - -(core) +| core // =============================================================== -| process.name -| Process name. -| keyword +| network.direction +| Direction of the network traffic. +Recommended values are: + * inbound + * outbound + * internal + * external + * unknown -(extended) +When mapping events from a host-based monitoring context, populate this field from the host's point of view. +When mapping events from a network or perimeter-based monitoring context, populate this field from the point of view of your network perimeter. -// =============================================================== +type: keyword -| process.ppid -| Process parent id. -| long +example: `inbound` -(extended) +| core // =============================================================== -| process.args -| Array of process arguments. -| keyword - -(extended) +| network.forwarded_ip +| Host IP address when the source IP address is the proxy. -// =============================================================== +type: ip -| process.executable -| Absolute path to the process executable. -| keyword +example: `192.1.1.2` -(extended) +| core // =============================================================== -| process.title -| Process title. -| keyword - -(extended) +| network.community_id +| A hash of source and destination IPs and ports, as well as the protocol used in a communication. This is a tool-agnostic standard to identify flows. +Learn more at https://github.com/corelight/community-id-spec. -// =============================================================== +type: keyword -| process.thread.id -| Thread ID. -| long +example: `1:hO+sN4H+MG5MY/8hIrXPqc4ZQz0=` -(extended) +| extended // =============================================================== -| process.start -| The time the process started. -| date - -(extended) +| network.bytes +| Total bytes transferred in both directions. +If `source.bytes` and `destination.bytes` are known, `network.bytes` is their sum. -// =============================================================== +type: long -| process.working_directory -| The working directory of the process. -| keyword +example: `368` -(extended) +| core // =============================================================== -|===== - -[[ecs-process-details]] -==== Field details - -===== process.pid - -Level: core - -Datatype: long - -Process id. - - - -===== process.name - -Level: extended - -Datatype: keyword - -Process name. -Sometimes called program name or similar. - -example: `ssh` - -===== process.ppid - -Level: extended - -Datatype: long - -Process parent id. - - - -===== process.args - -Level: extended - -Datatype: keyword - -Array of process arguments. -May be filtered to protect sensitive information. - -example: `['ssh', '-l', 'user', '10.0.0.16']` - -===== process.executable - -Level: extended - -Datatype: keyword - -Absolute path to the process executable. - -example: `/usr/bin/ssh` - -===== process.title - -Level: extended - -Datatype: keyword - -Process title. -The proctitle, some times the same as process name. Can also be different: for example a browser setting its title to the web page currently opened. - - - -===== process.thread.id - -Level: extended - -Datatype: long - -Thread ID. - -example: `4242` - -===== process.start - -Level: extended - -Datatype: date - -The time the process started. - -example: `2016-05-23T08:05:34.853Z` - -===== process.working_directory - -Level: extended - -Datatype: keyword - -The working directory of the process. - -example: `/home/alice` - -[[ecs-related]] -=== Related fields - -This field set is meant to facilitate pivoting around a piece of data. -Some pieces of information can be seen in many places in an ECS event. To facilitate searching for them, store an array of all seen values to their corresponding field in `related.`. -A concrete example is IP addresses, which can be under host, observer, source, destination, client, server, and network.forwarded_ip. If you append all IPs to `related.ip`, you can then search for a given IP trivially, no matter where it appeared, by querying `related.ip:a.b.c.d`. - -==== Fields summary - -[options="header"] -|===== -| Field | Description | Type (Level) +| network.packets +| Total packets transferred in both directions. +If `source.packets` and `destination.packets` are known, `network.packets` is their sum. -// =============================================================== +type: long -| related.ip -| All of the IPs seen on your event. -| ip +example: `24` -(extended) +| core // =============================================================== |===== -[[ecs-related-details]] -==== Field details - -===== related.ip - -Level: extended - -Datatype: ip - -All of the IPs seen on your event. - - - -[[ecs-server]] -=== Server fields - -A Server is defined as the responder in a network connection for events regarding sessions, connections, or bidirectional flow records. -For TCP events, the server is the receiver of the initial SYN packet(s) of the TCP connection. For other protocols, the server is generally the responder in the network transaction. Some systems actually use the term "responder" to refer the server in TCP connections. The server fields describe details about the system acting as the server in the network event. Server fields are usually populated in conjunction with client fields. Server fields are generally not populated for packet-level events. -Client / server representations can add semantic context to an exchange, which is helpful to visualize the data in certain situations. If your context falls in that category, you should still ensure that source and destination are filled appropriately. - -==== Fields summary - -[options="header"] -|===== -| Field | Description | Type (Level) - -// =============================================================== - -| server.address -| Server network address. -| keyword - -(extended) - -// =============================================================== - -| server.geo.location -| Longitude and latitude. -| geo_point - -(core) - -// =============================================================== - -| server.user.id -| One or multiple unique identifiers of the user. -| keyword - -(core) - -// =============================================================== - -| server.user.group.id -| Unique identifier for the group on the system/platform. -| keyword - -(extended) - -// =============================================================== - -| server.ip -| IP address of the server. -| ip - -(core) +[[ecs-observer]] +=== Observer fields -// =============================================================== +An observer is defined as a special network, security, or application device used to detect, observe, or create network, security, or application-related events and metrics. +This could be a custom hardware appliance or a server that has been configured to run special network, security, or application software. Examples include firewalls, intrusion detection/prevention systems, network monitoring sensors, web application firewalls, data loss prevention systems, and APM servers. The observer.* fields shall be populated with details of the system, if any, that detects, observes and/or creates a network, security, or application event or metric. Message queues and ETL components used in processing events or metrics are not considered observers in ECS. -| server.user.group.name -| Name of the group. -| keyword +==== Fields summary -(extended) +[options="header"] +|===== +| Field | Description | Level // =============================================================== -| server.geo.continent_name -| Name of the continent. -| keyword +| observer.mac +| MAC address of the observer -(core) +type: keyword -// =============================================================== -| server.user.name -| Short name or login of the user. -| keyword -(core) +| core // =============================================================== -| server.port -| Port of the server. -| long - -(core) +| observer.geo.location +| Longitude and latitude. -// =============================================================== +type: geo_point -| server.geo.country_name -| Country name. -| keyword +example: `{ "lon": -73.614830, "lat": 45.505918 }` -(core) +| core // =============================================================== -| server.user.full_name -| User's full name, if available. -| keyword - -(extended) +| observer.os.platform +| Operating system platform (such centos, ubuntu, windows). -// =============================================================== +type: keyword -| server.mac -| MAC address of the server. -| keyword +example: `darwin` -(core) +| extended // =============================================================== -| server.user.email -| User email address. -| keyword +| observer.ip +| IP address of the observer. -(extended) +type: ip -// =============================================================== -| server.geo.region_name -| Region name. -| keyword -(core) +| core // =============================================================== -| server.domain -| Server domain. -| keyword - -(core) +| observer.geo.continent_name +| Name of the continent. -// =============================================================== +type: keyword -| server.user.hash -| Unique user hash to correlate information for a user in anonymized form. -| keyword +example: `North America` -(extended) +| core // =============================================================== -| server.geo.city_name -| City name. -| keyword - -(core) +| observer.os.name +| Operating system name, without the version. -// =============================================================== +type: keyword -| server.bytes -| Bytes sent from the server to the client. -| long +example: `Mac OS X` -(core) +| extended // =============================================================== -| server.geo.country_iso_code -| Country ISO code. -| keyword +| observer.hostname +| Hostname of the observer. -(core) +type: keyword -// =============================================================== -| server.packets -| Packets sent from the server to the client. -| long -(core) +| core // =============================================================== -| server.geo.region_iso_code -| Region ISO code. -| keyword - -(core) +| observer.geo.country_name +| Country name. -// =============================================================== +type: keyword -| server.geo.name -| User-defined description of a location. -| keyword +example: `Canada` -(extended) +| core // =============================================================== -|===== +| observer.os.full +| Operating system name, including the version or code name. -[[ecs-server-details]] -==== Field details +type: keyword -===== server.address +example: `Mac OS Mojave` -Level: extended +| extended -Datatype: keyword +// =============================================================== -Some event server addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. -Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. +| observer.vendor +| observer vendor information. + +type: keyword -===== server.geo.location +| core -Level: core +// =============================================================== -Datatype: geo_point +| observer.os.family +| OS family (such as redhat, debian, freebsd, windows). -Longitude and latitude. +type: keyword -example: `{ "lon": -73.614830, "lat": 45.505918 }` +example: `debian` -===== server.user.id +| extended -Level: core +// =============================================================== -Datatype: keyword +| observer.geo.region_name +| Region name. -One or multiple unique identifiers of the user. +type: keyword +example: `Quebec` +| core -===== server.user.group.id +// =============================================================== -Level: extended +| observer.version +| Observer version. -Datatype: keyword +type: keyword -Unique identifier for the group on the system/platform. +| core -===== server.ip +// =============================================================== -Level: core +| observer.geo.city_name +| City name. -Datatype: ip +type: keyword -IP address of the server. -Can be one or multiple IPv4 or IPv6 addresses. +example: `Montreal` +| core +// =============================================================== -===== server.user.group.name +| observer.os.version +| Operating system version as a raw string. -Level: extended +type: keyword -Datatype: keyword +example: `10.14.1` -Name of the group. +| extended +// =============================================================== +| observer.serial_number +| Observer serial number. -===== server.geo.continent_name +type: keyword -Level: core -Datatype: keyword -Name of the continent. +| extended -example: `North America` +// =============================================================== -===== server.user.name +| observer.geo.country_iso_code +| Country ISO code. -Level: core +type: keyword -Datatype: keyword +example: `CA` -Short name or login of the user. +| core -example: `albert` +// =============================================================== -===== server.port +| observer.os.kernel +| Operating system kernel version as a raw string. -Level: core +type: keyword -Datatype: long +example: `4.4.0-112-generic` -Port of the server. +| extended +// =============================================================== +| observer.type +| The type of the observer the data is coming from. +There is no predefined list of observer types. Some examples are `forwarder`, `firewall`, `ids`, `ips`, `proxy`, `poller`, `sensor`, `APM server`. -===== server.geo.country_name +type: keyword -Level: core +example: `firewall` -Datatype: keyword +| core -Country name. +// =============================================================== -example: `Canada` +| observer.geo.region_iso_code +| Region ISO code. -===== server.user.full_name +type: keyword -Level: extended +example: `CA-QC` -Datatype: keyword +| core -User's full name, if available. +// =============================================================== -example: `Albert Einstein` +| observer.geo.name +| User-defined description of a location, at the level of granularity they care about. +Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. +Not typically used in automated geolocation. -===== server.mac +type: keyword -Level: core +example: `boston-dc` -Datatype: keyword +| extended -MAC address of the server. +// =============================================================== +|===== +[[ecs-organization]] +=== Organization fields -===== server.user.email +The organization fields enrich data with information about the company or entity the data is associated with. +These fields help you arrange or filter data stored in an index by one or multiple organizations. -Level: extended +==== Fields summary -Datatype: keyword +[options="header"] +|===== +| Field | Description | Level -User email address. +// =============================================================== +| organization.name +| Organization name. +type: keyword -===== server.geo.region_name -Level: core -Datatype: keyword +| extended -Region name. +// =============================================================== -example: `Quebec` +| organization.id +| Unique identifier for the organization. -===== server.domain +type: keyword -Level: core -Datatype: keyword -Server domain. +| extended +// =============================================================== +|===== -===== server.user.hash +[[ecs-os]] +=== Operating System fields -Level: extended +The OS fields contain information about the operating system. -Datatype: keyword +==== Fields summary -Unique user hash to correlate information for a user in anonymized form. -Useful if `user.id` or `user.name` contain confidential information and cannot be used. +[options="header"] +|===== +| Field | Description | Level +// =============================================================== +| os.platform +| Operating system platform (such centos, ubuntu, windows). -===== server.geo.city_name +type: keyword -Level: core +example: `darwin` -Datatype: keyword +| extended -City name. +// =============================================================== -example: `Montreal` +| os.name +| Operating system name, without the version. -===== server.bytes +type: keyword -Level: core +example: `Mac OS X` -Datatype: long +| extended -Bytes sent from the server to the client. +// =============================================================== -example: `184` +| os.full +| Operating system name, including the version or code name. -===== server.geo.country_iso_code +type: keyword -Level: core +example: `Mac OS Mojave` -Datatype: keyword +| extended -Country ISO code. +// =============================================================== -example: `CA` +| os.family +| OS family (such as redhat, debian, freebsd, windows). -===== server.packets +type: keyword -Level: core +example: `debian` -Datatype: long +| extended -Packets sent from the server to the client. +// =============================================================== -example: `12` +| os.version +| Operating system version as a raw string. -===== server.geo.region_iso_code +type: keyword -Level: core +example: `10.14.1` -Datatype: keyword +| extended -Region ISO code. +// =============================================================== -example: `CA-QC` +| os.kernel +| Operating system kernel version as a raw string. -===== server.geo.name +type: keyword -Level: extended +example: `4.4.0-112-generic` -Datatype: keyword +| extended -User-defined description of a location, at the level of granularity they care about. -Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. -Not typically used in automated geolocation. +// =============================================================== -example: `boston-dc` +|===== -[[ecs-service]] -=== Service fields +[[ecs-process]] +=== Process fields -The service fields describe the service for or from which the data was collected. -These fields help you find and correlate logs for a specific service and version. +These fields contain information about a process. +These fields can help you correlate metrics information with a process id/name from a log message. The `process.pid` often stays in the metric itself and is copied to the global field for correlation. ==== Fields summary [options="header"] |===== -| Field | Description | Type (Level) +| Field | Description | Level // =============================================================== -| service.id -| Unique identifier of the running service. -| keyword +| process.pid +| Process id. -(core) +type: long -// =============================================================== -| service.name -| Name of the service. -| keyword -(core) +| core // =============================================================== -| service.type -| The type of the service. -| keyword +| process.name +| Process name. +Sometimes called program name or similar. + +type: keyword + +example: `ssh` -(core) +| extended // =============================================================== -| service.state -| Current state of the service. -| keyword +| process.ppid +| Process parent id. -(core) +type: long + + + +| extended // =============================================================== -| service.version -| Version of the service. -| keyword +| process.args +| Array of process arguments. +May be filtered to protect sensitive information. + +type: keyword -(core) +example: `['ssh', '-l', 'user', '10.0.0.16']` + +| extended // =============================================================== -| service.ephemeral_id -| Ephemeral identifier of this service. -| keyword +| process.executable +| Absolute path to the process executable. + +type: keyword + +example: `/usr/bin/ssh` -(extended) +| extended // =============================================================== -|===== +| process.title +| Process title. +The proctitle, some times the same as process name. Can also be different: for example a browser setting its title to the web page currently opened. -[[ecs-service-details]] -==== Field details +type: keyword -===== service.id -Level: core -Datatype: keyword +| extended -Unique identifier of the running service. -This id should uniquely identify this service. This makes it possible to correlate logs and metrics for one specific service. -Example: If you are experiencing issues with one redis instance, you can filter on that id to see metrics and logs for that single instance. +// =============================================================== -example: `d37e5ebfe0ae6c4972dbe9f0174a1637bb8247f6` +| process.thread.id +| Thread ID. -===== service.name +type: long -Level: core +example: `4242` -Datatype: keyword +| extended -Name of the service data is collected from. -The name of the service is normally user given. This allows if two instances of the same service are running on the same machine they can be differentiated by the `service.name`. -Also it allows for distributed services that run on multiple hosts to correlate the related instances based on the name. -In the case of Elasticsearch the service.name could contain the cluster name. For Beats the service.name is by default a copy of the `service.type` field if no name is specified. +// =============================================================== -example: `elasticsearch-metrics` +| process.start +| The time the process started. -===== service.type +type: date -Level: core +example: `2016-05-23T08:05:34.853Z` -Datatype: keyword +| extended -The type of the service data is collected from. -The type can be used to group and correlate logs and metrics from one service type. -Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. +// =============================================================== + +| process.working_directory +| The working directory of the process. -example: `elasticsearch` +type: keyword -===== service.state +example: `/home/alice` -Level: core +| extended -Datatype: keyword +// =============================================================== -Current state of the service. +|===== +[[ecs-related]] +=== Related fields +This field set is meant to facilitate pivoting around a piece of data. +Some pieces of information can be seen in many places in an ECS event. To facilitate searching for them, store an array of all seen values to their corresponding field in `related.`. +A concrete example is IP addresses, which can be under host, observer, source, destination, client, server, and network.forwarded_ip. If you append all IPs to `related.ip`, you can then search for a given IP trivially, no matter where it appeared, by querying `related.ip:a.b.c.d`. -===== service.version +==== Fields summary -Level: core +[options="header"] +|===== +| Field | Description | Level -Datatype: keyword +// =============================================================== -Version of the service the data was collected from. -This allows to look at a data set only for a specific version of a service. +| related.ip +| All of the IPs seen on your event. -example: `3.2.4` +type: ip -===== service.ephemeral_id -Level: extended -Datatype: keyword +| extended -Ephemeral identifier of this service (if one exists). -This id normally changes across restarts, but `service.id` does not. +// =============================================================== -example: `8a4f500f` +|===== -[[ecs-source]] -=== Source fields +[[ecs-server]] +=== Server fields -Source fields describe details about the source of a packet/event. -Source fields are usually populated in conjunction with destination fields. +A Server is defined as the responder in a network connection for events regarding sessions, connections, or bidirectional flow records. +For TCP events, the server is the receiver of the initial SYN packet(s) of the TCP connection. For other protocols, the server is generally the responder in the network transaction. Some systems actually use the term "responder" to refer the server in TCP connections. The server fields describe details about the system acting as the server in the network event. Server fields are usually populated in conjunction with client fields. Server fields are generally not populated for packet-level events. +Client / server representations can add semantic context to an exchange, which is helpful to visualize the data in certain situations. If your context falls in that category, you should still ensure that source and destination are filled appropriately. ==== Fields summary [options="header"] |===== -| Field | Description | Type (Level) +| Field | Description | Level // =============================================================== -| source.address -| Source network address. -| keyword +| server.address +| Some event server addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. +Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. + +type: keyword + -(extended) + +| extended // =============================================================== -| source.geo.location +| server.geo.location | Longitude and latitude. -| geo_point -(core) +type: geo_point + +example: `{ "lon": -73.614830, "lat": 45.505918 }` + +| core // =============================================================== -| source.user.id +| server.user.id | One or multiple unique identifiers of the user. -| keyword -(core) +type: keyword -// =============================================================== -| source.user.group.id -| Unique identifier for the group on the system/platform. -| keyword -(extended) +| core // =============================================================== -| source.ip -| IP address of the source. -| ip +| server.user.group.id +| Unique identifier for the group on the system/platform. -(core) +type: keyword -// =============================================================== -| source.user.group.name -| Name of the group. -| keyword -(extended) +| extended // =============================================================== -| source.geo.continent_name -| Name of the continent. -| keyword +| server.ip +| IP address of the server. +Can be one or multiple IPv4 or IPv6 addresses. -(core) +type: ip -// =============================================================== -| source.user.name -| Short name or login of the user. -| keyword -(core) +| core // =============================================================== -| source.port -| Port of the source. -| long +| server.user.group.name +| Name of the group. -(core) +type: keyword -// =============================================================== -| source.geo.country_name -| Country name. -| keyword -(core) +| extended // =============================================================== -| source.user.full_name -| User's full name, if available. -| keyword - -(extended) +| server.geo.continent_name +| Name of the continent. -// =============================================================== +type: keyword -| source.mac -| MAC address of the source. -| keyword +example: `North America` -(core) +| core // =============================================================== -| source.user.email -| User email address. -| keyword - -(extended) +| server.user.name +| Short name or login of the user. -// =============================================================== +type: keyword -| source.geo.region_name -| Region name. -| keyword +example: `albert` -(core) +| core // =============================================================== -| source.domain -| Source domain. -| keyword +| server.port +| Port of the server. -(core) +type: long -// =============================================================== -| source.user.hash -| Unique user hash to correlate information for a user in anonymized form. -| keyword -(extended) +| core // =============================================================== -| source.geo.city_name -| City name. -| keyword - -(core) +| server.geo.country_name +| Country name. -// =============================================================== +type: keyword -| source.bytes -| Bytes sent from the source to the destination. -| long +example: `Canada` -(core) +| core // =============================================================== -| source.geo.country_iso_code -| Country ISO code. -| keyword - -(core) +| server.user.full_name +| User's full name, if available. -// =============================================================== +type: keyword -| source.packets -| Packets sent from the source to the destination. -| long +example: `Albert Einstein` -(core) +| extended // =============================================================== -| source.geo.region_iso_code -| Region ISO code. -| keyword +| server.mac +| MAC address of the server. -(core) +type: keyword -// =============================================================== -| source.geo.name -| User-defined description of a location. -| keyword -(extended) +| core // =============================================================== -|===== +| server.user.email +| User email address. -[[ecs-source-details]] -==== Field details +type: keyword -===== source.address -Level: extended -Datatype: keyword +| extended -Some event source addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. -Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. +// =============================================================== + +| server.geo.region_name +| Region name. +type: keyword +example: `Quebec` -===== source.geo.location +| core -Level: core +// =============================================================== -Datatype: geo_point +| server.domain +| Server domain. -Longitude and latitude. +type: keyword -example: `{ "lon": -73.614830, "lat": 45.505918 }` -===== source.user.id -Level: core +| core -Datatype: keyword +// =============================================================== -One or multiple unique identifiers of the user. +| server.user.hash +| Unique user hash to correlate information for a user in anonymized form. +Useful if `user.id` or `user.name` contain confidential information and cannot be used. +type: keyword -===== source.user.group.id -Level: extended +| extended -Datatype: keyword +// =============================================================== -Unique identifier for the group on the system/platform. +| server.geo.city_name +| City name. +type: keyword +example: `Montreal` -===== source.ip +| core -Level: core +// =============================================================== -Datatype: ip +| server.bytes +| Bytes sent from the server to the client. -IP address of the source. -Can be one or multiple IPv4 or IPv6 addresses. +type: long +example: `184` +| core -===== source.user.group.name +// =============================================================== -Level: extended +| server.geo.country_iso_code +| Country ISO code. -Datatype: keyword +type: keyword -Name of the group. +example: `CA` +| core +// =============================================================== -===== source.geo.continent_name +| server.packets +| Packets sent from the server to the client. -Level: core +type: long -Datatype: keyword +example: `12` -Name of the continent. +| core -example: `North America` +// =============================================================== -===== source.user.name +| server.geo.region_iso_code +| Region ISO code. -Level: core +type: keyword -Datatype: keyword +example: `CA-QC` -Short name or login of the user. +| core -example: `albert` +// =============================================================== -===== source.port +| server.geo.name +| User-defined description of a location, at the level of granularity they care about. +Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. +Not typically used in automated geolocation. -Level: core +type: keyword -Datatype: long +example: `boston-dc` -Port of the source. +| extended +// =============================================================== +|===== -===== source.geo.country_name +[[ecs-service]] +=== Service fields -Level: core +The service fields describe the service for or from which the data was collected. +These fields help you find and correlate logs for a specific service and version. -Datatype: keyword +==== Fields summary -Country name. +[options="header"] +|===== +| Field | Description | Level -example: `Canada` +// =============================================================== -===== source.user.full_name +| service.id +| Unique identifier of the running service. +This id should uniquely identify this service. This makes it possible to correlate logs and metrics for one specific service. +Example: If you are experiencing issues with one redis instance, you can filter on that id to see metrics and logs for that single instance. -Level: extended +type: keyword -Datatype: keyword +example: `d37e5ebfe0ae6c4972dbe9f0174a1637bb8247f6` -User's full name, if available. +| core -example: `Albert Einstein` +// =============================================================== -===== source.mac +| service.name +| Name of the service data is collected from. +The name of the service is normally user given. This allows if two instances of the same service are running on the same machine they can be differentiated by the `service.name`. +Also it allows for distributed services that run on multiple hosts to correlate the related instances based on the name. +In the case of Elasticsearch the service.name could contain the cluster name. For Beats the service.name is by default a copy of the `service.type` field if no name is specified. -Level: core +type: keyword -Datatype: keyword +example: `elasticsearch-metrics` -MAC address of the source. +| core +// =============================================================== +| service.type +| The type of the service data is collected from. +The type can be used to group and correlate logs and metrics from one service type. +Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. -===== source.user.email +type: keyword -Level: extended +example: `elasticsearch` -Datatype: keyword +| core -User email address. +// =============================================================== +| service.state +| Current state of the service. +type: keyword -===== source.geo.region_name -Level: core -Datatype: keyword +| core -Region name. +// =============================================================== -example: `Quebec` +| service.version +| Version of the service the data was collected from. +This allows to look at a data set only for a specific version of a service. -===== source.domain +type: keyword -Level: core +example: `3.2.4` -Datatype: keyword +| core -Source domain. +// =============================================================== +| service.ephemeral_id +| Ephemeral identifier of this service (if one exists). +This id normally changes across restarts, but `service.id` does not. +type: keyword -===== source.user.hash +example: `8a4f500f` -Level: extended +| extended -Datatype: keyword +// =============================================================== -Unique user hash to correlate information for a user in anonymized form. -Useful if `user.id` or `user.name` contain confidential information and cannot be used. +|===== +[[ecs-source]] +=== Source fields +Source fields describe details about the source of a packet/event. +Source fields are usually populated in conjunction with destination fields. -===== source.geo.city_name +==== Fields summary -Level: core +[options="header"] +|===== +| Field | Description | Level -Datatype: keyword +// =============================================================== -City name. +| source.address +| Some event source addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. +Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. -example: `Montreal` +type: keyword -===== source.bytes -Level: core -Datatype: long +| extended -Bytes sent from the source to the destination. +// =============================================================== -example: `184` +| source.geo.location +| Longitude and latitude. -===== source.geo.country_iso_code +type: geo_point -Level: core +example: `{ "lon": -73.614830, "lat": 45.505918 }` -Datatype: keyword +| core -Country ISO code. +// =============================================================== -example: `CA` +| source.user.id +| One or multiple unique identifiers of the user. -===== source.packets +type: keyword -Level: core -Datatype: long -Packets sent from the source to the destination. +| core -example: `12` +// =============================================================== -===== source.geo.region_iso_code +| source.user.group.id +| Unique identifier for the group on the system/platform. -Level: core +type: keyword -Datatype: keyword -Region ISO code. -example: `CA-QC` +| extended -===== source.geo.name +// =============================================================== -Level: extended +| source.ip +| IP address of the source. +Can be one or multiple IPv4 or IPv6 addresses. -Datatype: keyword +type: ip -User-defined description of a location, at the level of granularity they care about. -Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. -Not typically used in automated geolocation. -example: `boston-dc` -[[ecs-url]] -=== URL fields +| core -URL fields provide support for complete or partial URLs, and supports the breaking down into scheme, domain, path, and so on. +// =============================================================== -==== Fields summary +| source.user.group.name +| Name of the group. -[options="header"] -|===== -| Field | Description | Type (Level) +type: keyword -// =============================================================== -| url.original -| Unmodified original url as seen in the event source. -| keyword -(extended) +| extended // =============================================================== -| url.full -| Full unparsed URL. -| keyword +| source.geo.continent_name +| Name of the continent. + +type: keyword + +example: `North America` -(extended) +| core // =============================================================== -| url.scheme -| Scheme of the url. -| keyword - -(extended) +| source.user.name +| Short name or login of the user. -// =============================================================== +type: keyword -| url.domain -| Domain of the url. -| keyword +example: `albert` -(extended) +| core // =============================================================== -| url.port -| Port of the request, such as 443. -| long +| source.port +| Port of the source. -(extended) +type: long -// =============================================================== -| url.path -| Path of the request, such as "/search". -| keyword -(extended) +| core // =============================================================== -| url.query -| Query string of the request. -| keyword - -(extended) +| source.geo.country_name +| Country name. -// =============================================================== +type: keyword -| url.fragment -| Portion of the url after the `#`. -| keyword +example: `Canada` -(extended) +| core // =============================================================== -| url.username -| Username of the request. -| keyword +| source.user.full_name +| User's full name, if available. + +type: keyword + +example: `Albert Einstein` -(extended) +| extended // =============================================================== -| url.password -| Password of the request. -| keyword +| source.mac +| MAC address of the source. -(extended) +type: keyword -// =============================================================== -|===== -[[ecs-url-details]] -==== Field details +| core -===== url.original +// =============================================================== -Level: extended +| source.user.email +| User email address. -Datatype: keyword +type: keyword -Unmodified original url as seen in the event source. -Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. -This field is meant to represent the URL as it was observed, complete or not. -example: `https://www.elastic.co:443/search?q=elasticsearch#top or /search?q=elasticsearch` -===== url.full +| extended -Level: extended +// =============================================================== -Datatype: keyword +| source.geo.region_name +| Region name. -If full URLs are important to your use case, they should be stored in `url.full`, whether this field is reconstructed or present in the event source. +type: keyword -example: `https://www.elastic.co:443/search?q=elasticsearch#top` +example: `Quebec` -===== url.scheme +| core -Level: extended +// =============================================================== -Datatype: keyword +| source.domain +| Source domain. -Scheme of the request, such as "https". -Note: The `:` is not part of the scheme. +type: keyword -example: `https` -===== url.domain -Level: extended +| core -Datatype: keyword +// =============================================================== -Domain of the url, such as "www.elastic.co". -In some cases a URL may refer to an IP and/or port directly, without a domain name. In this case, the IP address would go to the `domain` field. +| source.user.hash +| Unique user hash to correlate information for a user in anonymized form. +Useful if `user.id` or `user.name` contain confidential information and cannot be used. -example: `www.elastic.co` +type: keyword -===== url.port -Level: extended -Datatype: long +| extended -Port of the request, such as 443. +// =============================================================== -example: `443` +| source.geo.city_name +| City name. -===== url.path +type: keyword -Level: extended +example: `Montreal` -Datatype: keyword +| core -Path of the request, such as "/search". +// =============================================================== +| source.bytes +| Bytes sent from the source to the destination. +type: long -===== url.query +example: `184` -Level: extended +| core -Datatype: keyword +// =============================================================== -The query field describes the query string of the request, such as "q=elasticsearch". -The `?` is excluded from the query string. If a URL contains no `?`, there is no query field. If there is a `?` but no query, the query field exists with an empty string. The `exists` query can be used to differentiate between the two cases. +| source.geo.country_iso_code +| Country ISO code. +type: keyword +example: `CA` -===== url.fragment +| core -Level: extended +// =============================================================== -Datatype: keyword +| source.packets +| Packets sent from the source to the destination. -Portion of the url after the `#`, such as "top". -The `#` is not part of the fragment. +type: long +example: `12` +| core -===== url.username +// =============================================================== -Level: extended +| source.geo.region_iso_code +| Region ISO code. -Datatype: keyword +type: keyword -Username of the request. +example: `CA-QC` +| core +// =============================================================== -===== url.password +| source.geo.name +| User-defined description of a location, at the level of granularity they care about. +Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. +Not typically used in automated geolocation. -Level: extended +type: keyword -Datatype: keyword +example: `boston-dc` -Password of the request. +| extended +// =============================================================== +|===== -[[ecs-user]] -=== User fields +[[ecs-url]] +=== URL fields -The user fields describe information about the user that is relevant to the event. -Fields can have one entry or multiple entries. If a user has more than one id, provide an array that includes all of them. +URL fields provide support for complete or partial URLs, and supports the breaking down into scheme, domain, path, and so on. ==== Fields summary [options="header"] |===== -| Field | Description | Type (Level) +| Field | Description | Level // =============================================================== -| user.id -| One or multiple unique identifiers of the user. -| keyword - -(core) +| url.original +| Unmodified original url as seen in the event source. +Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. +This field is meant to represent the URL as it was observed, complete or not. -// =============================================================== +type: keyword -| user.group.id -| Unique identifier for the group on the system/platform. -| keyword +example: `https://www.elastic.co:443/search?q=elasticsearch#top or /search?q=elasticsearch` -(extended) +| extended // =============================================================== -| user.group.name -| Name of the group. -| keyword - -(extended) +| url.full +| If full URLs are important to your use case, they should be stored in `url.full`, whether this field is reconstructed or present in the event source. -// =============================================================== +type: keyword -| user.name -| Short name or login of the user. -| keyword +example: `https://www.elastic.co:443/search?q=elasticsearch#top` -(core) +| extended // =============================================================== -| user.full_name -| User's full name, if available. -| keyword - -(extended) +| url.scheme +| Scheme of the request, such as "https". +Note: The `:` is not part of the scheme. -// =============================================================== +type: keyword -| user.email -| User email address. -| keyword +example: `https` -(extended) +| extended // =============================================================== -| user.hash -| Unique user hash to correlate information for a user in anonymized form. -| keyword +| url.domain +| Domain of the url, such as "www.elastic.co". +In some cases a URL may refer to an IP and/or port directly, without a domain name. In this case, the IP address would go to the `domain` field. + +type: keyword + +example: `www.elastic.co` -(extended) +| extended // =============================================================== -|===== +| url.port +| Port of the request, such as 443. -[[ecs-user-details]] -==== Field details +type: long -===== user.id +example: `443` -Level: core +| extended -Datatype: keyword +// =============================================================== -One or multiple unique identifiers of the user. +| url.path +| Path of the request, such as "/search". +type: keyword -===== user.group.id -Level: extended +| extended -Datatype: keyword +// =============================================================== -Unique identifier for the group on the system/platform. +| url.query +| The query field describes the query string of the request, such as "q=elasticsearch". +The `?` is excluded from the query string. If a URL contains no `?`, there is no query field. If there is a `?` but no query, the query field exists with an empty string. The `exists` query can be used to differentiate between the two cases. +type: keyword -===== user.group.name -Level: extended +| extended -Datatype: keyword +// =============================================================== -Name of the group. +| url.fragment +| Portion of the url after the `#`, such as "top". +The `#` is not part of the fragment. +type: keyword -===== user.name -Level: core +| extended -Datatype: keyword +// =============================================================== -Short name or login of the user. +| url.username +| Username of the request. -example: `albert` +type: keyword -===== user.full_name -Level: extended -Datatype: keyword +| extended -User's full name, if available. +// =============================================================== -example: `Albert Einstein` +| url.password +| Password of the request. -===== user.email +type: keyword -Level: extended -Datatype: keyword -User email address. +| extended +// =============================================================== +|===== -===== user.hash +[[ecs-user]] +=== User fields -Level: extended +The user fields describe information about the user that is relevant to the event. +Fields can have one entry or multiple entries. If a user has more than one id, provide an array that includes all of them. -Datatype: keyword +==== Fields summary -Unique user hash to correlate information for a user in anonymized form. -Useful if `user.id` or `user.name` contain confidential information and cannot be used. +[options="header"] +|===== +| Field | Description | Level +// =============================================================== +| user.id +| One or multiple unique identifiers of the user. -[[ecs-user_agent]] -=== User agent fields +type: keyword -The user_agent fields normally come from a browser request. -They often show up in web service logs coming from the parsed user agent string. -==== Fields summary -[options="header"] -|===== -| Field | Description | Type (Level) +| core // =============================================================== -| user_agent.original -| Unparsed version of the user_agent. -| keyword +| user.group.id +| Unique identifier for the group on the system/platform. -(extended) +type: keyword -// =============================================================== -| user_agent.os.platform -| Operating system platform (such centos, ubuntu, windows). -| keyword -(extended) +| extended // =============================================================== -| user_agent.name -| Name of the user agent. -| keyword +| user.group.name +| Name of the group. -(extended) +type: keyword -// =============================================================== -| user_agent.os.name -| Operating system name, without the version. -| keyword -(extended) +| extended // =============================================================== -| user_agent.version -| Version of the user agent. -| keyword +| user.name +| Short name or login of the user. + +type: keyword -(extended) +example: `albert` + +| core // =============================================================== -| user_agent.os.full -| Operating system name, including the version or code name. -| keyword +| user.full_name +| User's full name, if available. + +type: keyword + +example: `Albert Einstein` -(extended) +| extended // =============================================================== -| user_agent.device.name -| Name of the device. -| keyword +| user.email +| User email address. -(extended) +type: keyword -// =============================================================== -| user_agent.os.family -| OS family (such as redhat, debian, freebsd, windows). -| keyword -(extended) +| extended // =============================================================== -| user_agent.os.version -| Operating system version as a raw string. -| keyword +| user.hash +| Unique user hash to correlate information for a user in anonymized form. +Useful if `user.id` or `user.name` contain confidential information and cannot be used. -(extended) +type: keyword -// =============================================================== -| user_agent.os.kernel -| Operating system kernel version as a raw string. -| keyword -(extended) +| extended // =============================================================== |===== -[[ecs-user_agent-details]] -==== Field details +[[ecs-user_agent]] +=== User agent fields + +The user_agent fields normally come from a browser request. +They often show up in web service logs coming from the parsed user agent string. + +==== Fields summary -===== user_agent.original +[options="header"] +|===== +| Field | Description | Level -Level: extended +// =============================================================== -Datatype: keyword +| user_agent.original +| Unparsed version of the user_agent. -Unparsed version of the user_agent. +type: keyword example: `Mozilla/5.0 (iPhone; CPU iPhone OS 12_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.0 Mobile/15E148 Safari/604.1` -===== user_agent.os.platform +| extended -Level: extended +// =============================================================== -Datatype: keyword +| user_agent.os.platform +| Operating system platform (such centos, ubuntu, windows). -Operating system platform (such centos, ubuntu, windows). +type: keyword example: `darwin` -===== user_agent.name +| extended -Level: extended +// =============================================================== -Datatype: keyword +| user_agent.name +| Name of the user agent. -Name of the user agent. +type: keyword example: `Safari` -===== user_agent.os.name +| extended -Level: extended +// =============================================================== -Datatype: keyword +| user_agent.os.name +| Operating system name, without the version. -Operating system name, without the version. +type: keyword example: `Mac OS X` -===== user_agent.version +| extended -Level: extended +// =============================================================== -Datatype: keyword +| user_agent.version +| Version of the user agent. -Version of the user agent. +type: keyword example: `12.0` -===== user_agent.os.full +| extended -Level: extended +// =============================================================== -Datatype: keyword +| user_agent.os.full +| Operating system name, including the version or code name. -Operating system name, including the version or code name. +type: keyword example: `Mac OS Mojave` -===== user_agent.device.name +| extended -Level: extended +// =============================================================== -Datatype: keyword +| user_agent.device.name +| Name of the device. -Name of the device. +type: keyword example: `iPhone` -===== user_agent.os.family +| extended -Level: extended +// =============================================================== -Datatype: keyword +| user_agent.os.family +| OS family (such as redhat, debian, freebsd, windows). -OS family (such as redhat, debian, freebsd, windows). +type: keyword example: `debian` -===== user_agent.os.version +| extended -Level: extended +// =============================================================== -Datatype: keyword +| user_agent.os.version +| Operating system version as a raw string. -Operating system version as a raw string. +type: keyword example: `10.14.1` -===== user_agent.os.kernel +| extended -Level: extended +// =============================================================== -Datatype: keyword +| user_agent.os.kernel +| Operating system kernel version as a raw string. -Operating system kernel version as a raw string. +type: keyword example: `4.4.0-112-generic` + +| extended + +// =============================================================== + +|===== diff --git a/scripts/generators/asciidoc_fields.py b/scripts/generators/asciidoc_fields.py index 4491de053f..2ae9477311 100644 --- a/scripts/generators/asciidoc_fields.py +++ b/scripts/generators/asciidoc_fields.py @@ -56,32 +56,33 @@ def page_field_details(ecs_nested): def render_fieldset(fieldset): - summary_text = fieldset_summary_header().format( + table = fieldset_summary_header().format( fieldset_name=fieldset['name'], fieldset_description=fieldset['description'], fieldset_title=fieldset['title'] ) - detail_text = fieldset_details_header().format( - fieldset_name=fieldset['name'], - fieldset_title=fieldset['title'] - ) + # detail_text = fieldset_details_header().format( + # fieldset_name=fieldset['name'], + # fieldset_title=fieldset['title'] + # ) for field in sorted_by_keys(fieldset['fields'], 'order'): - summary_text += render_field_summary_row(field) - detail_text += render_field_details(field) + table += render_field_summary_row(field) + # detail_text += render_field_details(field) - summary_text += table_footer() - return summary_text + detail_text + table += table_footer() + # return table + detail_text + return table def render_field_summary_row(field): - # example = '' - # if 'example' in field: - # example = "example: `{}`".format(str(field['example'])) + example = '' + if 'example' in field: + example = "example: `{}`".format(str(field['example'])) field_text = field_row().format( field_flat_name=field['flat_name'], - field_short=field['short'], - # field_example=example, + field_description=field['description'], + field_example=example, field_level=field['level'], field_type=field['type'], ) @@ -154,7 +155,7 @@ def fieldset_summary_header(): [options="header"] |===== -| Field | Description | Type (Level) +| Field | Description | Level // =============================================================== ''' @@ -163,10 +164,13 @@ def fieldset_summary_header(): def field_row(): return ''' | {field_flat_name} -| {field_short} -| {field_type} +| {field_description} + +type: {field_type} + +{field_example} -({field_level}) +| {field_level} // =============================================================== ''' From f9fa10c5ebc3b8264831cb143a81f4d8406a79c8 Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Fri, 1 Mar 2019 15:19:47 -0500 Subject: [PATCH 15/28] Remove unnecessary code for the details section --- scripts/generators/asciidoc_fields.py | 27 --------------------------- 1 file changed, 27 deletions(-) diff --git a/scripts/generators/asciidoc_fields.py b/scripts/generators/asciidoc_fields.py index 2ae9477311..6ad487f409 100644 --- a/scripts/generators/asciidoc_fields.py +++ b/scripts/generators/asciidoc_fields.py @@ -61,17 +61,11 @@ def render_fieldset(fieldset): fieldset_description=fieldset['description'], fieldset_title=fieldset['title'] ) - # detail_text = fieldset_details_header().format( - # fieldset_name=fieldset['name'], - # fieldset_title=fieldset['title'] - # ) for field in sorted_by_keys(fieldset['fields'], 'order'): table += render_field_summary_row(field) - # detail_text += render_field_details(field) table += table_footer() - # return table + detail_text return table @@ -176,27 +170,6 @@ def field_row(): ''' -def fieldset_details_header(): - return ''' -[[ecs-{fieldset_name}-details]] -==== Field details -''' - - -def field_details(): - return ''' -===== {field_flat_name} - -Level: {field_level} - -Datatype: {field_type} - -{field_description} - -{field_example} -''' - - # File From c6f29148466bb3cf34e02f13edf3a04c6e69ed6b Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Fri, 1 Mar 2019 16:15:41 -0500 Subject: [PATCH 16/28] Total WIP of what 'can be nested' under a field set --- docs/field-details.asciidoc | 757 +++++++++++++++++++++++++- scripts/generators/asciidoc_fields.py | 33 +- 2 files changed, 762 insertions(+), 28 deletions(-) diff --git a/docs/field-details.asciidoc b/docs/field-details.asciidoc index d975fd724f..eae4076693 100644 --- a/docs/field-details.asciidoc +++ b/docs/field-details.asciidoc @@ -4,7 +4,7 @@ The `base` field set contains all fields which are on the top level. These fields are common across all types of events. -==== Fields summary +==== Base Fields [options="header"] |===== @@ -65,13 +65,39 @@ example: `Hello World` |===== +[[ecs-base-nestings]] +==== Can be nested under Host + +[options="header"] +|===== +| Nested fields | Description + +// =============================================================== + +| http://localhost:8000/ecs-geo.html[host.geo.*] +| Fields describing a location. + +// =============================================================== + +| http://localhost:8000/ecs-os.html[host.os.*] +| OS fields contain information about the operating system. + +// =============================================================== + +| http://localhost:8000/ecs-user.html[host.user.*] +| Fields to describe the user relevant to the event. + +// =============================================================== + +|===== + [[ecs-agent]] === Agent fields The agent fields contain the data about the software entity, if any, that collects, detects, or observes events on a host, or takes measurements on a host. Examples include Beats. Agents may also run on observers. ECS agent.* fields shall be populated with details of the agent running on the host or observer where the event happened or the measurement was taken. -==== Fields summary +==== Agent Fields [options="header"] |===== @@ -141,6 +167,32 @@ example: `8a4f500f` |===== +[[ecs-agent-nestings]] +==== Can be nested under Host + +[options="header"] +|===== +| Nested fields | Description + +// =============================================================== + +| http://localhost:8000/ecs-geo.html[host.geo.*] +| Fields describing a location. + +// =============================================================== + +| http://localhost:8000/ecs-os.html[host.os.*] +| OS fields contain information about the operating system. + +// =============================================================== + +| http://localhost:8000/ecs-user.html[host.user.*] +| Fields to describe the user relevant to the event. + +// =============================================================== + +|===== + [[ecs-client]] === Client fields @@ -148,7 +200,7 @@ A client is defined as the initiator of a network connection for events regardin For TCP events, the client is the initiator of the TCP connection that sends the SYN packet(s). For other protocols, the client is generally the initiator or requestor in the network transaction. Some systems use the term "originator" to refer the client in TCP connections. The client fields describe details about the system acting as the client in the network event. Client fields are usually populated in conjunction with server fields. Client fields are generally not populated for packet-level events. Client / server representations can add semantic context to an exchange, which is helpful to visualize the data in certain situations. If your context falls in that category, you should still ensure that source and destination are filled appropriately. -==== Fields summary +==== Client Fields [options="header"] |===== @@ -405,12 +457,38 @@ example: `boston-dc` |===== +[[ecs-client-nestings]] +==== Can be nested under Host + +[options="header"] +|===== +| Nested fields | Description + +// =============================================================== + +| http://localhost:8000/ecs-geo.html[host.geo.*] +| Fields describing a location. + +// =============================================================== + +| http://localhost:8000/ecs-os.html[host.os.*] +| OS fields contain information about the operating system. + +// =============================================================== + +| http://localhost:8000/ecs-user.html[host.user.*] +| Fields to describe the user relevant to the event. + +// =============================================================== + +|===== + [[ecs-cloud]] === Cloud fields Fields related to the cloud or infrastructure the events are coming from. -==== Fields summary +==== Cloud Fields [options="header"] |===== @@ -498,13 +576,39 @@ example: `666777888999` |===== +[[ecs-cloud-nestings]] +==== Can be nested under Host + +[options="header"] +|===== +| Nested fields | Description + +// =============================================================== + +| http://localhost:8000/ecs-geo.html[host.geo.*] +| Fields describing a location. + +// =============================================================== + +| http://localhost:8000/ecs-os.html[host.os.*] +| OS fields contain information about the operating system. + +// =============================================================== + +| http://localhost:8000/ecs-user.html[host.user.*] +| Fields to describe the user relevant to the event. + +// =============================================================== + +|===== + [[ecs-container]] === Container fields Container fields are used for meta information about the specific container that is the source of information. These fields help correlate data based containers from any runtime. -==== Fields summary +==== Container Fields [options="header"] |===== @@ -580,13 +684,39 @@ type: object |===== +[[ecs-container-nestings]] +==== Can be nested under Host + +[options="header"] +|===== +| Nested fields | Description + +// =============================================================== + +| http://localhost:8000/ecs-geo.html[host.geo.*] +| Fields describing a location. + +// =============================================================== + +| http://localhost:8000/ecs-os.html[host.os.*] +| OS fields contain information about the operating system. + +// =============================================================== + +| http://localhost:8000/ecs-user.html[host.user.*] +| Fields to describe the user relevant to the event. + +// =============================================================== + +|===== + [[ecs-destination]] === Destination fields Destination fields describe details about the destination of a packet/event. Destination fields are usually populated in conjunction with source fields. -==== Fields summary +==== Destination Fields [options="header"] |===== @@ -843,12 +973,38 @@ example: `boston-dc` |===== +[[ecs-destination-nestings]] +==== Can be nested under Host + +[options="header"] +|===== +| Nested fields | Description + +// =============================================================== + +| http://localhost:8000/ecs-geo.html[host.geo.*] +| Fields describing a location. + +// =============================================================== + +| http://localhost:8000/ecs-os.html[host.os.*] +| OS fields contain information about the operating system. + +// =============================================================== + +| http://localhost:8000/ecs-user.html[host.user.*] +| Fields to describe the user relevant to the event. + +// =============================================================== + +|===== + [[ecs-ecs]] === ECS fields Meta-information specific to ECS. -==== Fields summary +==== ECS Fields [options="header"] |===== @@ -870,13 +1026,39 @@ example: `1.0.0` |===== +[[ecs-ecs-nestings]] +==== Can be nested under Host + +[options="header"] +|===== +| Nested fields | Description + +// =============================================================== + +| http://localhost:8000/ecs-geo.html[host.geo.*] +| Fields describing a location. + +// =============================================================== + +| http://localhost:8000/ecs-os.html[host.os.*] +| OS fields contain information about the operating system. + +// =============================================================== + +| http://localhost:8000/ecs-user.html[host.user.*] +| Fields to describe the user relevant to the event. + +// =============================================================== + +|===== + [[ecs-error]] === Error fields These fields can represent errors of any kind. Use them for errors that happen while fetching events or in cases where the event itself contains an error. -==== Fields summary +==== Error Fields [options="header"] |===== @@ -919,13 +1101,39 @@ type: keyword |===== +[[ecs-error-nestings]] +==== Can be nested under Host + +[options="header"] +|===== +| Nested fields | Description + +// =============================================================== + +| http://localhost:8000/ecs-geo.html[host.geo.*] +| Fields describing a location. + +// =============================================================== + +| http://localhost:8000/ecs-os.html[host.os.*] +| OS fields contain information about the operating system. + +// =============================================================== + +| http://localhost:8000/ecs-user.html[host.user.*] +| Fields to describe the user relevant to the event. + +// =============================================================== + +|===== + [[ecs-event]] === Event fields The event fields are used for context information about the log or metric event itself. A log is defined as an event containing details of something that happened. Log events must include the time at which the thing happened. Examples of log events include a process starting on a host, a network packet being sent from a source to a destination, or a network connection between a client and a server being initiated or closed. A metric is defined as an event containing one or more numerical or categorical measurements and the time at which the measurement was taken. Examples of metric events include memory pressure measured on a host, or vulnerabilities measured on a scanned host. -==== Fields summary +==== Event Fields [options="header"] |===== @@ -1147,13 +1355,39 @@ type: float |===== +[[ecs-event-nestings]] +==== Can be nested under Host + +[options="header"] +|===== +| Nested fields | Description + +// =============================================================== + +| http://localhost:8000/ecs-geo.html[host.geo.*] +| Fields describing a location. + +// =============================================================== + +| http://localhost:8000/ecs-os.html[host.os.*] +| OS fields contain information about the operating system. + +// =============================================================== + +| http://localhost:8000/ecs-user.html[host.user.*] +| Fields to describe the user relevant to the event. + +// =============================================================== + +|===== + [[ecs-file]] === File fields A file is defined as a set of information that has been created on, or has existed on a filesystem. File objects can be associated with host events, network events, and/or file events (e.g., those produced by File Integrity Monitoring [FIM] products or services). File fields provide details about the affected file associated with the event or metric. -==== Fields summary +==== File Fields [options="header"] |===== @@ -1318,13 +1552,39 @@ type: date |===== +[[ecs-file-nestings]] +==== Can be nested under Host + +[options="header"] +|===== +| Nested fields | Description + +// =============================================================== + +| http://localhost:8000/ecs-geo.html[host.geo.*] +| Fields describing a location. + +// =============================================================== + +| http://localhost:8000/ecs-os.html[host.os.*] +| OS fields contain information about the operating system. + +// =============================================================== + +| http://localhost:8000/ecs-user.html[host.user.*] +| Fields to describe the user relevant to the event. + +// =============================================================== + +|===== + [[ecs-geo]] === Geo fields Geo fields can carry data about a specific location related to an event. This geolocation information can be derived from techniques such as Geo IP, or be user-supplied. -==== Fields summary +==== Geo Fields [options="header"] |===== @@ -1424,12 +1684,38 @@ example: `boston-dc` |===== +[[ecs-geo-nestings]] +==== Can be nested under Host + +[options="header"] +|===== +| Nested fields | Description + +// =============================================================== + +| http://localhost:8000/ecs-geo.html[host.geo.*] +| Fields describing a location. + +// =============================================================== + +| http://localhost:8000/ecs-os.html[host.os.*] +| OS fields contain information about the operating system. + +// =============================================================== + +| http://localhost:8000/ecs-user.html[host.user.*] +| Fields to describe the user relevant to the event. + +// =============================================================== + +|===== + [[ecs-group]] === Group fields The group fields are meant to represent groups that are relevant to the event. -==== Fields summary +==== Group Fields [options="header"] |===== @@ -1461,13 +1747,39 @@ type: keyword |===== +[[ecs-group-nestings]] +==== Can be nested under Host + +[options="header"] +|===== +| Nested fields | Description + +// =============================================================== + +| http://localhost:8000/ecs-geo.html[host.geo.*] +| Fields describing a location. + +// =============================================================== + +| http://localhost:8000/ecs-os.html[host.os.*] +| OS fields contain information about the operating system. + +// =============================================================== + +| http://localhost:8000/ecs-user.html[host.user.*] +| Fields to describe the user relevant to the event. + +// =============================================================== + +|===== + [[ecs-host]] === Host fields A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes. -==== Fields summary +==== Host Fields [options="header"] |===== @@ -1793,12 +2105,38 @@ example: `boston-dc` |===== +[[ecs-host-nestings]] +==== Can be nested under Host + +[options="header"] +|===== +| Nested fields | Description + +// =============================================================== + +| http://localhost:8000/ecs-geo.html[host.geo.*] +| Fields describing a location. + +// =============================================================== + +| http://localhost:8000/ecs-os.html[host.os.*] +| OS fields contain information about the operating system. + +// =============================================================== + +| http://localhost:8000/ecs-user.html[host.user.*] +| Fields to describe the user relevant to the event. + +// =============================================================== + +|===== + [[ecs-http]] === HTTP fields Fields related to HTTP activity. Use the `url` field set to store the url of the request. -==== Fields summary +==== HTTP Fields [options="header"] |===== @@ -1919,12 +2257,38 @@ example: `887` |===== +[[ecs-http-nestings]] +==== Can be nested under Host + +[options="header"] +|===== +| Nested fields | Description + +// =============================================================== + +| http://localhost:8000/ecs-geo.html[host.geo.*] +| Fields describing a location. + +// =============================================================== + +| http://localhost:8000/ecs-os.html[host.os.*] +| OS fields contain information about the operating system. + +// =============================================================== + +| http://localhost:8000/ecs-user.html[host.user.*] +| Fields to describe the user relevant to the event. + +// =============================================================== + +|===== + [[ecs-log]] === Log fields Fields which are specific to log events. -==== Fields summary +==== Log Fields [options="header"] |===== @@ -1959,13 +2323,39 @@ example: `Sep 19 08:26:10 localhost My log` |===== +[[ecs-log-nestings]] +==== Can be nested under Host + +[options="header"] +|===== +| Nested fields | Description + +// =============================================================== + +| http://localhost:8000/ecs-geo.html[host.geo.*] +| Fields describing a location. + +// =============================================================== + +| http://localhost:8000/ecs-os.html[host.os.*] +| OS fields contain information about the operating system. + +// =============================================================== + +| http://localhost:8000/ecs-user.html[host.user.*] +| Fields to describe the user relevant to the event. + +// =============================================================== + +|===== + [[ecs-network]] === Network fields The network is defined as the communication path over which a host or network event happens. The network.* fields should be populated with details about the network activity associated with an event. -==== Fields summary +==== Network Fields [options="header"] |===== @@ -2112,13 +2502,39 @@ example: `24` |===== +[[ecs-network-nestings]] +==== Can be nested under Host + +[options="header"] +|===== +| Nested fields | Description + +// =============================================================== + +| http://localhost:8000/ecs-geo.html[host.geo.*] +| Fields describing a location. + +// =============================================================== + +| http://localhost:8000/ecs-os.html[host.os.*] +| OS fields contain information about the operating system. + +// =============================================================== + +| http://localhost:8000/ecs-user.html[host.user.*] +| Fields to describe the user relevant to the event. + +// =============================================================== + +|===== + [[ecs-observer]] === Observer fields An observer is defined as a special network, security, or application device used to detect, observe, or create network, security, or application-related events and metrics. This could be a custom hardware appliance or a server that has been configured to run special network, security, or application software. Examples include firewalls, intrusion detection/prevention systems, network monitoring sensors, web application firewalls, data loss prevention systems, and APM servers. The observer.* fields shall be populated with details of the system, if any, that detects, observes and/or creates a network, security, or application event or metric. Message queues and ETL components used in processing events or metrics are not considered observers in ECS. -==== Fields summary +==== Observer Fields [options="header"] |===== @@ -2362,13 +2778,39 @@ example: `boston-dc` |===== +[[ecs-observer-nestings]] +==== Can be nested under Host + +[options="header"] +|===== +| Nested fields | Description + +// =============================================================== + +| http://localhost:8000/ecs-geo.html[host.geo.*] +| Fields describing a location. + +// =============================================================== + +| http://localhost:8000/ecs-os.html[host.os.*] +| OS fields contain information about the operating system. + +// =============================================================== + +| http://localhost:8000/ecs-user.html[host.user.*] +| Fields to describe the user relevant to the event. + +// =============================================================== + +|===== + [[ecs-organization]] === Organization fields The organization fields enrich data with information about the company or entity the data is associated with. These fields help you arrange or filter data stored in an index by one or multiple organizations. -==== Fields summary +==== Organization Fields [options="header"] |===== @@ -2400,12 +2842,38 @@ type: keyword |===== +[[ecs-organization-nestings]] +==== Can be nested under Host + +[options="header"] +|===== +| Nested fields | Description + +// =============================================================== + +| http://localhost:8000/ecs-geo.html[host.geo.*] +| Fields describing a location. + +// =============================================================== + +| http://localhost:8000/ecs-os.html[host.os.*] +| OS fields contain information about the operating system. + +// =============================================================== + +| http://localhost:8000/ecs-user.html[host.user.*] +| Fields to describe the user relevant to the event. + +// =============================================================== + +|===== + [[ecs-os]] === Operating System fields The OS fields contain information about the operating system. -==== Fields summary +==== Operating System Fields [options="header"] |===== @@ -2481,13 +2949,39 @@ example: `4.4.0-112-generic` |===== +[[ecs-os-nestings]] +==== Can be nested under Host + +[options="header"] +|===== +| Nested fields | Description + +// =============================================================== + +| http://localhost:8000/ecs-geo.html[host.geo.*] +| Fields describing a location. + +// =============================================================== + +| http://localhost:8000/ecs-os.html[host.os.*] +| OS fields contain information about the operating system. + +// =============================================================== + +| http://localhost:8000/ecs-user.html[host.user.*] +| Fields to describe the user relevant to the event. + +// =============================================================== + +|===== + [[ecs-process]] === Process fields These fields contain information about a process. These fields can help you correlate metrics information with a process id/name from a log message. The `process.pid` often stays in the metric itself and is copied to the global field for correlation. -==== Fields summary +==== Process Fields [options="header"] |===== @@ -2599,6 +3093,32 @@ example: `/home/alice` |===== +[[ecs-process-nestings]] +==== Can be nested under Host + +[options="header"] +|===== +| Nested fields | Description + +// =============================================================== + +| http://localhost:8000/ecs-geo.html[host.geo.*] +| Fields describing a location. + +// =============================================================== + +| http://localhost:8000/ecs-os.html[host.os.*] +| OS fields contain information about the operating system. + +// =============================================================== + +| http://localhost:8000/ecs-user.html[host.user.*] +| Fields to describe the user relevant to the event. + +// =============================================================== + +|===== + [[ecs-related]] === Related fields @@ -2606,7 +3126,7 @@ This field set is meant to facilitate pivoting around a piece of data. Some pieces of information can be seen in many places in an ECS event. To facilitate searching for them, store an array of all seen values to their corresponding field in `related.`. A concrete example is IP addresses, which can be under host, observer, source, destination, client, server, and network.forwarded_ip. If you append all IPs to `related.ip`, you can then search for a given IP trivially, no matter where it appeared, by querying `related.ip:a.b.c.d`. -==== Fields summary +==== Related Fields [options="header"] |===== @@ -2627,6 +3147,32 @@ type: ip |===== +[[ecs-related-nestings]] +==== Can be nested under Host + +[options="header"] +|===== +| Nested fields | Description + +// =============================================================== + +| http://localhost:8000/ecs-geo.html[host.geo.*] +| Fields describing a location. + +// =============================================================== + +| http://localhost:8000/ecs-os.html[host.os.*] +| OS fields contain information about the operating system. + +// =============================================================== + +| http://localhost:8000/ecs-user.html[host.user.*] +| Fields to describe the user relevant to the event. + +// =============================================================== + +|===== + [[ecs-server]] === Server fields @@ -2634,7 +3180,7 @@ A Server is defined as the responder in a network connection for events regardin For TCP events, the server is the receiver of the initial SYN packet(s) of the TCP connection. For other protocols, the server is generally the responder in the network transaction. Some systems actually use the term "responder" to refer the server in TCP connections. The server fields describe details about the system acting as the server in the network event. Server fields are usually populated in conjunction with client fields. Server fields are generally not populated for packet-level events. Client / server representations can add semantic context to an exchange, which is helpful to visualize the data in certain situations. If your context falls in that category, you should still ensure that source and destination are filled appropriately. -==== Fields summary +==== Server Fields [options="header"] |===== @@ -2891,13 +3437,39 @@ example: `boston-dc` |===== +[[ecs-server-nestings]] +==== Can be nested under Host + +[options="header"] +|===== +| Nested fields | Description + +// =============================================================== + +| http://localhost:8000/ecs-geo.html[host.geo.*] +| Fields describing a location. + +// =============================================================== + +| http://localhost:8000/ecs-os.html[host.os.*] +| OS fields contain information about the operating system. + +// =============================================================== + +| http://localhost:8000/ecs-user.html[host.user.*] +| Fields to describe the user relevant to the event. + +// =============================================================== + +|===== + [[ecs-service]] === Service fields The service fields describe the service for or from which the data was collected. These fields help you find and correlate logs for a specific service and version. -==== Fields summary +==== Service Fields [options="header"] |===== @@ -2982,13 +3554,39 @@ example: `8a4f500f` |===== +[[ecs-service-nestings]] +==== Can be nested under Host + +[options="header"] +|===== +| Nested fields | Description + +// =============================================================== + +| http://localhost:8000/ecs-geo.html[host.geo.*] +| Fields describing a location. + +// =============================================================== + +| http://localhost:8000/ecs-os.html[host.os.*] +| OS fields contain information about the operating system. + +// =============================================================== + +| http://localhost:8000/ecs-user.html[host.user.*] +| Fields to describe the user relevant to the event. + +// =============================================================== + +|===== + [[ecs-source]] === Source fields Source fields describe details about the source of a packet/event. Source fields are usually populated in conjunction with destination fields. -==== Fields summary +==== Source Fields [options="header"] |===== @@ -3245,12 +3843,38 @@ example: `boston-dc` |===== +[[ecs-source-nestings]] +==== Can be nested under Host + +[options="header"] +|===== +| Nested fields | Description + +// =============================================================== + +| http://localhost:8000/ecs-geo.html[host.geo.*] +| Fields describing a location. + +// =============================================================== + +| http://localhost:8000/ecs-os.html[host.os.*] +| OS fields contain information about the operating system. + +// =============================================================== + +| http://localhost:8000/ecs-user.html[host.user.*] +| Fields to describe the user relevant to the event. + +// =============================================================== + +|===== + [[ecs-url]] === URL fields URL fields provide support for complete or partial URLs, and supports the breaking down into scheme, domain, path, and so on. -==== Fields summary +==== URL Fields [options="header"] |===== @@ -3376,13 +4000,39 @@ type: keyword |===== +[[ecs-url-nestings]] +==== Can be nested under Host + +[options="header"] +|===== +| Nested fields | Description + +// =============================================================== + +| http://localhost:8000/ecs-geo.html[host.geo.*] +| Fields describing a location. + +// =============================================================== + +| http://localhost:8000/ecs-os.html[host.os.*] +| OS fields contain information about the operating system. + +// =============================================================== + +| http://localhost:8000/ecs-user.html[host.user.*] +| Fields to describe the user relevant to the event. + +// =============================================================== + +|===== + [[ecs-user]] === User fields The user fields describe information about the user that is relevant to the event. Fields can have one entry or multiple entries. If a user has more than one id, provide an array that includes all of them. -==== Fields summary +==== User Fields [options="header"] |===== @@ -3470,13 +4120,39 @@ type: keyword |===== +[[ecs-user-nestings]] +==== Can be nested under Host + +[options="header"] +|===== +| Nested fields | Description + +// =============================================================== + +| http://localhost:8000/ecs-geo.html[host.geo.*] +| Fields describing a location. + +// =============================================================== + +| http://localhost:8000/ecs-os.html[host.os.*] +| OS fields contain information about the operating system. + +// =============================================================== + +| http://localhost:8000/ecs-user.html[host.user.*] +| Fields to describe the user relevant to the event. + +// =============================================================== + +|===== + [[ecs-user_agent]] === User agent fields The user_agent fields normally come from a browser request. They often show up in web service logs coming from the parsed user agent string. -==== Fields summary +==== User agent Fields [options="header"] |===== @@ -3595,3 +4271,30 @@ example: `4.4.0-112-generic` // =============================================================== |===== + +[[ecs-user_agent-nestings]] +==== Can be nested under Host + +[options="header"] +|===== +| Nested fields | Description + +// =============================================================== + +| http://localhost:8000/ecs-geo.html[host.geo.*] +| Fields describing a location. + +// =============================================================== + +| http://localhost:8000/ecs-os.html[host.os.*] +| OS fields contain information about the operating system. + +// =============================================================== + +| http://localhost:8000/ecs-user.html[host.user.*] +| Fields to describe the user relevant to the event. + +// =============================================================== + +|===== + \ No newline at end of file diff --git a/scripts/generators/asciidoc_fields.py b/scripts/generators/asciidoc_fields.py index 6ad487f409..9cceddc7bb 100644 --- a/scripts/generators/asciidoc_fields.py +++ b/scripts/generators/asciidoc_fields.py @@ -66,6 +66,37 @@ def render_fieldset(fieldset): table += render_field_summary_row(field) table += table_footer() + + table += ''' +[[ecs-{fieldset_name}-nestings]] +==== Can be nested under Host + +[options="header"] +|===== +| Nested fields | Description + +// =============================================================== + +| http://localhost:8000/ecs-geo.html[host.geo.*] +| Fields describing a location. + +// =============================================================== + +| http://localhost:8000/ecs-os.html[host.os.*] +| OS fields contain information about the operating system. + +// =============================================================== + +| http://localhost:8000/ecs-user.html[host.user.*] +| Fields to describe the user relevant to the event. + +// =============================================================== + +|===== + '''.format( + fieldset_name=fieldset['name'] + ) + return table @@ -145,7 +176,7 @@ def fieldset_summary_header(): {fieldset_description} -==== Fields summary +==== {fieldset_title} Fields [options="header"] |===== From c66908c9ea3392da9dc5b0a81573eb3142ffd84d Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Fri, 1 Mar 2019 16:19:32 -0500 Subject: [PATCH 17/28] A few renames to make room for the nestings table --- scripts/generators/asciidoc_fields.py | 49 ++++++++++----------------- 1 file changed, 17 insertions(+), 32 deletions(-) diff --git a/scripts/generators/asciidoc_fields.py b/scripts/generators/asciidoc_fields.py index 9cceddc7bb..32268287c8 100644 --- a/scripts/generators/asciidoc_fields.py +++ b/scripts/generators/asciidoc_fields.py @@ -24,6 +24,14 @@ def sorted_by_keys(dict, sort_keys): return list(map(lambda t: t[-1], sorted(tuples))) +def save_asciidoc(file, text): + open_mode = "wb" + if sys.version_info >= (3, 0): + open_mode = "w" + with open(file, open_mode) as outfile: + outfile.write(text) + + # Rendering # Field Index @@ -56,14 +64,14 @@ def page_field_details(ecs_nested): def render_fieldset(fieldset): - table = fieldset_summary_header().format( + table = field_details_table_header().format( fieldset_name=fieldset['name'], fieldset_description=fieldset['description'], fieldset_title=fieldset['title'] ) for field in sorted_by_keys(fieldset['fields'], 'order'): - table += render_field_summary_row(field) + table += render_field_details_row(field) table += table_footer() @@ -100,11 +108,11 @@ def render_fieldset(fieldset): return table -def render_field_summary_row(field): +def render_field_details_row(field): example = '' if 'example' in field: example = "example: `{}`".format(str(field['example'])) - field_text = field_row().format( + field_text = field_details_row().format( field_flat_name=field['flat_name'], field_description=field['description'], field_example=example, @@ -114,20 +122,6 @@ def render_field_summary_row(field): return field_text -def render_field_details(field): - example = '' - if 'example' in field: - example = "example: `{}`".format(str(field['example'])) - field_text = field_details().format( - field_name=field['name'], - field_flat_name=field['flat_name'], - field_description=field['description'], - field_example=example, - field_level=field['level'], - field_type=field['type'], - ) - return field_text - # Templates @@ -166,10 +160,12 @@ def index_footer(): ''' -# Field Details +# Field Details Page + +# Main Fields Table -def fieldset_summary_header(): +def field_details_table_header(): return ''' [[ecs-{fieldset_name}]] === {fieldset_title} fields @@ -186,7 +182,7 @@ def fieldset_summary_header(): ''' -def field_row(): +def field_details_row(): return ''' | {field_flat_name} | {field_description} @@ -199,14 +195,3 @@ def field_row(): // =============================================================== ''' - - -# File - - -def save_asciidoc(file, text): - open_mode = "wb" - if sys.version_info >= (3, 0): - open_mode = "w" - with open(file, open_mode) as outfile: - outfile.write(text) From a1084ca37d9dcd0d4c3239ae54ac738d5ea255c6 Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Fri, 1 Mar 2019 16:49:10 -0500 Subject: [PATCH 18/28] Start structuring the code to display nestings. The same 3 hardcoded nestings are rendered on each field set page :-) --- docs/field-details.asciidoc | 213 +++++++++++++++++++------- scripts/generators/asciidoc_fields.py | 99 ++++++++---- 2 files changed, 225 insertions(+), 87 deletions(-) diff --git a/docs/field-details.asciidoc b/docs/field-details.asciidoc index eae4076693..8f9bc05927 100644 --- a/docs/field-details.asciidoc +++ b/docs/field-details.asciidoc @@ -66,7 +66,7 @@ example: `Hello World` |===== [[ecs-base-nestings]] -==== Can be nested under Host +==== Can be nested under Base [options="header"] |===== @@ -74,23 +74,27 @@ example: `Hello World` // =============================================================== + | http://localhost:8000/ecs-geo.html[host.geo.*] | Fields describing a location. // =============================================================== + | http://localhost:8000/ecs-os.html[host.os.*] | OS fields contain information about the operating system. // =============================================================== + | http://localhost:8000/ecs-user.html[host.user.*] | Fields to describe the user relevant to the event. // =============================================================== + |===== - + [[ecs-agent]] === Agent fields @@ -168,7 +172,7 @@ example: `8a4f500f` |===== [[ecs-agent-nestings]] -==== Can be nested under Host +==== Can be nested under Agent [options="header"] |===== @@ -176,23 +180,27 @@ example: `8a4f500f` // =============================================================== + | http://localhost:8000/ecs-geo.html[host.geo.*] | Fields describing a location. // =============================================================== + | http://localhost:8000/ecs-os.html[host.os.*] | OS fields contain information about the operating system. // =============================================================== + | http://localhost:8000/ecs-user.html[host.user.*] | Fields to describe the user relevant to the event. // =============================================================== + |===== - + [[ecs-client]] === Client fields @@ -458,7 +466,7 @@ example: `boston-dc` |===== [[ecs-client-nestings]] -==== Can be nested under Host +==== Can be nested under Client [options="header"] |===== @@ -466,23 +474,27 @@ example: `boston-dc` // =============================================================== + | http://localhost:8000/ecs-geo.html[host.geo.*] | Fields describing a location. // =============================================================== + | http://localhost:8000/ecs-os.html[host.os.*] | OS fields contain information about the operating system. // =============================================================== + | http://localhost:8000/ecs-user.html[host.user.*] | Fields to describe the user relevant to the event. // =============================================================== + |===== - + [[ecs-cloud]] === Cloud fields @@ -577,7 +589,7 @@ example: `666777888999` |===== [[ecs-cloud-nestings]] -==== Can be nested under Host +==== Can be nested under Cloud [options="header"] |===== @@ -585,23 +597,27 @@ example: `666777888999` // =============================================================== + | http://localhost:8000/ecs-geo.html[host.geo.*] | Fields describing a location. // =============================================================== + | http://localhost:8000/ecs-os.html[host.os.*] | OS fields contain information about the operating system. // =============================================================== + | http://localhost:8000/ecs-user.html[host.user.*] | Fields to describe the user relevant to the event. // =============================================================== + |===== - + [[ecs-container]] === Container fields @@ -685,7 +701,7 @@ type: object |===== [[ecs-container-nestings]] -==== Can be nested under Host +==== Can be nested under Container [options="header"] |===== @@ -693,23 +709,27 @@ type: object // =============================================================== + | http://localhost:8000/ecs-geo.html[host.geo.*] | Fields describing a location. // =============================================================== + | http://localhost:8000/ecs-os.html[host.os.*] | OS fields contain information about the operating system. // =============================================================== + | http://localhost:8000/ecs-user.html[host.user.*] | Fields to describe the user relevant to the event. // =============================================================== + |===== - + [[ecs-destination]] === Destination fields @@ -974,7 +994,7 @@ example: `boston-dc` |===== [[ecs-destination-nestings]] -==== Can be nested under Host +==== Can be nested under Destination [options="header"] |===== @@ -982,23 +1002,27 @@ example: `boston-dc` // =============================================================== + | http://localhost:8000/ecs-geo.html[host.geo.*] | Fields describing a location. // =============================================================== + | http://localhost:8000/ecs-os.html[host.os.*] | OS fields contain information about the operating system. // =============================================================== + | http://localhost:8000/ecs-user.html[host.user.*] | Fields to describe the user relevant to the event. // =============================================================== + |===== - + [[ecs-ecs]] === ECS fields @@ -1027,7 +1051,7 @@ example: `1.0.0` |===== [[ecs-ecs-nestings]] -==== Can be nested under Host +==== Can be nested under ECS [options="header"] |===== @@ -1035,23 +1059,27 @@ example: `1.0.0` // =============================================================== + | http://localhost:8000/ecs-geo.html[host.geo.*] | Fields describing a location. // =============================================================== + | http://localhost:8000/ecs-os.html[host.os.*] | OS fields contain information about the operating system. // =============================================================== + | http://localhost:8000/ecs-user.html[host.user.*] | Fields to describe the user relevant to the event. // =============================================================== + |===== - + [[ecs-error]] === Error fields @@ -1102,7 +1130,7 @@ type: keyword |===== [[ecs-error-nestings]] -==== Can be nested under Host +==== Can be nested under Error [options="header"] |===== @@ -1110,23 +1138,27 @@ type: keyword // =============================================================== + | http://localhost:8000/ecs-geo.html[host.geo.*] | Fields describing a location. // =============================================================== + | http://localhost:8000/ecs-os.html[host.os.*] | OS fields contain information about the operating system. // =============================================================== + | http://localhost:8000/ecs-user.html[host.user.*] | Fields to describe the user relevant to the event. // =============================================================== + |===== - + [[ecs-event]] === Event fields @@ -1356,7 +1388,7 @@ type: float |===== [[ecs-event-nestings]] -==== Can be nested under Host +==== Can be nested under Event [options="header"] |===== @@ -1364,23 +1396,27 @@ type: float // =============================================================== + | http://localhost:8000/ecs-geo.html[host.geo.*] | Fields describing a location. // =============================================================== + | http://localhost:8000/ecs-os.html[host.os.*] | OS fields contain information about the operating system. // =============================================================== + | http://localhost:8000/ecs-user.html[host.user.*] | Fields to describe the user relevant to the event. // =============================================================== + |===== - + [[ecs-file]] === File fields @@ -1553,7 +1589,7 @@ type: date |===== [[ecs-file-nestings]] -==== Can be nested under Host +==== Can be nested under File [options="header"] |===== @@ -1561,23 +1597,27 @@ type: date // =============================================================== + | http://localhost:8000/ecs-geo.html[host.geo.*] | Fields describing a location. // =============================================================== + | http://localhost:8000/ecs-os.html[host.os.*] | OS fields contain information about the operating system. // =============================================================== + | http://localhost:8000/ecs-user.html[host.user.*] | Fields to describe the user relevant to the event. // =============================================================== + |===== - + [[ecs-geo]] === Geo fields @@ -1685,7 +1725,7 @@ example: `boston-dc` |===== [[ecs-geo-nestings]] -==== Can be nested under Host +==== Can be nested under Geo [options="header"] |===== @@ -1693,23 +1733,27 @@ example: `boston-dc` // =============================================================== + | http://localhost:8000/ecs-geo.html[host.geo.*] | Fields describing a location. // =============================================================== + | http://localhost:8000/ecs-os.html[host.os.*] | OS fields contain information about the operating system. // =============================================================== + | http://localhost:8000/ecs-user.html[host.user.*] | Fields to describe the user relevant to the event. // =============================================================== + |===== - + [[ecs-group]] === Group fields @@ -1748,7 +1792,7 @@ type: keyword |===== [[ecs-group-nestings]] -==== Can be nested under Host +==== Can be nested under Group [options="header"] |===== @@ -1756,23 +1800,27 @@ type: keyword // =============================================================== + | http://localhost:8000/ecs-geo.html[host.geo.*] | Fields describing a location. // =============================================================== + | http://localhost:8000/ecs-os.html[host.os.*] | OS fields contain information about the operating system. // =============================================================== + | http://localhost:8000/ecs-user.html[host.user.*] | Fields to describe the user relevant to the event. // =============================================================== + |===== - + [[ecs-host]] === Host fields @@ -2114,23 +2162,27 @@ example: `boston-dc` // =============================================================== + | http://localhost:8000/ecs-geo.html[host.geo.*] | Fields describing a location. // =============================================================== + | http://localhost:8000/ecs-os.html[host.os.*] | OS fields contain information about the operating system. // =============================================================== + | http://localhost:8000/ecs-user.html[host.user.*] | Fields to describe the user relevant to the event. // =============================================================== + |===== - + [[ecs-http]] === HTTP fields @@ -2258,7 +2310,7 @@ example: `887` |===== [[ecs-http-nestings]] -==== Can be nested under Host +==== Can be nested under HTTP [options="header"] |===== @@ -2266,23 +2318,27 @@ example: `887` // =============================================================== + | http://localhost:8000/ecs-geo.html[host.geo.*] | Fields describing a location. // =============================================================== + | http://localhost:8000/ecs-os.html[host.os.*] | OS fields contain information about the operating system. // =============================================================== + | http://localhost:8000/ecs-user.html[host.user.*] | Fields to describe the user relevant to the event. // =============================================================== + |===== - + [[ecs-log]] === Log fields @@ -2324,7 +2380,7 @@ example: `Sep 19 08:26:10 localhost My log` |===== [[ecs-log-nestings]] -==== Can be nested under Host +==== Can be nested under Log [options="header"] |===== @@ -2332,23 +2388,27 @@ example: `Sep 19 08:26:10 localhost My log` // =============================================================== + | http://localhost:8000/ecs-geo.html[host.geo.*] | Fields describing a location. // =============================================================== + | http://localhost:8000/ecs-os.html[host.os.*] | OS fields contain information about the operating system. // =============================================================== + | http://localhost:8000/ecs-user.html[host.user.*] | Fields to describe the user relevant to the event. // =============================================================== + |===== - + [[ecs-network]] === Network fields @@ -2503,7 +2563,7 @@ example: `24` |===== [[ecs-network-nestings]] -==== Can be nested under Host +==== Can be nested under Network [options="header"] |===== @@ -2511,23 +2571,27 @@ example: `24` // =============================================================== + | http://localhost:8000/ecs-geo.html[host.geo.*] | Fields describing a location. // =============================================================== + | http://localhost:8000/ecs-os.html[host.os.*] | OS fields contain information about the operating system. // =============================================================== + | http://localhost:8000/ecs-user.html[host.user.*] | Fields to describe the user relevant to the event. // =============================================================== + |===== - + [[ecs-observer]] === Observer fields @@ -2779,7 +2843,7 @@ example: `boston-dc` |===== [[ecs-observer-nestings]] -==== Can be nested under Host +==== Can be nested under Observer [options="header"] |===== @@ -2787,23 +2851,27 @@ example: `boston-dc` // =============================================================== + | http://localhost:8000/ecs-geo.html[host.geo.*] | Fields describing a location. // =============================================================== + | http://localhost:8000/ecs-os.html[host.os.*] | OS fields contain information about the operating system. // =============================================================== + | http://localhost:8000/ecs-user.html[host.user.*] | Fields to describe the user relevant to the event. // =============================================================== + |===== - + [[ecs-organization]] === Organization fields @@ -2843,7 +2911,7 @@ type: keyword |===== [[ecs-organization-nestings]] -==== Can be nested under Host +==== Can be nested under Organization [options="header"] |===== @@ -2851,23 +2919,27 @@ type: keyword // =============================================================== + | http://localhost:8000/ecs-geo.html[host.geo.*] | Fields describing a location. // =============================================================== + | http://localhost:8000/ecs-os.html[host.os.*] | OS fields contain information about the operating system. // =============================================================== + | http://localhost:8000/ecs-user.html[host.user.*] | Fields to describe the user relevant to the event. // =============================================================== + |===== - + [[ecs-os]] === Operating System fields @@ -2950,7 +3022,7 @@ example: `4.4.0-112-generic` |===== [[ecs-os-nestings]] -==== Can be nested under Host +==== Can be nested under Operating System [options="header"] |===== @@ -2958,23 +3030,27 @@ example: `4.4.0-112-generic` // =============================================================== + | http://localhost:8000/ecs-geo.html[host.geo.*] | Fields describing a location. // =============================================================== + | http://localhost:8000/ecs-os.html[host.os.*] | OS fields contain information about the operating system. // =============================================================== + | http://localhost:8000/ecs-user.html[host.user.*] | Fields to describe the user relevant to the event. // =============================================================== + |===== - + [[ecs-process]] === Process fields @@ -3094,7 +3170,7 @@ example: `/home/alice` |===== [[ecs-process-nestings]] -==== Can be nested under Host +==== Can be nested under Process [options="header"] |===== @@ -3102,23 +3178,27 @@ example: `/home/alice` // =============================================================== + | http://localhost:8000/ecs-geo.html[host.geo.*] | Fields describing a location. // =============================================================== + | http://localhost:8000/ecs-os.html[host.os.*] | OS fields contain information about the operating system. // =============================================================== + | http://localhost:8000/ecs-user.html[host.user.*] | Fields to describe the user relevant to the event. // =============================================================== + |===== - + [[ecs-related]] === Related fields @@ -3148,7 +3228,7 @@ type: ip |===== [[ecs-related-nestings]] -==== Can be nested under Host +==== Can be nested under Related [options="header"] |===== @@ -3156,23 +3236,27 @@ type: ip // =============================================================== + | http://localhost:8000/ecs-geo.html[host.geo.*] | Fields describing a location. // =============================================================== + | http://localhost:8000/ecs-os.html[host.os.*] | OS fields contain information about the operating system. // =============================================================== + | http://localhost:8000/ecs-user.html[host.user.*] | Fields to describe the user relevant to the event. // =============================================================== + |===== - + [[ecs-server]] === Server fields @@ -3438,7 +3522,7 @@ example: `boston-dc` |===== [[ecs-server-nestings]] -==== Can be nested under Host +==== Can be nested under Server [options="header"] |===== @@ -3446,23 +3530,27 @@ example: `boston-dc` // =============================================================== + | http://localhost:8000/ecs-geo.html[host.geo.*] | Fields describing a location. // =============================================================== + | http://localhost:8000/ecs-os.html[host.os.*] | OS fields contain information about the operating system. // =============================================================== + | http://localhost:8000/ecs-user.html[host.user.*] | Fields to describe the user relevant to the event. // =============================================================== + |===== - + [[ecs-service]] === Service fields @@ -3555,7 +3643,7 @@ example: `8a4f500f` |===== [[ecs-service-nestings]] -==== Can be nested under Host +==== Can be nested under Service [options="header"] |===== @@ -3563,23 +3651,27 @@ example: `8a4f500f` // =============================================================== + | http://localhost:8000/ecs-geo.html[host.geo.*] | Fields describing a location. // =============================================================== + | http://localhost:8000/ecs-os.html[host.os.*] | OS fields contain information about the operating system. // =============================================================== + | http://localhost:8000/ecs-user.html[host.user.*] | Fields to describe the user relevant to the event. // =============================================================== + |===== - + [[ecs-source]] === Source fields @@ -3844,7 +3936,7 @@ example: `boston-dc` |===== [[ecs-source-nestings]] -==== Can be nested under Host +==== Can be nested under Source [options="header"] |===== @@ -3852,23 +3944,27 @@ example: `boston-dc` // =============================================================== + | http://localhost:8000/ecs-geo.html[host.geo.*] | Fields describing a location. // =============================================================== + | http://localhost:8000/ecs-os.html[host.os.*] | OS fields contain information about the operating system. // =============================================================== + | http://localhost:8000/ecs-user.html[host.user.*] | Fields to describe the user relevant to the event. // =============================================================== + |===== - + [[ecs-url]] === URL fields @@ -4001,7 +4097,7 @@ type: keyword |===== [[ecs-url-nestings]] -==== Can be nested under Host +==== Can be nested under URL [options="header"] |===== @@ -4009,23 +4105,27 @@ type: keyword // =============================================================== + | http://localhost:8000/ecs-geo.html[host.geo.*] | Fields describing a location. // =============================================================== + | http://localhost:8000/ecs-os.html[host.os.*] | OS fields contain information about the operating system. // =============================================================== + | http://localhost:8000/ecs-user.html[host.user.*] | Fields to describe the user relevant to the event. // =============================================================== + |===== - + [[ecs-user]] === User fields @@ -4121,7 +4221,7 @@ type: keyword |===== [[ecs-user-nestings]] -==== Can be nested under Host +==== Can be nested under User [options="header"] |===== @@ -4129,23 +4229,27 @@ type: keyword // =============================================================== + | http://localhost:8000/ecs-geo.html[host.geo.*] | Fields describing a location. // =============================================================== + | http://localhost:8000/ecs-os.html[host.os.*] | OS fields contain information about the operating system. // =============================================================== + | http://localhost:8000/ecs-user.html[host.user.*] | Fields to describe the user relevant to the event. // =============================================================== + |===== - + [[ecs-user_agent]] === User agent fields @@ -4273,7 +4377,7 @@ example: `4.4.0-112-generic` |===== [[ecs-user_agent-nestings]] -==== Can be nested under Host +==== Can be nested under User agent [options="header"] |===== @@ -4281,20 +4385,23 @@ example: `4.4.0-112-generic` // =============================================================== + | http://localhost:8000/ecs-geo.html[host.geo.*] | Fields describing a location. // =============================================================== + | http://localhost:8000/ecs-os.html[host.os.*] | OS fields contain information about the operating system. // =============================================================== + | http://localhost:8000/ecs-user.html[host.user.*] | Fields to describe the user relevant to the event. // =============================================================== + |===== - \ No newline at end of file diff --git a/scripts/generators/asciidoc_fields.py b/scripts/generators/asciidoc_fields.py index 32268287c8..89fa47e351 100644 --- a/scripts/generators/asciidoc_fields.py +++ b/scripts/generators/asciidoc_fields.py @@ -64,62 +64,68 @@ def page_field_details(ecs_nested): def render_fieldset(fieldset): - table = field_details_table_header().format( + text = field_details_table_header().format( fieldset_name=fieldset['name'], fieldset_description=fieldset['description'], fieldset_title=fieldset['title'] ) for field in sorted_by_keys(fieldset['fields'], 'order'): - table += render_field_details_row(field) + text += render_field_details_row(field) - table += table_footer() + text += table_footer() - table += ''' -[[ecs-{fieldset_name}-nestings]] -==== Can be nested under Host - -[options="header"] -|===== -| Nested fields | Description - -// =============================================================== - -| http://localhost:8000/ecs-geo.html[host.geo.*] -| Fields describing a location. - -// =============================================================== - -| http://localhost:8000/ecs-os.html[host.os.*] -| OS fields contain information about the operating system. - -// =============================================================== - -| http://localhost:8000/ecs-user.html[host.user.*] -| Fields to describe the user relevant to the event. - -// =============================================================== - -|===== - '''.format( - fieldset_name=fieldset['name'] + text += nestings_table_header().format( + fieldset_name=fieldset['name'], + fieldset_title=fieldset['title'] ) - return table + # TODO Un-hardcode + fieldset['nestings'] = [ + { + 'flat_nesting': 'host.geo.*', + 'name':'geo', + 'short': 'Fields describing a location.' + }, + { + 'flat_nesting': 'host.os.*', + 'name':'os', + 'short': 'OS fields contain information about the operating system.' + }, + { + 'flat_nesting': 'host.user.*', + 'name':'user', + 'short': 'Fields to describe the user relevant to the event.' + } + ] + for nesting in fieldset['nestings']: + text += render_nesting_row(nesting) + text += table_footer() + + return text def render_field_details_row(field): example = '' if 'example' in field: example = "example: `{}`".format(str(field['example'])) - field_text = field_details_row().format( + text = field_details_row().format( field_flat_name=field['flat_name'], field_description=field['description'], field_example=example, field_level=field['level'], field_type=field['type'], ) - return field_text + return text + + +def render_nesting_row(nesting): + text = nestings_row().format( + nesting_name=nesting['name'], + flat_nesting=nesting['flat_nesting'], + nesting_short=nesting['short'], + ) + return text # Templates @@ -195,3 +201,28 @@ def field_details_row(): // =============================================================== ''' + + +# Nestings table + +def nestings_table_header(): + return ''' +[[ecs-{fieldset_name}-nestings]] +==== Can be nested under {fieldset_title} + +[options="header"] +|===== +| Nested fields | Description + +// =============================================================== + +''' + +def nestings_row(): + return ''' +| http://localhost:8000/ecs-{nesting_name}.html[{flat_nesting}] +| {nesting_short} + +// =============================================================== + +''' From 10da069ff591b49a368bec5f68483d4f2eb6235a Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Mon, 4 Mar 2019 17:56:49 -0500 Subject: [PATCH 19/28] code format --- scripts/generators/asciidoc_fields.py | 31 ++++++++++++++------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/scripts/generators/asciidoc_fields.py b/scripts/generators/asciidoc_fields.py index 89fa47e351..15b5760e67 100644 --- a/scripts/generators/asciidoc_fields.py +++ b/scripts/generators/asciidoc_fields.py @@ -82,21 +82,21 @@ def render_fieldset(fieldset): # TODO Un-hardcode fieldset['nestings'] = [ - { - 'flat_nesting': 'host.geo.*', - 'name':'geo', - 'short': 'Fields describing a location.' - }, - { - 'flat_nesting': 'host.os.*', - 'name':'os', - 'short': 'OS fields contain information about the operating system.' - }, - { - 'flat_nesting': 'host.user.*', - 'name':'user', - 'short': 'Fields to describe the user relevant to the event.' - } + { + 'flat_nesting': 'host.geo.*', + 'name': 'geo', + 'short': 'Fields describing a location.' + }, + { + 'flat_nesting': 'host.os.*', + 'name': 'os', + 'short': 'OS fields contain information about the operating system.' + }, + { + 'flat_nesting': 'host.user.*', + 'name': 'user', + 'short': 'Fields to describe the user relevant to the event.' + } ] for nesting in fieldset['nestings']: text += render_nesting_row(nesting) @@ -218,6 +218,7 @@ def nestings_table_header(): ''' + def nestings_row(): return ''' | http://localhost:8000/ecs-{nesting_name}.html[{flat_nesting}] From 6f03efb2b10216dc3876d89bde972c88f68a167a Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Fri, 8 Mar 2019 16:08:30 -0500 Subject: [PATCH 20/28] Start moving generic helpers to a helper file --- scripts/ecs_helpers.py | 36 ++++++++++++++++++++++ scripts/generators/beats.py | 38 ++++-------------------- scripts/generators/intermediate_files.py | 10 ++----- 3 files changed, 44 insertions(+), 40 deletions(-) create mode 100644 scripts/ecs_helpers.py diff --git a/scripts/ecs_helpers.py b/scripts/ecs_helpers.py new file mode 100644 index 0000000000..6af5caf190 --- /dev/null +++ b/scripts/ecs_helpers.py @@ -0,0 +1,36 @@ +import yaml + +from collections import OrderedDict + +# Dictionary helpers + + +def dict_copy_keys_ordered(dict, copied_keys): + ordered_dict = OrderedDict() + for key in copied_keys: + if key in dict: + ordered_dict[key] = dict[key] + return ordered_dict + + +def yaml_ordereddict(dumper, data): + # YAML representation of an OrderedDict will be like a dictionary, but + # respecting the order of the dictionary. + # Almost sure it's unndecessary with Python 3. + value = [] + for item_key, item_value in data.items(): + node_key = dumper.represent_data(item_key) + node_value = dumper.represent_data(item_value) + value.append((node_key, node_value)) + return yaml.nodes.MappingNode(u'tag:yaml.org,2002:map', value) + +yaml.add_representer(OrderedDict, yaml_ordereddict) + +# File helpers + + +def yaml_dump(filename, data, preamble=None): + with open(filename, 'w') as outfile: + if preamble: + outfile.write(preamble) + yaml.dump(data, outfile, default_flow_style=False) diff --git a/scripts/generators/beats.py b/scripts/generators/beats.py index aaa5959ecb..94b41ea857 100644 --- a/scripts/generators/beats.py +++ b/scripts/generators/beats.py @@ -1,4 +1,6 @@ +import ecs_helpers import yaml + from collections import OrderedDict @@ -13,7 +15,7 @@ def generate(ecs_nested, ecs_version): continue fieldset = ecs_nested[fieldset_name] - beats_field = dict_copy_keys_ordered(fieldset, allowed_fieldset_keys) + beats_field = ecs_helpers.dict_copy_keys_ordered(fieldset, allowed_fieldset_keys) beats_field['fields'] = fieldset_field_array(fieldset['fields']) beats_fields.append(beats_field) @@ -33,7 +35,7 @@ def fieldset_field_array(source_fields): fields = [] for nested_field_name in source_fields: ecs_field = source_fields[nested_field_name] - beats_field = dict_copy_keys_ordered(ecs_field, allowed_keys) + beats_field = ecs_helpers.dict_copy_keys_ordered(ecs_field, allowed_keys) beats_field['name'] = nested_field_name fields.append(beats_field) return sorted(fields, lambda x, y: cmp(x['name'], y['name'])) @@ -42,37 +44,9 @@ def fieldset_field_array(source_fields): def write_beats_yaml(beats_file, ecs_version): + warning = file_header().format(version=ecs_version) + ecs_helpers.yaml_dump('generated/beats/fields.ecs.yml', [beats_file], preamble=warning) - with open('generated/beats/fields.ecs.yml', 'w') as outfile: - outfile.write(file_header().format(version=ecs_version)) - yaml.dump([beats_file], outfile, default_flow_style=False) - - -def dict_copy_keys_ordered(dict, copied_keys): - ordered_dict = OrderedDict() - for key in copied_keys: - if key in dict: - ordered_dict[key] = dict[key] - return ordered_dict - - -def indent(text, indent, char=' '): - padding = indent * char - return ''.join(padding + line for line in text.splitlines(True)) - -# Rendering - - -def yaml_ordereddict(dumper, data): - value = [] - for item_key, item_value in data.items(): - node_key = dumper.represent_data(item_key) - node_value = dumper.represent_data(item_value) - value.append((node_key, node_value)) - return yaml.nodes.MappingNode(u'tag:yaml.org,2002:map', value) - - -yaml.add_representer(OrderedDict, yaml_ordereddict) # Templates diff --git a/scripts/generators/intermediate_files.py b/scripts/generators/intermediate_files.py index cdf2c46303..3deb97e5c3 100644 --- a/scripts/generators/intermediate_files.py +++ b/scripts/generators/intermediate_files.py @@ -1,9 +1,3 @@ -import yaml - - def generate(ecs_nested, ecs_flat): - with open('generated/ecs/fields_flat.yml', 'w') as outfile: - yaml.dump(ecs_flat, outfile, default_flow_style=False) - - with open('generated/ecs/fields_nested.yml', 'w') as outfile: - yaml.dump(ecs_nested, outfile, default_flow_style=False) + ecs_helpers.yaml_dump('generated/ecs/fields_flat.yml', ecs_flat) + ecs_helpers.yaml_dump('generated/ecs/fields_nested.yml', ecs_nested) From ee81be730da1020de1d50c53a050d28a4e7014f8 Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Mon, 18 Mar 2019 16:01:13 -0400 Subject: [PATCH 21/28] Move more stuff to ecs_helper.py Also, damn Python imports :-) --- scripts/generators/beats.py | 2 +- scripts/{ => generators}/ecs_helpers.py | 7 ++++++ scripts/generators/es_template.py | 14 +++++------- scripts/generators/intermediate_files.py | 3 +++ scripts/tests/test_asciidoc_fields.py | 5 +++++ scripts/tests/test_ecs_helpers.py | 27 ++++++++++++++++++++++++ scripts/tests/test_ecs_spec.py | 6 ++++++ scripts/tests/test_es_template.py | 19 +++++------------ scripts/tests/test_schema_reader.py | 5 +++++ 9 files changed, 64 insertions(+), 24 deletions(-) rename scripts/{ => generators}/ecs_helpers.py (87%) create mode 100644 scripts/tests/test_ecs_helpers.py diff --git a/scripts/generators/beats.py b/scripts/generators/beats.py index 94b41ea857..d0f8d4d0b2 100644 --- a/scripts/generators/beats.py +++ b/scripts/generators/beats.py @@ -1,7 +1,7 @@ -import ecs_helpers import yaml from collections import OrderedDict +from generators import ecs_helpers def generate(ecs_nested, ecs_version): diff --git a/scripts/ecs_helpers.py b/scripts/generators/ecs_helpers.py similarity index 87% rename from scripts/ecs_helpers.py rename to scripts/generators/ecs_helpers.py index 6af5caf190..e4059a898f 100644 --- a/scripts/ecs_helpers.py +++ b/scripts/generators/ecs_helpers.py @@ -13,6 +13,12 @@ def dict_copy_keys_ordered(dict, copied_keys): return ordered_dict +def dict_copy_existing_keys(source, destination, keys): + for key in keys: + if key in source: + destination[key] = source[key] + + def yaml_ordereddict(dumper, data): # YAML representation of an OrderedDict will be like a dictionary, but # respecting the order of the dictionary. @@ -24,6 +30,7 @@ def yaml_ordereddict(dumper, data): value.append((node_key, node_value)) return yaml.nodes.MappingNode(u'tag:yaml.org,2002:map', value) + yaml.add_representer(OrderedDict, yaml_ordereddict) # File helpers diff --git a/scripts/generators/es_template.py b/scripts/generators/es_template.py index 12502148b9..62fae3d3c3 100644 --- a/scripts/generators/es_template.py +++ b/scripts/generators/es_template.py @@ -1,6 +1,8 @@ import json import sys +from generators import ecs_helpers + def generate(ecs_flat, ecs_version): field_mappings = {} @@ -32,22 +34,16 @@ def dict_add_nested(dict, nestings, value): dict[current_nesting] = value -def dict_copy_existing_keys(source, destination, keys): - for key in keys: - if key in source: - destination[key] = source[key] - - def entry_for(field): dict = {'type': field['type']} try: if 'index' in field and not field['index']: - dict_copy_existing_keys(field, dict, ['index', 'doc_values']) + ecs_helpers.dict_copy_existing_keys(field, dict, ['index', 'doc_values']) if field['type'] == 'keyword': - dict_copy_existing_keys(field, dict, ['ignore_above']) + ecs_helpers.dict_copy_existing_keys(field, dict, ['ignore_above']) elif field['type'] == 'text': - dict_copy_existing_keys(field, dict, ['norms']) + ecs_helpers.dict_copy_existing_keys(field, dict, ['norms']) except KeyError as ex: print ex, field raise ex diff --git a/scripts/generators/intermediate_files.py b/scripts/generators/intermediate_files.py index 3deb97e5c3..b6a25a1868 100644 --- a/scripts/generators/intermediate_files.py +++ b/scripts/generators/intermediate_files.py @@ -1,3 +1,6 @@ +from generators import ecs_helpers + + def generate(ecs_nested, ecs_flat): ecs_helpers.yaml_dump('generated/ecs/fields_flat.yml', ecs_flat) ecs_helpers.yaml_dump('generated/ecs/fields_nested.yml', ecs_nested) diff --git a/scripts/tests/test_asciidoc_fields.py b/scripts/tests/test_asciidoc_fields.py index dc8161ac36..a1c7431196 100644 --- a/scripts/tests/test_asciidoc_fields.py +++ b/scripts/tests/test_asciidoc_fields.py @@ -1,4 +1,9 @@ +import os +import sys import unittest + +sys.path.append(os.path.join(os.path.dirname(__file__), '..')) + from scripts.generators import asciidoc_fields diff --git a/scripts/tests/test_ecs_helpers.py b/scripts/tests/test_ecs_helpers.py new file mode 100644 index 0000000000..1d7d439257 --- /dev/null +++ b/scripts/tests/test_ecs_helpers.py @@ -0,0 +1,27 @@ +import os +import sys +import unittest + +sys.path.append(os.path.join(os.path.dirname(__file__), '..')) + +from scripts.generators import ecs_helpers + + +class TestECSHelpers(unittest.TestCase): + # dict_copy_existing_keys + + def test_dict_copy_existing_keys(self): + source = {'key1': 'value1'} + destination = {} + ecs_helpers.dict_copy_existing_keys(source, destination, ['key1', 'missingkey']) + self.assertEqual(destination, {'key1': 'value1'}) + + def test_dict_copy_existing_keys_overwrites(self): + source = {'key1': 'new_value'} + destination = {'key1': 'overwritten', 'untouched': 'untouched'} + ecs_helpers.dict_copy_existing_keys(source, destination, ['key1', 'untouched']) + self.assertEqual(destination, {'key1': 'new_value', 'untouched': 'untouched'}) + + +if __name__ == '__main__': + unittest.main() diff --git a/scripts/tests/test_ecs_spec.py b/scripts/tests/test_ecs_spec.py index b69e5e11fe..51dc806233 100644 --- a/scripts/tests/test_ecs_spec.py +++ b/scripts/tests/test_ecs_spec.py @@ -1,6 +1,12 @@ +import os +import sys import unittest + +sys.path.append(os.path.join(os.path.dirname(__file__), '..')) + from scripts import schema_reader + (nested, flat) = schema_reader.load_ecs() diff --git a/scripts/tests/test_es_template.py b/scripts/tests/test_es_template.py index fb211fe20c..2c0dcb1062 100644 --- a/scripts/tests/test_es_template.py +++ b/scripts/tests/test_es_template.py @@ -1,4 +1,9 @@ +import os +import sys import unittest + +sys.path.append(os.path.join(os.path.dirname(__file__), '..')) + from scripts.generators import es_template @@ -25,20 +30,6 @@ def test_add_siblings(self): 'key3': 'value3' }) - # dict_copy_existing_keys - - def test_dict_copy_existing_keys(self): - source = {'key1': 'value1'} - destination = {} - es_template.dict_copy_existing_keys(source, destination, ['key1', 'missingkey']) - self.assertEqual(destination, {'key1': 'value1'}) - - def test_dict_copy_existing_keys_overwrites(self): - source = {'key1': 'new_value'} - destination = {'key1': 'overwritten', 'untouched': 'untouched'} - es_template.dict_copy_existing_keys(source, destination, ['key1', 'untouched']) - self.assertEqual(destination, {'key1': 'new_value', 'untouched': 'untouched'}) - if __name__ == '__main__': unittest.main() diff --git a/scripts/tests/test_schema_reader.py b/scripts/tests/test_schema_reader.py index f5fca996a9..1796308cea 100644 --- a/scripts/tests/test_schema_reader.py +++ b/scripts/tests/test_schema_reader.py @@ -1,4 +1,9 @@ +import os +import sys import unittest + +sys.path.append(os.path.join(os.path.dirname(__file__), '..')) + from scripts import schema_reader From 053edf3e7a782bd2b7fa3f9299027f3abadbdc44 Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Mon, 18 Mar 2019 16:12:37 -0400 Subject: [PATCH 22/28] Make sure make fmt doesn't mess up my imports in tests --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 1105bbae0b..280c5880f6 100644 --- a/Makefile +++ b/Makefile @@ -65,7 +65,7 @@ fields_legacy: # Format code and files in the repo. .PHONY: fmt fmt: ve - $(FIND) -name '*.py' -exec build/ve/bin/autopep8 --in-place --max-line-length 120 {} \; + $(FIND) -name '*.py' -exec build/ve/bin/autopep8 --ignore E402 --in-place --max-line-length 120 {} \; go get golang.org/x/tools/cmd/goimports goimports -w -l -local github.com/elastic $(shell $(FIND) -name '*.go') From 9ffb0634007926e17bad1f12ca137a2b237ff132 Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Mon, 18 Mar 2019 16:32:45 -0400 Subject: [PATCH 23/28] Move another helper to the pile --- scripts/generators/asciidoc_fields.py | 24 +++----------- scripts/generators/ecs_helpers.py | 16 ++++++++++ scripts/tests/test_asciidoc_fields.py | 46 --------------------------- scripts/tests/test_ecs_helpers.py | 34 ++++++++++++++++++++ 4 files changed, 55 insertions(+), 65 deletions(-) delete mode 100644 scripts/tests/test_asciidoc_fields.py diff --git a/scripts/generators/asciidoc_fields.py b/scripts/generators/asciidoc_fields.py index 15b5760e67..aaa9adf83f 100644 --- a/scripts/generators/asciidoc_fields.py +++ b/scripts/generators/asciidoc_fields.py @@ -1,5 +1,7 @@ import sys +from generators import ecs_helpers + def generate(ecs_nested, ecs_version): save_asciidoc('docs/fields.asciidoc', page_field_index(ecs_nested)) @@ -8,22 +10,6 @@ def generate(ecs_nested, ecs_version): # Helpers -def sorted_by_keys(dict, sort_keys): - if not isinstance(sort_keys, list): - sort_keys = [sort_keys] - tuples = [] - for key in dict: - nested = dict[key] - - sort_criteria = [] - for sort_key in sort_keys: - sort_criteria.append(nested[sort_key]) - sort_criteria.append(nested) - tuples.append(sort_criteria) - - return list(map(lambda t: t[-1], sorted(tuples))) - - def save_asciidoc(file, text): open_mode = "wb" if sys.version_info >= (3, 0): @@ -39,7 +25,7 @@ def save_asciidoc(file, text): def page_field_index(ecs_nested): page_text = index_header() - for fieldset in sorted_by_keys(ecs_nested, ['group', 'name']): + for fieldset in ecs_helpers.dict_sorted_by_keys(ecs_nested, ['group', 'name']): page_text += render_field_index_row(fieldset) page_text += table_footer() page_text += index_footer() @@ -58,7 +44,7 @@ def render_field_index_row(fieldset): def page_field_details(ecs_nested): page_text = '' - for fieldset in sorted_by_keys(ecs_nested, ['group', 'name']): + for fieldset in ecs_helpers.dict_sorted_by_keys(ecs_nested, ['group', 'name']): page_text += render_fieldset(fieldset) return page_text @@ -70,7 +56,7 @@ def render_fieldset(fieldset): fieldset_title=fieldset['title'] ) - for field in sorted_by_keys(fieldset['fields'], 'order'): + for field in ecs_helpers.dict_sorted_by_keys(fieldset['fields'], 'order'): text += render_field_details_row(field) text += table_footer() diff --git a/scripts/generators/ecs_helpers.py b/scripts/generators/ecs_helpers.py index e4059a898f..75adb8a096 100644 --- a/scripts/generators/ecs_helpers.py +++ b/scripts/generators/ecs_helpers.py @@ -19,6 +19,22 @@ def dict_copy_existing_keys(source, destination, keys): destination[key] = source[key] +def dict_sorted_by_keys(dict, sort_keys): + if not isinstance(sort_keys, list): + sort_keys = [sort_keys] + tuples = [] + for key in dict: + nested = dict[key] + + sort_criteria = [] + for sort_key in sort_keys: + sort_criteria.append(nested[sort_key]) + sort_criteria.append(nested) + tuples.append(sort_criteria) + + return list(map(lambda t: t[-1], sorted(tuples))) + + def yaml_ordereddict(dumper, data): # YAML representation of an OrderedDict will be like a dictionary, but # respecting the order of the dictionary. diff --git a/scripts/tests/test_asciidoc_fields.py b/scripts/tests/test_asciidoc_fields.py deleted file mode 100644 index a1c7431196..0000000000 --- a/scripts/tests/test_asciidoc_fields.py +++ /dev/null @@ -1,46 +0,0 @@ -import os -import sys -import unittest - -sys.path.append(os.path.join(os.path.dirname(__file__), '..')) - -from scripts.generators import asciidoc_fields - - -class TestGeneratorsAsciidocFields(unittest.TestCase): - - def test_sorted_by_one_key(self): - dict = { - '@timestamp': {'order': 0, 'name': '@timestamp'}, - 'message': {'order': 3, 'name': 'message'}, - 'labels': {'order': 1, 'name': 'labels'}, - 'tags': {'order': 2, 'name': 'tags'} - } - expected = [ - {'order': 0, 'name': '@timestamp'}, - {'order': 1, 'name': 'labels'}, - {'order': 2, 'name': 'tags'}, - {'order': 3, 'name': 'message'} - ] - result = asciidoc_fields.sorted_by_keys(dict, 'order') - self.assertEqual(result, expected) - result = asciidoc_fields.sorted_by_keys(dict, ['order']) - self.assertEqual(result, expected) - - def test_sorted_by_multiple_keys(self): - dict = { - 'cloud': {'group': 2, 'name': 'cloud'}, - 'agent': {'group': 2, 'name': 'agent'}, - 'base': {'group': 1, 'name': 'base'}, - } - expected = [ - {'group': 1, 'name': 'base'}, - {'group': 2, 'name': 'agent'}, - {'group': 2, 'name': 'cloud'} - ] - result = asciidoc_fields.sorted_by_keys(dict, ['group', 'name']) - self.assertEqual(result, expected) - - -if __name__ == '__main__': - unittest.main() diff --git a/scripts/tests/test_ecs_helpers.py b/scripts/tests/test_ecs_helpers.py index 1d7d439257..f13f7f7db6 100644 --- a/scripts/tests/test_ecs_helpers.py +++ b/scripts/tests/test_ecs_helpers.py @@ -22,6 +22,40 @@ def test_dict_copy_existing_keys_overwrites(self): ecs_helpers.dict_copy_existing_keys(source, destination, ['key1', 'untouched']) self.assertEqual(destination, {'key1': 'new_value', 'untouched': 'untouched'}) + # dict_sorted_by_keys + + def test_sorted_by_one_key(self): + dict = { + '@timestamp': {'order': 0, 'name': '@timestamp'}, + 'message': {'order': 3, 'name': 'message'}, + 'labels': {'order': 1, 'name': 'labels'}, + 'tags': {'order': 2, 'name': 'tags'} + } + expected = [ + {'order': 0, 'name': '@timestamp'}, + {'order': 1, 'name': 'labels'}, + {'order': 2, 'name': 'tags'}, + {'order': 3, 'name': 'message'} + ] + result = ecs_helpers.dict_sorted_by_keys(dict, 'order') + self.assertEqual(result, expected) + result = ecs_helpers.dict_sorted_by_keys(dict, ['order']) + self.assertEqual(result, expected) + + def test_sorted_by_multiple_keys(self): + dict = { + 'cloud': {'group': 2, 'name': 'cloud'}, + 'agent': {'group': 2, 'name': 'agent'}, + 'base': {'group': 1, 'name': 'base'}, + } + expected = [ + {'group': 1, 'name': 'base'}, + {'group': 2, 'name': 'agent'}, + {'group': 2, 'name': 'cloud'} + ] + result = ecs_helpers.dict_sorted_by_keys(dict, ['group', 'name']) + self.assertEqual(result, expected) + if __name__ == '__main__': unittest.main() From 6336907785077db76b626fed3394485ac4fa4000 Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Tue, 19 Mar 2019 10:53:31 -0400 Subject: [PATCH 24/28] remove sort by original order, sort by flat_name instead --- docs/field-details.asciidoc | 1936 ++++++++++++------------- scripts/generators/asciidoc_fields.py | 2 +- 2 files changed, 969 insertions(+), 969 deletions(-) diff --git a/docs/field-details.asciidoc b/docs/field-details.asciidoc index 8f9bc05927..93ff7bcabc 100644 --- a/docs/field-details.asciidoc +++ b/docs/field-details.asciidoc @@ -26,17 +26,6 @@ example: `2016-05-23T08:05:34.853Z` // =============================================================== -| tags -| List of keywords used to tag each event. - -type: keyword - -example: `["production", "env2"]` - -| core - -// =============================================================== - | labels | Custom key/value pairs. Can be used to add meta information to events. Should not contain nested objects. All values are stored as keyword. @@ -63,6 +52,17 @@ example: `Hello World` // =============================================================== +| tags +| List of keywords used to tag each event. + +type: keyword + +example: `["production", "env2"]` + +| core + +// =============================================================== + |===== [[ecs-base-nestings]] @@ -109,12 +109,25 @@ Examples include Beats. Agents may also run on observers. ECS agent.* fields sha // =============================================================== -| agent.version -| Version of the agent. +| agent.ephemeral_id +| Ephemeral identifier of this agent (if one exists). +This id normally changes across restarts, but `agent.id` does not. type: keyword -example: `6.0.0-rc2` +example: `8a4f500f` + +| extended + +// =============================================================== + +| agent.id +| Unique identifier of this agent (if one exists). +Example: For Beats this would be beat.id. + +type: keyword + +example: `8a4f500d` | core @@ -145,30 +158,17 @@ example: `filebeat` // =============================================================== -| agent.id -| Unique identifier of this agent (if one exists). -Example: For Beats this would be beat.id. +| agent.version +| Version of the agent. type: keyword -example: `8a4f500d` +example: `6.0.0-rc2` | core // =============================================================== -| agent.ephemeral_id -| Ephemeral identifier of this agent (if one exists). -This id normally changes across restarts, but `agent.id` does not. - -type: keyword - -example: `8a4f500f` - -| extended - -// =============================================================== - |===== [[ecs-agent-nestings]] @@ -228,19 +228,19 @@ type: keyword // =============================================================== -| client.geo.location -| Longitude and latitude. +| client.bytes +| Bytes sent from the client to the server. -type: geo_point +type: long -example: `{ "lon": -73.614830, "lat": 45.505918 }` +example: `184` | core // =============================================================== -| client.user.id -| One or multiple unique identifiers of the user. +| client.domain +| Client domain. type: keyword @@ -250,99 +250,101 @@ type: keyword // =============================================================== -| client.user.group.id -| Unique identifier for the group on the system/platform. +| client.geo.city_name +| City name. type: keyword +example: `Montreal` - -| extended +| core // =============================================================== -| client.ip -| IP address of the client. -Can be one or multiple IPv4 or IPv6 addresses. - -type: ip +| client.geo.continent_name +| Name of the continent. +type: keyword +example: `North America` | core // =============================================================== -| client.user.group.name -| Name of the group. +| client.geo.country_iso_code +| Country ISO code. type: keyword +example: `CA` - -| extended +| core // =============================================================== -| client.geo.continent_name -| Name of the continent. +| client.geo.country_name +| Country name. type: keyword -example: `North America` +example: `Canada` | core // =============================================================== -| client.user.name -| Short name or login of the user. +| client.geo.location +| Longitude and latitude. -type: keyword +type: geo_point -example: `albert` +example: `{ "lon": -73.614830, "lat": 45.505918 }` | core // =============================================================== -| client.port -| Port of the client. - -type: long +| client.geo.name +| User-defined description of a location, at the level of granularity they care about. +Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. +Not typically used in automated geolocation. +type: keyword +example: `boston-dc` -| core +| extended // =============================================================== -| client.geo.country_name -| Country name. +| client.geo.region_iso_code +| Region ISO code. type: keyword -example: `Canada` +example: `CA-QC` | core // =============================================================== -| client.user.full_name -| User's full name, if available. +| client.geo.region_name +| Region name. type: keyword -example: `Albert Einstein` +example: `Quebec` -| extended +| core // =============================================================== -| client.mac -| MAC address of the client. +| client.ip +| IP address of the client. +Can be one or multiple IPv4 or IPv6 addresses. -type: keyword +type: ip @@ -350,32 +352,32 @@ type: keyword // =============================================================== -| client.user.email -| User email address. +| client.mac +| MAC address of the client. type: keyword -| extended +| core // =============================================================== -| client.geo.region_name -| Region name. +| client.packets +| Packets sent from the client to the server. -type: keyword +type: long -example: `Quebec` +example: `12` | core // =============================================================== -| client.domain -| Client domain. +| client.port +| Port of the client. -type: keyword +type: long @@ -383,9 +385,8 @@ type: keyword // =============================================================== -| client.user.hash -| Unique user hash to correlate information for a user in anonymized form. -Useful if `user.id` or `user.name` contain confidential information and cannot be used. +| client.user.email +| User email address. type: keyword @@ -395,71 +396,70 @@ type: keyword // =============================================================== -| client.geo.city_name -| City name. +| client.user.full_name +| User's full name, if available. type: keyword -example: `Montreal` +example: `Albert Einstein` -| core +| extended // =============================================================== -| client.bytes -| Bytes sent from the client to the server. +| client.user.group.id +| Unique identifier for the group on the system/platform. -type: long +type: keyword -example: `184` -| core + +| extended // =============================================================== -| client.geo.country_iso_code -| Country ISO code. +| client.user.group.name +| Name of the group. type: keyword -example: `CA` -| core + +| extended // =============================================================== -| client.packets -| Packets sent from the client to the server. +| client.user.hash +| Unique user hash to correlate information for a user in anonymized form. +Useful if `user.id` or `user.name` contain confidential information and cannot be used. -type: long +type: keyword -example: `12` -| core + +| extended // =============================================================== -| client.geo.region_iso_code -| Region ISO code. +| client.user.id +| One or multiple unique identifiers of the user. type: keyword -example: `CA-QC` + | core // =============================================================== -| client.geo.name -| User-defined description of a location, at the level of granularity they care about. -Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. -Not typically used in automated geolocation. +| client.user.name +| Short name or login of the user. type: keyword -example: `boston-dc` +example: `albert` -| extended +| core // =============================================================== @@ -508,12 +508,13 @@ Fields related to the cloud or infrastructure the events are coming from. // =============================================================== -| cloud.provider -| Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. +| cloud.account.id +| The cloud account or organization id used to identify different entities in a multi-tenant environment. +Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. type: keyword -example: `aws` +example: `666777888999` | extended @@ -530,17 +531,6 @@ example: `us-east-1c` // =============================================================== -| cloud.region -| Region in which this host is running. - -type: keyword - -example: `us-east-1` - -| extended - -// =============================================================== - | cloud.instance.id | Instance ID of the host machine. @@ -574,26 +564,36 @@ example: `t2.medium` // =============================================================== -| cloud.account.id -| The cloud account or organization id used to identify different entities in a multi-tenant environment. -Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. +| cloud.provider +| Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. type: keyword -example: `666777888999` +example: `aws` | extended // =============================================================== -|===== +| cloud.region +| Region in which this host is running. -[[ecs-cloud-nestings]] -==== Can be nested under Cloud +type: keyword -[options="header"] -|===== -| Nested fields | Description +example: `us-east-1` + +| extended + +// =============================================================== + +|===== + +[[ecs-cloud-nestings]] +==== Can be nested under Cloud + +[options="header"] +|===== +| Nested fields | Description // =============================================================== @@ -632,17 +632,6 @@ These fields help correlate data based containers from any runtime. // =============================================================== -| container.runtime -| Runtime managing this container. - -type: keyword - -example: `docker` - -| extended - -// =============================================================== - | container.id | Unique container id. @@ -672,6 +661,17 @@ type: keyword +| extended + +// =============================================================== + +| container.labels +| Image labels. + +type: object + + + | extended // =============================================================== @@ -687,12 +687,12 @@ type: keyword // =============================================================== -| container.labels -| Image labels. - -type: object +| container.runtime +| Runtime managing this container. +type: keyword +example: `docker` | extended @@ -756,19 +756,19 @@ type: keyword // =============================================================== -| destination.geo.location -| Longitude and latitude. +| destination.bytes +| Bytes sent from the destination to the source. -type: geo_point +type: long -example: `{ "lon": -73.614830, "lat": 45.505918 }` +example: `184` | core // =============================================================== -| destination.user.id -| One or multiple unique identifiers of the user. +| destination.domain +| Destination domain. type: keyword @@ -778,99 +778,101 @@ type: keyword // =============================================================== -| destination.user.group.id -| Unique identifier for the group on the system/platform. +| destination.geo.city_name +| City name. type: keyword +example: `Montreal` - -| extended +| core // =============================================================== -| destination.ip -| IP address of the destination. -Can be one or multiple IPv4 or IPv6 addresses. - -type: ip +| destination.geo.continent_name +| Name of the continent. +type: keyword +example: `North America` | core // =============================================================== -| destination.user.group.name -| Name of the group. +| destination.geo.country_iso_code +| Country ISO code. type: keyword +example: `CA` - -| extended +| core // =============================================================== -| destination.geo.continent_name -| Name of the continent. +| destination.geo.country_name +| Country name. type: keyword -example: `North America` +example: `Canada` | core // =============================================================== -| destination.user.name -| Short name or login of the user. +| destination.geo.location +| Longitude and latitude. -type: keyword +type: geo_point -example: `albert` +example: `{ "lon": -73.614830, "lat": 45.505918 }` | core // =============================================================== -| destination.port -| Port of the destination. - -type: long +| destination.geo.name +| User-defined description of a location, at the level of granularity they care about. +Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. +Not typically used in automated geolocation. +type: keyword +example: `boston-dc` -| core +| extended // =============================================================== -| destination.geo.country_name -| Country name. +| destination.geo.region_iso_code +| Region ISO code. type: keyword -example: `Canada` +example: `CA-QC` | core // =============================================================== -| destination.user.full_name -| User's full name, if available. +| destination.geo.region_name +| Region name. type: keyword -example: `Albert Einstein` +example: `Quebec` -| extended +| core // =============================================================== -| destination.mac -| MAC address of the destination. +| destination.ip +| IP address of the destination. +Can be one or multiple IPv4 or IPv6 addresses. -type: keyword +type: ip @@ -878,32 +880,32 @@ type: keyword // =============================================================== -| destination.user.email -| User email address. +| destination.mac +| MAC address of the destination. type: keyword -| extended +| core // =============================================================== -| destination.geo.region_name -| Region name. +| destination.packets +| Packets sent from the destination to the source. -type: keyword +type: long -example: `Quebec` +example: `12` | core // =============================================================== -| destination.domain -| Destination domain. +| destination.port +| Port of the destination. -type: keyword +type: long @@ -911,9 +913,8 @@ type: keyword // =============================================================== -| destination.user.hash -| Unique user hash to correlate information for a user in anonymized form. -Useful if `user.id` or `user.name` contain confidential information and cannot be used. +| destination.user.email +| User email address. type: keyword @@ -923,71 +924,70 @@ type: keyword // =============================================================== -| destination.geo.city_name -| City name. +| destination.user.full_name +| User's full name, if available. type: keyword -example: `Montreal` +example: `Albert Einstein` -| core +| extended // =============================================================== -| destination.bytes -| Bytes sent from the destination to the source. +| destination.user.group.id +| Unique identifier for the group on the system/platform. -type: long +type: keyword -example: `184` -| core + +| extended // =============================================================== -| destination.geo.country_iso_code -| Country ISO code. +| destination.user.group.name +| Name of the group. type: keyword -example: `CA` -| core + +| extended // =============================================================== -| destination.packets -| Packets sent from the destination to the source. +| destination.user.hash +| Unique user hash to correlate information for a user in anonymized form. +Useful if `user.id` or `user.name` contain confidential information and cannot be used. -type: long +type: keyword -example: `12` -| core + +| extended // =============================================================== -| destination.geo.region_iso_code -| Region ISO code. +| destination.user.id +| One or multiple unique identifiers of the user. type: keyword -example: `CA-QC` + | core // =============================================================== -| destination.geo.name -| User-defined description of a location, at the level of granularity they care about. -Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. -Not typically used in automated geolocation. +| destination.user.name +| Short name or login of the user. type: keyword -example: `boston-dc` +example: `albert` -| extended +| core // =============================================================== @@ -1094,8 +1094,8 @@ Use them for errors that happen while fetching events or in cases where the even // =============================================================== -| error.id -| Unique identifier for the error. +| error.code +| Error code describing the error. type: keyword @@ -1105,10 +1105,10 @@ type: keyword // =============================================================== -| error.message -| Error message. +| error.id +| Unique identifier for the error. -type: text +type: keyword @@ -1116,10 +1116,10 @@ type: text // =============================================================== -| error.code -| Error code describing the error. +| error.message +| Error message. -type: keyword +type: text @@ -1173,29 +1173,18 @@ A log is defined as an event containing details of something that happened. Log // =============================================================== -| event.id -| Unique ID to describe the event. +| event.action +| The action captured by the event. +This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. type: keyword -example: `8a4f500d` +example: `user-password-change` | core // =============================================================== -| event.kind -| The kind of the event. -This gives information about what type of information the event contains, without being specific to the contents of the event. Examples are `event`, `state`, `alarm`. Warning: In future versions of ECS, we plan to provide a list of acceptable values for this field, please use with caution. - -type: keyword - -example: `state` - -| extended - -// =============================================================== - | event.category | Event category. This contains high-level information about the contents of the event. It is more generic than `event.action`, in the sense that typically a category contains multiple actions. Warning: In future versions of ECS, we plan to provide a list of acceptable values for this field, please use with caution. @@ -1208,35 +1197,37 @@ example: `user-management` // =============================================================== -| event.action -| The action captured by the event. -This describes the information in the event. It is more specific than `event.category`. Examples are `group-add`, `process-started`, `file-created`. The value is normally defined by the implementer. +| event.created +| event.created contains the date/time when the event was first read by an agent, or by your pipeline. +This field is distinct from @timestamp in that @timestamp typically contain the time extracted from the original event. +In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. +In case the two timestamps are identical, @timestamp should be used. + +type: date -type: keyword -example: `user-password-change` | core // =============================================================== -| event.outcome -| The outcome of the event. -If the event describes an action, this fields contains the outcome of that action. Examples outcomes are `success` and `failure`. Warning: In future versions of ECS, we plan to provide a list of acceptable values for this field, please use with caution. +| event.dataset +| Name of the dataset. +The concept of a `dataset` (fileset / metricset) is used in Beats as a subset of modules. It contains the information which is currently stored in metricset.name and metricset.module or fileset.name. type: keyword -example: `success` +example: `stats` -| extended +| core // =============================================================== -| event.type -| Reserved for future usage. -Please avoid using this field for user data. +| event.duration +| Duration of the event in nanoseconds. +If event.start and event.end are known this value should be the difference between the end and start time. -type: keyword +type: long @@ -1244,36 +1235,58 @@ type: keyword // =============================================================== -| event.module -| Name of the module this data is coming from. -This information is coming from the modules used in Beats or Logstash. +| event.end +| event.end contains the date when the event ended or when the activity was last observed. + +type: date + + + +| extended + +// =============================================================== + +| event.hash +| Hash (perhaps logstash fingerprint) of raw field to be able to demonstrate log integrity. type: keyword -example: `mysql` +example: `123456789012345678901234567890ABCD` -| core +| extended // =============================================================== -| event.dataset -| Name of the dataset. -The concept of a `dataset` (fileset / metricset) is used in Beats as a subset of modules. It contains the information which is currently stored in metricset.name and metricset.module or fileset.name. +| event.id +| Unique ID to describe the event. type: keyword -example: `stats` +example: `8a4f500d` | core // =============================================================== -| event.severity -| Severity describes the original severity of the event. What the different severity values mean can very different between use cases. It's up to the implementer to make sure severities are consistent across events. +| event.kind +| The kind of the event. +This gives information about what type of information the event contains, without being specific to the contents of the event. Examples are `event`, `state`, `alarm`. Warning: In future versions of ECS, we plan to provide a list of acceptable values for this field, please use with caution. -type: long +type: keyword -example: `7` +example: `state` + +| extended + +// =============================================================== + +| event.module +| Name of the module this data is coming from. +This information is coming from the modules used in Beats or Logstash. + +type: keyword + +example: `mysql` | core @@ -1291,22 +1304,22 @@ example: `Sep 19 08:26:10 host CEF:0|Security| threatmanager|1.0& // =============================================================== -| event.hash -| Hash (perhaps logstash fingerprint) of raw field to be able to demonstrate log integrity. +| event.outcome +| The outcome of the event. +If the event describes an action, this fields contains the outcome of that action. Examples outcomes are `success` and `failure`. Warning: In future versions of ECS, we plan to provide a list of acceptable values for this field, please use with caution. type: keyword -example: `123456789012345678901234567890ABCD` +example: `success` | extended // =============================================================== -| event.duration -| Duration of the event in nanoseconds. -If event.start and event.end are known this value should be the difference between the end and start time. +| event.risk_score +| Risk score or priority of the event (e.g. security solutions). Use your system's original value here. -type: long +type: float @@ -1314,11 +1327,11 @@ type: long // =============================================================== -| event.timezone -| This field should be populated when the event's timestamp does not include timezone information already (e.g. default Syslog timestamps). It's optional otherwise. -Acceptable timezone formats are: a canonical ID (e.g. "Europe/Amsterdam"), abbreviated (e.g. "EST") or an HH:mm differential (e.g. "-05:00"). +| event.risk_score_norm +| Normalized risk score or priority of the event, on a scale of 0 to 100. +This is mainly useful if you use more than one system that assigns risk scores, and you want to see a normalized value across all systems. -type: keyword +type: float @@ -1326,15 +1339,12 @@ type: keyword // =============================================================== -| event.created -| event.created contains the date/time when the event was first read by an agent, or by your pipeline. -This field is distinct from @timestamp in that @timestamp typically contain the time extracted from the original event. -In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. -In case the two timestamps are identical, @timestamp should be used. - -type: date +| event.severity +| Severity describes the original severity of the event. What the different severity values mean can very different between use cases. It's up to the implementer to make sure severities are consistent across events. +type: long +example: `7` | core @@ -1351,10 +1361,11 @@ type: date // =============================================================== -| event.end -| event.end contains the date when the event ended or when the activity was last observed. +| event.timezone +| This field should be populated when the event's timestamp does not include timezone information already (e.g. default Syslog timestamps). It's optional otherwise. +Acceptable timezone formats are: a canonical ID (e.g. "Europe/Amsterdam"), abbreviated (e.g. "EST") or an HH:mm differential (e.g. "-05:00"). -type: date +type: keyword @@ -1362,10 +1373,11 @@ type: date // =============================================================== -| event.risk_score -| Risk score or priority of the event (e.g. security solutions). Use your system's original value here. +| event.type +| Reserved for future usage. +Please avoid using this field for user data. -type: float +type: keyword @@ -1373,18 +1385,6 @@ type: float // =============================================================== -| event.risk_score_norm -| Normalized risk score or priority of the event, on a scale of 0 to 100. -This is mainly useful if you use more than one system that assigns risk scores, and you want to see a normalized value across all systems. - -type: float - - - -| extended - -// =============================================================== - |===== [[ecs-event-nestings]] @@ -1431,10 +1431,10 @@ File objects can be associated with host events, network events, and/or file eve // =============================================================== -| file.path -| Path to the file. +| file.ctime +| Last time file metadata changed. -type: keyword +type: date @@ -1442,8 +1442,8 @@ type: keyword // =============================================================== -| file.target_path -| Target path for symlinks. +| file.device +| Device that is the source of the file. type: keyword @@ -1465,8 +1465,8 @@ example: `png` // =============================================================== -| file.type -| File type (file, dir, or symlink). +| file.gid +| Primary group ID (GID) of the file. type: keyword @@ -1476,8 +1476,8 @@ type: keyword // =============================================================== -| file.device -| Device that is the source of the file. +| file.group +| Primary group name of the file. type: keyword @@ -1498,21 +1498,21 @@ type: keyword // =============================================================== -| file.uid -| The user ID (UID) or security identifier (SID) of the file owner. +| file.mode +| Mode of the file in octal representation. type: keyword - +example: `416` | extended // =============================================================== -| file.owner -| File owner's username. +| file.mtime +| Last time file content was modified. -type: keyword +type: date @@ -1520,8 +1520,8 @@ type: keyword // =============================================================== -| file.gid -| Primary group ID (GID) of the file. +| file.owner +| File owner's username. type: keyword @@ -1531,8 +1531,8 @@ type: keyword // =============================================================== -| file.group -| Primary group name of the file. +| file.path +| Path to the file. type: keyword @@ -1542,21 +1542,21 @@ type: keyword // =============================================================== -| file.mode -| Mode of the file in octal representation. +| file.size +| File size in bytes (field is only added when `type` is `file`). + +type: long -type: keyword -example: `416` | extended // =============================================================== -| file.size -| File size in bytes (field is only added when `type` is `file`). +| file.target_path +| Target path for symlinks. -type: long +type: keyword @@ -1564,10 +1564,10 @@ type: long // =============================================================== -| file.mtime -| Last time file content was modified. +| file.type +| File type (file, dir, or symlink). -type: date +type: keyword @@ -1575,10 +1575,10 @@ type: date // =============================================================== -| file.ctime -| Last time file metadata changed. +| file.uid +| The user ID (UID) or security identifier (SID) of the file owner. -type: date +type: keyword @@ -1632,12 +1632,12 @@ This geolocation information can be derived from techniques such as Geo IP, or b // =============================================================== -| geo.location -| Longitude and latitude. +| geo.city_name +| City name. -type: geo_point +type: keyword -example: `{ "lon": -73.614830, "lat": 45.505918 }` +example: `Montreal` | core @@ -1654,47 +1654,49 @@ example: `North America` // =============================================================== -| geo.country_name -| Country name. +| geo.country_iso_code +| Country ISO code. type: keyword -example: `Canada` +example: `CA` | core // =============================================================== -| geo.region_name -| Region name. +| geo.country_name +| Country name. type: keyword -example: `Quebec` +example: `Canada` | core // =============================================================== -| geo.city_name -| City name. +| geo.location +| Longitude and latitude. -type: keyword +type: geo_point -example: `Montreal` +example: `{ "lon": -73.614830, "lat": 45.505918 }` | core // =============================================================== -| geo.country_iso_code -| Country ISO code. +| geo.name +| User-defined description of a location, at the level of granularity they care about. +Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. +Not typically used in automated geolocation. type: keyword -example: `CA` +example: `boston-dc` -| core +| extended // =============================================================== @@ -1709,16 +1711,14 @@ example: `CA-QC` // =============================================================== -| geo.name -| User-defined description of a location, at the level of granularity they care about. -Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. -Not typically used in automated geolocation. +| geo.region_name +| Region name. type: keyword -example: `boston-dc` +example: `Quebec` -| extended +| core // =============================================================== @@ -1835,113 +1835,114 @@ ECS host.* fields should be populated with details about the host on which the e // =============================================================== -| host.hostname -| Hostname of the host. -It normally contains what the `hostname` command returns on the host machine. +| host.architecture +| Operating system architecture. type: keyword - +example: `x86_64` | core // =============================================================== -| host.geo.location -| Longitude and latitude. +| host.geo.city_name +| City name. -type: geo_point +type: keyword -example: `{ "lon": -73.614830, "lat": 45.505918 }` +example: `Montreal` | core // =============================================================== -| host.user.id -| One or multiple unique identifiers of the user. +| host.geo.continent_name +| Name of the continent. type: keyword - +example: `North America` | core // =============================================================== -| host.user.group.id -| Unique identifier for the group on the system/platform. +| host.geo.country_iso_code +| Country ISO code. type: keyword +example: `CA` - -| extended +| core // =============================================================== -| host.os.platform -| Operating system platform (such centos, ubuntu, windows). +| host.geo.country_name +| Country name. type: keyword -example: `darwin` +example: `Canada` -| extended +| core // =============================================================== -| host.name -| Name of the host. -It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. - -type: keyword +| host.geo.location +| Longitude and latitude. +type: geo_point +example: `{ "lon": -73.614830, "lat": 45.505918 }` | core // =============================================================== -| host.user.group.name -| Name of the group. +| host.geo.name +| User-defined description of a location, at the level of granularity they care about. +Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. +Not typically used in automated geolocation. type: keyword - +example: `boston-dc` | extended // =============================================================== -| host.geo.continent_name -| Name of the continent. +| host.geo.region_iso_code +| Region ISO code. type: keyword -example: `North America` +example: `CA-QC` | core // =============================================================== -| host.os.name -| Operating system name, without the version. +| host.geo.region_name +| Region name. type: keyword -example: `Mac OS X` +example: `Quebec` -| extended +| core // =============================================================== -| host.user.name -| Short name or login of the user. +| host.hostname +| Hostname of the host. +It normally contains what the `hostname` command returns on the host machine. type: keyword -example: `albert` + | core @@ -1960,194 +1961,193 @@ type: keyword // =============================================================== -| host.geo.country_name -| Country name. +| host.ip +| Host ip address. + +type: ip -type: keyword -example: `Canada` | core // =============================================================== -| host.os.full -| Operating system name, including the version or code name. +| host.mac +| Host mac address. type: keyword -example: `Mac OS Mojave` -| extended + +| core // =============================================================== -| host.user.full_name -| User's full name, if available. +| host.name +| Name of the host. +It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. type: keyword -example: `Albert Einstein` -| extended -// =============================================================== +| core -| host.ip -| Host ip address. +// =============================================================== -type: ip +| host.os.family +| OS family (such as redhat, debian, freebsd, windows). +type: keyword +example: `debian` -| core +| extended // =============================================================== -| host.user.email -| User email address. +| host.os.full +| Operating system name, including the version or code name. type: keyword - +example: `Mac OS Mojave` | extended // =============================================================== -| host.os.family -| OS family (such as redhat, debian, freebsd, windows). +| host.os.kernel +| Operating system kernel version as a raw string. type: keyword -example: `debian` +example: `4.4.0-112-generic` | extended // =============================================================== -| host.geo.region_name -| Region name. +| host.os.name +| Operating system name, without the version. type: keyword -example: `Quebec` +example: `Mac OS X` -| core +| extended // =============================================================== -| host.mac -| Host mac address. +| host.os.platform +| Operating system platform (such centos, ubuntu, windows). type: keyword +example: `darwin` - -| core +| extended // =============================================================== -| host.user.hash -| Unique user hash to correlate information for a user in anonymized form. -Useful if `user.id` or `user.name` contain confidential information and cannot be used. +| host.os.version +| Operating system version as a raw string. type: keyword - +example: `10.14.1` | extended // =============================================================== -| host.geo.city_name -| City name. +| host.type +| Type of host. +For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. type: keyword -example: `Montreal` + | core // =============================================================== -| host.os.version -| Operating system version as a raw string. +| host.user.email +| User email address. type: keyword -example: `10.14.1` + | extended // =============================================================== -| host.type -| Type of host. -For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. +| host.user.full_name +| User's full name, if available. type: keyword +example: `Albert Einstein` - -| core +| extended // =============================================================== -| host.geo.country_iso_code -| Country ISO code. +| host.user.group.id +| Unique identifier for the group on the system/platform. type: keyword -example: `CA` -| core + +| extended // =============================================================== -| host.os.kernel -| Operating system kernel version as a raw string. +| host.user.group.name +| Name of the group. type: keyword -example: `4.4.0-112-generic` + | extended // =============================================================== -| host.architecture -| Operating system architecture. +| host.user.hash +| Unique user hash to correlate information for a user in anonymized form. +Useful if `user.id` or `user.name` contain confidential information and cannot be used. type: keyword -example: `x86_64` -| core + +| extended // =============================================================== -| host.geo.region_iso_code -| Region ISO code. +| host.user.id +| One or multiple unique identifiers of the user. type: keyword -example: `CA-QC` + | core // =============================================================== -| host.geo.name -| User-defined description of a location, at the level of granularity they care about. -Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. -Not typically used in automated geolocation. +| host.user.name +| Short name or login of the user. type: keyword -example: `boston-dc` +example: `albert` -| extended +| core // =============================================================== @@ -2196,13 +2196,12 @@ Fields related to HTTP activity. Use the `url` field set to store the url of the // =============================================================== -| http.request.method -| HTTP request method. -The field value must be normalized to lowercase for querying. See the documentation section "Implementing ECS". +| http.request.body.bytes +| Size in bytes of the request body. -type: keyword +type: long -example: `get, post, put` +example: `887` | extended @@ -2219,89 +2218,90 @@ example: `Hello world` // =============================================================== -| http.request.referrer -| Referrer for this HTTP request. +| http.request.bytes +| Total size in bytes of the request (body and headers). -type: keyword +type: long -example: `https://blog.example.com/` +example: `1437` | extended // =============================================================== -| http.response.status_code -| HTTP response status code. +| http.request.method +| HTTP request method. +The field value must be normalized to lowercase for querying. See the documentation section "Implementing ECS". -type: long +type: keyword -example: `404` +example: `get, post, put` | extended // =============================================================== -| http.response.body.content -| The full HTTP response body. +| http.request.referrer +| Referrer for this HTTP request. type: keyword -example: `Hello world` +example: `https://blog.example.com/` | extended // =============================================================== -| http.version -| HTTP version. +| http.response.body.bytes +| Size in bytes of the response body. -type: keyword +type: long -example: `1.1` +example: `887` | extended // =============================================================== -| http.request.bytes -| Total size in bytes of the request (body and headers). +| http.response.body.content +| The full HTTP response body. -type: long +type: keyword -example: `1437` +example: `Hello world` | extended // =============================================================== -| http.request.body.bytes -| Size in bytes of the request body. +| http.response.bytes +| Total size in bytes of the response (body and headers). type: long -example: `887` +example: `1437` | extended // =============================================================== -| http.response.bytes -| Total size in bytes of the response (body and headers). +| http.response.status_code +| HTTP response status code. type: long -example: `1437` +example: `404` | extended // =============================================================== -| http.response.body.bytes -| Size in bytes of the response body. +| http.version +| HTTP version. -type: long +type: keyword -example: `887` +example: `1.1` | extended @@ -2423,76 +2423,42 @@ The network.* fields should be populated with details about the network activity // =============================================================== -| network.name -| Name given by operators to sections of their network. - -type: keyword - -example: `Guest Wifi` - -| extended - -// =============================================================== - -| network.type -| In the OSI Model this would be the Network Layer. ipv4, ipv6, ipsec, pim, etc +| network.application +| A name given to an application level protocol. This can be arbitrarily assigned for things like microservices, but also apply to things like skype, icq, facebook, twitter. This would be used in situations where the vendor or service can be decoded such as from the source/dest IP owners, ports, or wire format. The field value must be normalized to lowercase for querying. See the documentation section "Implementing ECS". type: keyword -example: `ipv4` - -| core - -// =============================================================== - -| network.iana_number -| IANA Protocol Number (https://www.iana.org/assignments/protocol-numbers/protocol-numbers.xhtml). Standardized list of protocols. This aligns well with NetFlow and sFlow related logs which use the IANA Protocol Number. - -type: keyword - -example: `6` +example: `aim` | extended // =============================================================== -| network.transport -| Same as network.iana_number, but instead using the Keyword name of the transport layer (udp, tcp, ipv6-icmp, etc.) -The field value must be normalized to lowercase for querying. See the documentation section "Implementing ECS". +| network.bytes +| Total bytes transferred in both directions. +If `source.bytes` and `destination.bytes` are known, `network.bytes` is their sum. -type: keyword +type: long -example: `tcp` +example: `368` | core // =============================================================== -| network.application -| A name given to an application level protocol. This can be arbitrarily assigned for things like microservices, but also apply to things like skype, icq, facebook, twitter. This would be used in situations where the vendor or service can be decoded such as from the source/dest IP owners, ports, or wire format. -The field value must be normalized to lowercase for querying. See the documentation section "Implementing ECS". +| network.community_id +| A hash of source and destination IPs and ports, as well as the protocol used in a communication. This is a tool-agnostic standard to identify flows. +Learn more at https://github.com/corelight/community-id-spec. type: keyword -example: `aim` +example: `1:hO+sN4H+MG5MY/8hIrXPqc4ZQz0=` | extended // =============================================================== -| network.protocol -| L7 Network protocol name. ex. http, lumberjack, transport protocol. -The field value must be normalized to lowercase for querying. See the documentation section "Implementing ECS". - -type: keyword - -example: `http` - -| core - -// =============================================================== - | network.direction | Direction of the network traffic. Recommended values are: @@ -2524,27 +2490,25 @@ example: `192.1.1.2` // =============================================================== -| network.community_id -| A hash of source and destination IPs and ports, as well as the protocol used in a communication. This is a tool-agnostic standard to identify flows. -Learn more at https://github.com/corelight/community-id-spec. +| network.iana_number +| IANA Protocol Number (https://www.iana.org/assignments/protocol-numbers/protocol-numbers.xhtml). Standardized list of protocols. This aligns well with NetFlow and sFlow related logs which use the IANA Protocol Number. type: keyword -example: `1:hO+sN4H+MG5MY/8hIrXPqc4ZQz0=` +example: `6` | extended // =============================================================== -| network.bytes -| Total bytes transferred in both directions. -If `source.bytes` and `destination.bytes` are known, `network.bytes` is their sum. +| network.name +| Name given by operators to sections of their network. -type: long +type: keyword -example: `368` +example: `Guest Wifi` -| core +| extended // =============================================================== @@ -2560,6 +2524,42 @@ example: `24` // =============================================================== +| network.protocol +| L7 Network protocol name. ex. http, lumberjack, transport protocol. +The field value must be normalized to lowercase for querying. See the documentation section "Implementing ECS". + +type: keyword + +example: `http` + +| core + +// =============================================================== + +| network.transport +| Same as network.iana_number, but instead using the Keyword name of the transport layer (udp, tcp, ipv6-icmp, etc.) +The field value must be normalized to lowercase for querying. See the documentation section "Implementing ECS". + +type: keyword + +example: `tcp` + +| core + +// =============================================================== + +| network.type +| In the OSI Model this would be the Network Layer. ipv4, ipv6, ipsec, pim, etc +The field value must be normalized to lowercase for querying. See the documentation section "Implementing ECS". + +type: keyword + +example: `ipv4` + +| core + +// =============================================================== + |===== [[ecs-network-nestings]] @@ -2606,109 +2606,111 @@ This could be a custom hardware appliance or a server that has been configured t // =============================================================== -| observer.mac -| MAC address of the observer +| observer.geo.city_name +| City name. type: keyword - +example: `Montreal` | core // =============================================================== -| observer.geo.location -| Longitude and latitude. +| observer.geo.continent_name +| Name of the continent. -type: geo_point +type: keyword -example: `{ "lon": -73.614830, "lat": 45.505918 }` +example: `North America` | core // =============================================================== -| observer.os.platform -| Operating system platform (such centos, ubuntu, windows). +| observer.geo.country_iso_code +| Country ISO code. type: keyword -example: `darwin` +example: `CA` -| extended +| core // =============================================================== -| observer.ip -| IP address of the observer. - -type: ip +| observer.geo.country_name +| Country name. +type: keyword +example: `Canada` | core // =============================================================== -| observer.geo.continent_name -| Name of the continent. +| observer.geo.location +| Longitude and latitude. -type: keyword +type: geo_point -example: `North America` +example: `{ "lon": -73.614830, "lat": 45.505918 }` | core // =============================================================== -| observer.os.name -| Operating system name, without the version. +| observer.geo.name +| User-defined description of a location, at the level of granularity they care about. +Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. +Not typically used in automated geolocation. type: keyword -example: `Mac OS X` +example: `boston-dc` | extended // =============================================================== -| observer.hostname -| Hostname of the observer. +| observer.geo.region_iso_code +| Region ISO code. type: keyword - +example: `CA-QC` | core // =============================================================== -| observer.geo.country_name -| Country name. +| observer.geo.region_name +| Region name. type: keyword -example: `Canada` +example: `Quebec` | core // =============================================================== -| observer.os.full -| Operating system name, including the version or code name. +| observer.hostname +| Hostname of the observer. type: keyword -example: `Mac OS Mojave` -| extended + +| core // =============================================================== -| observer.vendor -| observer vendor information. +| observer.ip +| IP address of the observer. -type: keyword +type: ip @@ -2716,89 +2718,89 @@ type: keyword // =============================================================== -| observer.os.family -| OS family (such as redhat, debian, freebsd, windows). +| observer.mac +| MAC address of the observer type: keyword -example: `debian` -| extended + +| core // =============================================================== -| observer.geo.region_name -| Region name. +| observer.os.family +| OS family (such as redhat, debian, freebsd, windows). type: keyword -example: `Quebec` +example: `debian` -| core +| extended // =============================================================== -| observer.version -| Observer version. +| observer.os.full +| Operating system name, including the version or code name. type: keyword +example: `Mac OS Mojave` - -| core +| extended // =============================================================== -| observer.geo.city_name -| City name. +| observer.os.kernel +| Operating system kernel version as a raw string. type: keyword -example: `Montreal` +example: `4.4.0-112-generic` -| core +| extended // =============================================================== -| observer.os.version -| Operating system version as a raw string. +| observer.os.name +| Operating system name, without the version. type: keyword -example: `10.14.1` +example: `Mac OS X` | extended // =============================================================== -| observer.serial_number -| Observer serial number. +| observer.os.platform +| Operating system platform (such centos, ubuntu, windows). type: keyword - +example: `darwin` | extended // =============================================================== -| observer.geo.country_iso_code -| Country ISO code. +| observer.os.version +| Operating system version as a raw string. type: keyword -example: `CA` +example: `10.14.1` -| core +| extended // =============================================================== -| observer.os.kernel -| Operating system kernel version as a raw string. +| observer.serial_number +| Observer serial number. type: keyword -example: `4.4.0-112-generic` + | extended @@ -2816,27 +2818,25 @@ example: `firewall` // =============================================================== -| observer.geo.region_iso_code -| Region ISO code. +| observer.vendor +| observer vendor information. type: keyword -example: `CA-QC` + | core // =============================================================== -| observer.geo.name -| User-defined description of a location, at the level of granularity they care about. -Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. -Not typically used in automated geolocation. +| observer.version +| Observer version. type: keyword -example: `boston-dc` -| extended + +| core // =============================================================== @@ -2886,8 +2886,8 @@ These fields help you arrange or filter data stored in an index by one or multip // =============================================================== -| organization.name -| Organization name. +| organization.id +| Unique identifier for the organization. type: keyword @@ -2897,8 +2897,8 @@ type: keyword // =============================================================== -| organization.id -| Unique identifier for the organization. +| organization.name +| Organization name. type: keyword @@ -2953,67 +2953,67 @@ The OS fields contain information about the operating system. // =============================================================== -| os.platform -| Operating system platform (such centos, ubuntu, windows). +| os.family +| OS family (such as redhat, debian, freebsd, windows). type: keyword -example: `darwin` +example: `debian` | extended // =============================================================== -| os.name -| Operating system name, without the version. +| os.full +| Operating system name, including the version or code name. type: keyword -example: `Mac OS X` +example: `Mac OS Mojave` | extended // =============================================================== -| os.full -| Operating system name, including the version or code name. +| os.kernel +| Operating system kernel version as a raw string. type: keyword -example: `Mac OS Mojave` +example: `4.4.0-112-generic` | extended // =============================================================== -| os.family -| OS family (such as redhat, debian, freebsd, windows). +| os.name +| Operating system name, without the version. type: keyword -example: `debian` +example: `Mac OS X` | extended // =============================================================== -| os.version -| Operating system version as a raw string. +| os.platform +| Operating system platform (such centos, ubuntu, windows). type: keyword -example: `10.14.1` +example: `darwin` | extended // =============================================================== -| os.kernel -| Operating system kernel version as a raw string. +| os.version +| Operating system version as a raw string. type: keyword -example: `4.4.0-112-generic` +example: `10.14.1` | extended @@ -3065,14 +3065,26 @@ These fields can help you correlate metrics information with a process id/name f // =============================================================== -| process.pid -| Process id. +| process.args +| Array of process arguments. +May be filtered to protect sensitive information. -type: long +type: keyword +example: `['ssh', '-l', 'user', '10.0.0.16']` +| extended -| core +// =============================================================== + +| process.executable +| Absolute path to the process executable. + +type: keyword + +example: `/usr/bin/ssh` + +| extended // =============================================================== @@ -3088,47 +3100,34 @@ example: `ssh` // =============================================================== -| process.ppid -| Process parent id. +| process.pid +| Process id. type: long -| extended +| core // =============================================================== -| process.args -| Array of process arguments. -May be filtered to protect sensitive information. - -type: keyword - -example: `['ssh', '-l', 'user', '10.0.0.16']` - -| extended - -// =============================================================== +| process.ppid +| Process parent id. -| process.executable -| Absolute path to the process executable. +type: long -type: keyword -example: `/usr/bin/ssh` | extended // =============================================================== -| process.title -| Process title. -The proctitle, some times the same as process name. Can also be different: for example a browser setting its title to the web page currently opened. - -type: keyword +| process.start +| The time the process started. +type: date +example: `2016-05-23T08:05:34.853Z` | extended @@ -3145,12 +3144,13 @@ example: `4242` // =============================================================== -| process.start -| The time the process started. +| process.title +| Process title. +The proctitle, some times the same as process name. Can also be different: for example a browser setting its title to the web page currently opened. + +type: keyword -type: date -example: `2016-05-23T08:05:34.853Z` | extended @@ -3284,88 +3284,123 @@ type: keyword // =============================================================== -| server.geo.location -| Longitude and latitude. +| server.bytes +| Bytes sent from the server to the client. -type: geo_point +type: long -example: `{ "lon": -73.614830, "lat": 45.505918 }` +example: `184` + +| core + +// =============================================================== + +| server.domain +| Server domain. + +type: keyword + + + +| core + +// =============================================================== + +| server.geo.city_name +| City name. + +type: keyword + +example: `Montreal` + +| core + +// =============================================================== + +| server.geo.continent_name +| Name of the continent. + +type: keyword + +example: `North America` | core // =============================================================== -| server.user.id -| One or multiple unique identifiers of the user. +| server.geo.country_iso_code +| Country ISO code. type: keyword - +example: `CA` | core // =============================================================== -| server.user.group.id -| Unique identifier for the group on the system/platform. +| server.geo.country_name +| Country name. type: keyword +example: `Canada` - -| extended +| core // =============================================================== -| server.ip -| IP address of the server. -Can be one or multiple IPv4 or IPv6 addresses. - -type: ip +| server.geo.location +| Longitude and latitude. +type: geo_point +example: `{ "lon": -73.614830, "lat": 45.505918 }` | core // =============================================================== -| server.user.group.name -| Name of the group. +| server.geo.name +| User-defined description of a location, at the level of granularity they care about. +Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. +Not typically used in automated geolocation. type: keyword - +example: `boston-dc` | extended // =============================================================== -| server.geo.continent_name -| Name of the continent. +| server.geo.region_iso_code +| Region ISO code. type: keyword -example: `North America` +example: `CA-QC` | core // =============================================================== -| server.user.name -| Short name or login of the user. +| server.geo.region_name +| Region name. type: keyword -example: `albert` +example: `Quebec` | core // =============================================================== -| server.port -| Port of the server. +| server.ip +| IP address of the server. +Can be one or multiple IPv4 or IPv6 addresses. -type: long +type: ip @@ -3373,32 +3408,32 @@ type: long // =============================================================== -| server.geo.country_name -| Country name. +| server.mac +| MAC address of the server. type: keyword -example: `Canada` + | core // =============================================================== -| server.user.full_name -| User's full name, if available. +| server.packets +| Packets sent from the server to the client. -type: keyword +type: long -example: `Albert Einstein` +example: `12` -| extended +| core // =============================================================== -| server.mac -| MAC address of the server. +| server.port +| Port of the server. -type: keyword +type: long @@ -3417,31 +3452,30 @@ type: keyword // =============================================================== -| server.geo.region_name -| Region name. +| server.user.full_name +| User's full name, if available. type: keyword -example: `Quebec` +example: `Albert Einstein` -| core +| extended // =============================================================== -| server.domain -| Server domain. +| server.user.group.id +| Unique identifier for the group on the system/platform. type: keyword -| core +| extended // =============================================================== -| server.user.hash -| Unique user hash to correlate information for a user in anonymized form. -Useful if `user.id` or `user.name` contain confidential information and cannot be used. +| server.user.group.name +| Name of the group. type: keyword @@ -3451,74 +3485,40 @@ type: keyword // =============================================================== -| server.geo.city_name -| City name. +| server.user.hash +| Unique user hash to correlate information for a user in anonymized form. +Useful if `user.id` or `user.name` contain confidential information and cannot be used. type: keyword -example: `Montreal` - -| core - -// =============================================================== - -| server.bytes -| Bytes sent from the server to the client. - -type: long -example: `184` -| core +| extended // =============================================================== -| server.geo.country_iso_code -| Country ISO code. +| server.user.id +| One or multiple unique identifiers of the user. type: keyword -example: `CA` - -| core - -// =============================================================== - -| server.packets -| Packets sent from the server to the client. - -type: long -example: `12` | core // =============================================================== -| server.geo.region_iso_code -| Region ISO code. +| server.user.name +| Short name or login of the user. type: keyword -example: `CA-QC` +example: `albert` | core // =============================================================== -| server.geo.name -| User-defined description of a location, at the level of granularity they care about. -Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. -Not typically used in automated geolocation. - -type: keyword - -example: `boston-dc` - -| extended - -// =============================================================== - |===== [[ecs-server-nestings]] @@ -3565,6 +3565,18 @@ These fields help you find and correlate logs for a specific service and version // =============================================================== +| service.ephemeral_id +| Ephemeral identifier of this service (if one exists). +This id normally changes across restarts, but `service.id` does not. + +type: keyword + +example: `8a4f500f` + +| extended + +// =============================================================== + | service.id | Unique identifier of the running service. This id should uniquely identify this service. This makes it possible to correlate logs and metrics for one specific service. @@ -3592,25 +3604,25 @@ example: `elasticsearch-metrics` // =============================================================== -| service.type -| The type of the service data is collected from. -The type can be used to group and correlate logs and metrics from one service type. -Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. +| service.state +| Current state of the service. type: keyword -example: `elasticsearch` + | core // =============================================================== -| service.state -| Current state of the service. +| service.type +| The type of the service data is collected from. +The type can be used to group and correlate logs and metrics from one service type. +Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. type: keyword - +example: `elasticsearch` | core @@ -3628,18 +3640,6 @@ example: `3.2.4` // =============================================================== -| service.ephemeral_id -| Ephemeral identifier of this service (if one exists). -This id normally changes across restarts, but `service.id` does not. - -type: keyword - -example: `8a4f500f` - -| extended - -// =============================================================== - |===== [[ecs-service-nestings]] @@ -3698,19 +3698,19 @@ type: keyword // =============================================================== -| source.geo.location -| Longitude and latitude. +| source.bytes +| Bytes sent from the source to the destination. -type: geo_point +type: long -example: `{ "lon": -73.614830, "lat": 45.505918 }` +example: `184` | core // =============================================================== -| source.user.id -| One or multiple unique identifiers of the user. +| source.domain +| Source domain. type: keyword @@ -3720,99 +3720,101 @@ type: keyword // =============================================================== -| source.user.group.id -| Unique identifier for the group on the system/platform. +| source.geo.city_name +| City name. type: keyword +example: `Montreal` - -| extended +| core // =============================================================== -| source.ip -| IP address of the source. -Can be one or multiple IPv4 or IPv6 addresses. - -type: ip +| source.geo.continent_name +| Name of the continent. +type: keyword +example: `North America` | core // =============================================================== -| source.user.group.name -| Name of the group. +| source.geo.country_iso_code +| Country ISO code. type: keyword +example: `CA` - -| extended +| core // =============================================================== -| source.geo.continent_name -| Name of the continent. +| source.geo.country_name +| Country name. type: keyword -example: `North America` +example: `Canada` | core // =============================================================== -| source.user.name -| Short name or login of the user. +| source.geo.location +| Longitude and latitude. -type: keyword +type: geo_point -example: `albert` +example: `{ "lon": -73.614830, "lat": 45.505918 }` | core // =============================================================== -| source.port -| Port of the source. - -type: long +| source.geo.name +| User-defined description of a location, at the level of granularity they care about. +Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. +Not typically used in automated geolocation. +type: keyword +example: `boston-dc` -| core +| extended // =============================================================== -| source.geo.country_name -| Country name. +| source.geo.region_iso_code +| Region ISO code. type: keyword -example: `Canada` +example: `CA-QC` | core // =============================================================== -| source.user.full_name -| User's full name, if available. +| source.geo.region_name +| Region name. type: keyword -example: `Albert Einstein` +example: `Quebec` -| extended +| core // =============================================================== -| source.mac -| MAC address of the source. +| source.ip +| IP address of the source. +Can be one or multiple IPv4 or IPv6 addresses. -type: keyword +type: ip @@ -3820,32 +3822,32 @@ type: keyword // =============================================================== -| source.user.email -| User email address. +| source.mac +| MAC address of the source. type: keyword -| extended +| core // =============================================================== -| source.geo.region_name -| Region name. +| source.packets +| Packets sent from the source to the destination. -type: keyword +type: long -example: `Quebec` +example: `12` | core // =============================================================== -| source.domain -| Source domain. +| source.port +| Port of the source. -type: keyword +type: long @@ -3853,9 +3855,8 @@ type: keyword // =============================================================== -| source.user.hash -| Unique user hash to correlate information for a user in anonymized form. -Useful if `user.id` or `user.name` contain confidential information and cannot be used. +| source.user.email +| User email address. type: keyword @@ -3865,71 +3866,70 @@ type: keyword // =============================================================== -| source.geo.city_name -| City name. +| source.user.full_name +| User's full name, if available. type: keyword -example: `Montreal` +example: `Albert Einstein` -| core +| extended // =============================================================== -| source.bytes -| Bytes sent from the source to the destination. +| source.user.group.id +| Unique identifier for the group on the system/platform. -type: long +type: keyword -example: `184` -| core + +| extended // =============================================================== -| source.geo.country_iso_code -| Country ISO code. +| source.user.group.name +| Name of the group. type: keyword -example: `CA` -| core + +| extended // =============================================================== -| source.packets -| Packets sent from the source to the destination. +| source.user.hash +| Unique user hash to correlate information for a user in anonymized form. +Useful if `user.id` or `user.name` contain confidential information and cannot be used. -type: long +type: keyword -example: `12` -| core + +| extended // =============================================================== -| source.geo.region_iso_code -| Region ISO code. +| source.user.id +| One or multiple unique identifiers of the user. type: keyword -example: `CA-QC` + | core // =============================================================== -| source.geo.name -| User-defined description of a location, at the level of granularity they care about. -Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. -Not typically used in automated geolocation. +| source.user.name +| Short name or login of the user. type: keyword -example: `boston-dc` +example: `albert` -| extended +| core // =============================================================== @@ -3978,60 +3978,60 @@ URL fields provide support for complete or partial URLs, and supports the breaki // =============================================================== -| url.original -| Unmodified original url as seen in the event source. -Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. -This field is meant to represent the URL as it was observed, complete or not. +| url.domain +| Domain of the url, such as "www.elastic.co". +In some cases a URL may refer to an IP and/or port directly, without a domain name. In this case, the IP address would go to the `domain` field. type: keyword -example: `https://www.elastic.co:443/search?q=elasticsearch#top or /search?q=elasticsearch` +example: `www.elastic.co` | extended // =============================================================== -| url.full -| If full URLs are important to your use case, they should be stored in `url.full`, whether this field is reconstructed or present in the event source. +| url.fragment +| Portion of the url after the `#`, such as "top". +The `#` is not part of the fragment. type: keyword -example: `https://www.elastic.co:443/search?q=elasticsearch#top` + | extended // =============================================================== -| url.scheme -| Scheme of the request, such as "https". -Note: The `:` is not part of the scheme. +| url.full +| If full URLs are important to your use case, they should be stored in `url.full`, whether this field is reconstructed or present in the event source. type: keyword -example: `https` +example: `https://www.elastic.co:443/search?q=elasticsearch#top` | extended // =============================================================== -| url.domain -| Domain of the url, such as "www.elastic.co". -In some cases a URL may refer to an IP and/or port directly, without a domain name. In this case, the IP address would go to the `domain` field. +| url.original +| Unmodified original url as seen in the event source. +Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. +This field is meant to represent the URL as it was observed, complete or not. type: keyword -example: `www.elastic.co` +example: `https://www.elastic.co:443/search?q=elasticsearch#top or /search?q=elasticsearch` | extended // =============================================================== -| url.port -| Port of the request, such as 443. +| url.password +| Password of the request. + +type: keyword -type: long -example: `443` | extended @@ -4048,21 +4048,20 @@ type: keyword // =============================================================== -| url.query -| The query field describes the query string of the request, such as "q=elasticsearch". -The `?` is excluded from the query string. If a URL contains no `?`, there is no query field. If there is a `?` but no query, the query field exists with an empty string. The `exists` query can be used to differentiate between the two cases. - -type: keyword +| url.port +| Port of the request, such as 443. +type: long +example: `443` | extended // =============================================================== -| url.fragment -| Portion of the url after the `#`, such as "top". -The `#` is not part of the fragment. +| url.query +| The query field describes the query string of the request, such as "q=elasticsearch". +The `?` is excluded from the query string. If a URL contains no `?`, there is no query field. If there is a `?` but no query, the query field exists with an empty string. The `exists` query can be used to differentiate between the two cases. type: keyword @@ -4072,19 +4071,20 @@ type: keyword // =============================================================== -| url.username -| Username of the request. +| url.scheme +| Scheme of the request, such as "https". +Note: The `:` is not part of the scheme. type: keyword - +example: `https` | extended // =============================================================== -| url.password -| Password of the request. +| url.username +| Username of the request. type: keyword @@ -4140,30 +4140,30 @@ Fields can have one entry or multiple entries. If a user has more than one id, p // =============================================================== -| user.id -| One or multiple unique identifiers of the user. +| user.email +| User email address. type: keyword -| core +| extended // =============================================================== -| user.group.id -| Unique identifier for the group on the system/platform. +| user.full_name +| User's full name, if available. type: keyword - +example: `Albert Einstein` | extended // =============================================================== -| user.group.name -| Name of the group. +| user.group.id +| Unique identifier for the group on the system/platform. type: keyword @@ -4173,48 +4173,48 @@ type: keyword // =============================================================== -| user.name -| Short name or login of the user. +| user.group.name +| Name of the group. type: keyword -example: `albert` -| core + +| extended // =============================================================== -| user.full_name -| User's full name, if available. +| user.hash +| Unique user hash to correlate information for a user in anonymized form. +Useful if `user.id` or `user.name` contain confidential information and cannot be used. type: keyword -example: `Albert Einstein` + | extended // =============================================================== -| user.email -| User email address. +| user.id +| One or multiple unique identifiers of the user. type: keyword -| extended +| core // =============================================================== -| user.hash -| Unique user hash to correlate information for a user in anonymized form. -Useful if `user.id` or `user.name` contain confidential information and cannot be used. +| user.name +| Short name or login of the user. type: keyword +example: `albert` - -| extended +| core // =============================================================== @@ -4264,89 +4264,89 @@ They often show up in web service logs coming from the parsed user agent string. // =============================================================== -| user_agent.original -| Unparsed version of the user_agent. +| user_agent.device.name +| Name of the device. type: keyword -example: `Mozilla/5.0 (iPhone; CPU iPhone OS 12_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.0 Mobile/15E148 Safari/604.1` +example: `iPhone` | extended // =============================================================== -| user_agent.os.platform -| Operating system platform (such centos, ubuntu, windows). +| user_agent.name +| Name of the user agent. type: keyword -example: `darwin` +example: `Safari` | extended // =============================================================== -| user_agent.name -| Name of the user agent. +| user_agent.original +| Unparsed version of the user_agent. type: keyword -example: `Safari` +example: `Mozilla/5.0 (iPhone; CPU iPhone OS 12_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.0 Mobile/15E148 Safari/604.1` | extended // =============================================================== -| user_agent.os.name -| Operating system name, without the version. +| user_agent.os.family +| OS family (such as redhat, debian, freebsd, windows). type: keyword -example: `Mac OS X` +example: `debian` | extended // =============================================================== -| user_agent.version -| Version of the user agent. +| user_agent.os.full +| Operating system name, including the version or code name. type: keyword -example: `12.0` +example: `Mac OS Mojave` | extended // =============================================================== -| user_agent.os.full -| Operating system name, including the version or code name. +| user_agent.os.kernel +| Operating system kernel version as a raw string. type: keyword -example: `Mac OS Mojave` +example: `4.4.0-112-generic` | extended // =============================================================== -| user_agent.device.name -| Name of the device. +| user_agent.os.name +| Operating system name, without the version. type: keyword -example: `iPhone` +example: `Mac OS X` | extended // =============================================================== -| user_agent.os.family -| OS family (such as redhat, debian, freebsd, windows). +| user_agent.os.platform +| Operating system platform (such centos, ubuntu, windows). type: keyword -example: `debian` +example: `darwin` | extended @@ -4363,12 +4363,12 @@ example: `10.14.1` // =============================================================== -| user_agent.os.kernel -| Operating system kernel version as a raw string. +| user_agent.version +| Version of the user agent. type: keyword -example: `4.4.0-112-generic` +example: `12.0` | extended diff --git a/scripts/generators/asciidoc_fields.py b/scripts/generators/asciidoc_fields.py index aaa9adf83f..74231ce873 100644 --- a/scripts/generators/asciidoc_fields.py +++ b/scripts/generators/asciidoc_fields.py @@ -56,7 +56,7 @@ def render_fieldset(fieldset): fieldset_title=fieldset['title'] ) - for field in ecs_helpers.dict_sorted_by_keys(fieldset['fields'], 'order'): + for field in ecs_helpers.dict_sorted_by_keys(fieldset['fields'], 'flat_name'): text += render_field_details_row(field) text += table_footer() From b9cdebd0da8359e9d72d26e8b14160c20cb41216 Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Tue, 19 Mar 2019 14:47:00 -0400 Subject: [PATCH 25/28] Remove leaf reused fields from docs, list fieldsets separately in a table below --- docs/field-details.asciidoc | 2744 +++++-------------------- generated/ecs/fields_nested.yml | 23 + scripts/generators/asciidoc_fields.py | 47 +- scripts/schema_reader.py | 7 + 4 files changed, 533 insertions(+), 2288 deletions(-) diff --git a/docs/field-details.asciidoc b/docs/field-details.asciidoc index 93ff7bcabc..fb1ea3b8f0 100644 --- a/docs/field-details.asciidoc +++ b/docs/field-details.asciidoc @@ -63,36 +63,6 @@ example: `["production", "env2"]` // =============================================================== -|===== - -[[ecs-base-nestings]] -==== Can be nested under Base - -[options="header"] -|===== -| Nested fields | Description - -// =============================================================== - - -| http://localhost:8000/ecs-geo.html[host.geo.*] -| Fields describing a location. - -// =============================================================== - - -| http://localhost:8000/ecs-os.html[host.os.*] -| OS fields contain information about the operating system. - -// =============================================================== - - -| http://localhost:8000/ecs-user.html[host.user.*] -| Fields to describe the user relevant to the event. - -// =============================================================== - - |===== [[ecs-agent]] @@ -169,36 +139,6 @@ example: `6.0.0-rc2` // =============================================================== -|===== - -[[ecs-agent-nestings]] -==== Can be nested under Agent - -[options="header"] -|===== -| Nested fields | Description - -// =============================================================== - - -| http://localhost:8000/ecs-geo.html[host.geo.*] -| Fields describing a location. - -// =============================================================== - - -| http://localhost:8000/ecs-os.html[host.os.*] -| OS fields contain information about the operating system. - -// =============================================================== - - -| http://localhost:8000/ecs-user.html[host.user.*] -| Fields to describe the user relevant to the event. - -// =============================================================== - - |===== [[ecs-client]] @@ -246,96 +186,6 @@ type: keyword -| core - -// =============================================================== - -| client.geo.city_name -| City name. - -type: keyword - -example: `Montreal` - -| core - -// =============================================================== - -| client.geo.continent_name -| Name of the continent. - -type: keyword - -example: `North America` - -| core - -// =============================================================== - -| client.geo.country_iso_code -| Country ISO code. - -type: keyword - -example: `CA` - -| core - -// =============================================================== - -| client.geo.country_name -| Country name. - -type: keyword - -example: `Canada` - -| core - -// =============================================================== - -| client.geo.location -| Longitude and latitude. - -type: geo_point - -example: `{ "lon": -73.614830, "lat": 45.505918 }` - -| core - -// =============================================================== - -| client.geo.name -| User-defined description of a location, at the level of granularity they care about. -Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. -Not typically used in automated geolocation. - -type: keyword - -example: `boston-dc` - -| extended - -// =============================================================== - -| client.geo.region_iso_code -| Region ISO code. - -type: keyword - -example: `CA-QC` - -| core - -// =============================================================== - -| client.geo.region_name -| Region name. - -type: keyword - -example: `Quebec` - | core // =============================================================== @@ -381,84 +231,6 @@ type: long -| core - -// =============================================================== - -| client.user.email -| User email address. - -type: keyword - - - -| extended - -// =============================================================== - -| client.user.full_name -| User's full name, if available. - -type: keyword - -example: `Albert Einstein` - -| extended - -// =============================================================== - -| client.user.group.id -| Unique identifier for the group on the system/platform. - -type: keyword - - - -| extended - -// =============================================================== - -| client.user.group.name -| Name of the group. - -type: keyword - - - -| extended - -// =============================================================== - -| client.user.hash -| Unique user hash to correlate information for a user in anonymized form. -Useful if `user.id` or `user.name` contain confidential information and cannot be used. - -type: keyword - - - -| extended - -// =============================================================== - -| client.user.id -| One or multiple unique identifiers of the user. - -type: keyword - - - -| core - -// =============================================================== - -| client.user.name -| Short name or login of the user. - -type: keyword - -example: `albert` - | core // =============================================================== @@ -475,19 +247,13 @@ example: `albert` // =============================================================== -| http://localhost:8000/ecs-geo.html[host.geo.*] +| http://localhost:8000/ecs-geo.html[client.geo.*] | Fields describing a location. // =============================================================== -| http://localhost:8000/ecs-os.html[host.os.*] -| OS fields contain information about the operating system. - -// =============================================================== - - -| http://localhost:8000/ecs-user.html[host.user.*] +| http://localhost:8000/ecs-user.html[client.user.*] | Fields to describe the user relevant to the event. // =============================================================== @@ -586,36 +352,6 @@ example: `us-east-1` // =============================================================== -|===== - -[[ecs-cloud-nestings]] -==== Can be nested under Cloud - -[options="header"] -|===== -| Nested fields | Description - -// =============================================================== - - -| http://localhost:8000/ecs-geo.html[host.geo.*] -| Fields describing a location. - -// =============================================================== - - -| http://localhost:8000/ecs-os.html[host.os.*] -| OS fields contain information about the operating system. - -// =============================================================== - - -| http://localhost:8000/ecs-user.html[host.user.*] -| Fields to describe the user relevant to the event. - -// =============================================================== - - |===== [[ecs-container]] @@ -700,47 +436,17 @@ example: `docker` |===== -[[ecs-container-nestings]] -==== Can be nested under Container +[[ecs-destination]] +=== Destination fields + +Destination fields describe details about the destination of a packet/event. +Destination fields are usually populated in conjunction with source fields. + +==== Destination Fields [options="header"] |===== -| Nested fields | Description - -// =============================================================== - - -| http://localhost:8000/ecs-geo.html[host.geo.*] -| Fields describing a location. - -// =============================================================== - - -| http://localhost:8000/ecs-os.html[host.os.*] -| OS fields contain information about the operating system. - -// =============================================================== - - -| http://localhost:8000/ecs-user.html[host.user.*] -| Fields to describe the user relevant to the event. - -// =============================================================== - - -|===== - -[[ecs-destination]] -=== Destination fields - -Destination fields describe details about the destination of a packet/event. -Destination fields are usually populated in conjunction with source fields. - -==== Destination Fields - -[options="header"] -|===== -| Field | Description | Level +| Field | Description | Level // =============================================================== @@ -774,96 +480,6 @@ type: keyword -| core - -// =============================================================== - -| destination.geo.city_name -| City name. - -type: keyword - -example: `Montreal` - -| core - -// =============================================================== - -| destination.geo.continent_name -| Name of the continent. - -type: keyword - -example: `North America` - -| core - -// =============================================================== - -| destination.geo.country_iso_code -| Country ISO code. - -type: keyword - -example: `CA` - -| core - -// =============================================================== - -| destination.geo.country_name -| Country name. - -type: keyword - -example: `Canada` - -| core - -// =============================================================== - -| destination.geo.location -| Longitude and latitude. - -type: geo_point - -example: `{ "lon": -73.614830, "lat": 45.505918 }` - -| core - -// =============================================================== - -| destination.geo.name -| User-defined description of a location, at the level of granularity they care about. -Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. -Not typically used in automated geolocation. - -type: keyword - -example: `boston-dc` - -| extended - -// =============================================================== - -| destination.geo.region_iso_code -| Region ISO code. - -type: keyword - -example: `CA-QC` - -| core - -// =============================================================== - -| destination.geo.region_name -| Region name. - -type: keyword - -example: `Quebec` - | core // =============================================================== @@ -909,84 +525,6 @@ type: long -| core - -// =============================================================== - -| destination.user.email -| User email address. - -type: keyword - - - -| extended - -// =============================================================== - -| destination.user.full_name -| User's full name, if available. - -type: keyword - -example: `Albert Einstein` - -| extended - -// =============================================================== - -| destination.user.group.id -| Unique identifier for the group on the system/platform. - -type: keyword - - - -| extended - -// =============================================================== - -| destination.user.group.name -| Name of the group. - -type: keyword - - - -| extended - -// =============================================================== - -| destination.user.hash -| Unique user hash to correlate information for a user in anonymized form. -Useful if `user.id` or `user.name` contain confidential information and cannot be used. - -type: keyword - - - -| extended - -// =============================================================== - -| destination.user.id -| One or multiple unique identifiers of the user. - -type: keyword - - - -| core - -// =============================================================== - -| destination.user.name -| Short name or login of the user. - -type: keyword - -example: `albert` - | core // =============================================================== @@ -1003,19 +541,13 @@ example: `albert` // =============================================================== -| http://localhost:8000/ecs-geo.html[host.geo.*] +| http://localhost:8000/ecs-geo.html[destination.geo.*] | Fields describing a location. // =============================================================== -| http://localhost:8000/ecs-os.html[host.os.*] -| OS fields contain information about the operating system. - -// =============================================================== - - -| http://localhost:8000/ecs-user.html[host.user.*] +| http://localhost:8000/ecs-user.html[destination.user.*] | Fields to describe the user relevant to the event. // =============================================================== @@ -1048,36 +580,6 @@ example: `1.0.0` // =============================================================== -|===== - -[[ecs-ecs-nestings]] -==== Can be nested under ECS - -[options="header"] -|===== -| Nested fields | Description - -// =============================================================== - - -| http://localhost:8000/ecs-geo.html[host.geo.*] -| Fields describing a location. - -// =============================================================== - - -| http://localhost:8000/ecs-os.html[host.os.*] -| OS fields contain information about the operating system. - -// =============================================================== - - -| http://localhost:8000/ecs-user.html[host.user.*] -| Fields to describe the user relevant to the event. - -// =============================================================== - - |===== [[ecs-error]] @@ -1127,36 +629,6 @@ type: text // =============================================================== -|===== - -[[ecs-error-nestings]] -==== Can be nested under Error - -[options="header"] -|===== -| Nested fields | Description - -// =============================================================== - - -| http://localhost:8000/ecs-geo.html[host.geo.*] -| Fields describing a location. - -// =============================================================== - - -| http://localhost:8000/ecs-os.html[host.os.*] -| OS fields contain information about the operating system. - -// =============================================================== - - -| http://localhost:8000/ecs-user.html[host.user.*] -| Fields to describe the user relevant to the event. - -// =============================================================== - - |===== [[ecs-event]] @@ -1387,38 +859,8 @@ type: keyword |===== -[[ecs-event-nestings]] -==== Can be nested under Event - -[options="header"] -|===== -| Nested fields | Description - -// =============================================================== - - -| http://localhost:8000/ecs-geo.html[host.geo.*] -| Fields describing a location. - -// =============================================================== - - -| http://localhost:8000/ecs-os.html[host.os.*] -| OS fields contain information about the operating system. - -// =============================================================== - - -| http://localhost:8000/ecs-user.html[host.user.*] -| Fields to describe the user relevant to the event. - -// =============================================================== - - -|===== - -[[ecs-file]] -=== File fields +[[ecs-file]] +=== File fields A file is defined as a set of information that has been created on, or has existed on a filesystem. File objects can be associated with host events, network events, and/or file events (e.g., those produced by File Integrity Monitoring [FIM] products or services). File fields provide details about the affected file associated with the event or metric. @@ -1586,36 +1028,6 @@ type: keyword // =============================================================== -|===== - -[[ecs-file-nestings]] -==== Can be nested under File - -[options="header"] -|===== -| Nested fields | Description - -// =============================================================== - - -| http://localhost:8000/ecs-geo.html[host.geo.*] -| Fields describing a location. - -// =============================================================== - - -| http://localhost:8000/ecs-os.html[host.os.*] -| OS fields contain information about the operating system. - -// =============================================================== - - -| http://localhost:8000/ecs-user.html[host.user.*] -| Fields to describe the user relevant to the event. - -// =============================================================== - - |===== [[ecs-geo]] @@ -1722,36 +1134,6 @@ example: `Quebec` // =============================================================== -|===== - -[[ecs-geo-nestings]] -==== Can be nested under Geo - -[options="header"] -|===== -| Nested fields | Description - -// =============================================================== - - -| http://localhost:8000/ecs-geo.html[host.geo.*] -| Fields describing a location. - -// =============================================================== - - -| http://localhost:8000/ecs-os.html[host.os.*] -| OS fields contain information about the operating system. - -// =============================================================== - - -| http://localhost:8000/ecs-user.html[host.user.*] -| Fields to describe the user relevant to the event. - -// =============================================================== - - |===== [[ecs-group]] @@ -1789,36 +1171,6 @@ type: keyword // =============================================================== -|===== - -[[ecs-group-nestings]] -==== Can be nested under Group - -[options="header"] -|===== -| Nested fields | Description - -// =============================================================== - - -| http://localhost:8000/ecs-geo.html[host.geo.*] -| Fields describing a location. - -// =============================================================== - - -| http://localhost:8000/ecs-os.html[host.os.*] -| OS fields contain information about the operating system. - -// =============================================================== - - -| http://localhost:8000/ecs-user.html[host.user.*] -| Fields to describe the user relevant to the event. - -// =============================================================== - - |===== [[ecs-host]] @@ -1846,440 +1198,206 @@ example: `x86_64` // =============================================================== -| host.geo.city_name -| City name. +| host.hostname +| Hostname of the host. +It normally contains what the `hostname` command returns on the host machine. type: keyword -example: `Montreal` + | core // =============================================================== -| host.geo.continent_name -| Name of the continent. +| host.id +| Unique host id. +As hostname is not always unique, use values that are meaningful in your environment. +Example: The current usage of `beat.name`. type: keyword -example: `North America` + | core // =============================================================== -| host.geo.country_iso_code -| Country ISO code. +| host.ip +| Host ip address. + +type: ip -type: keyword -example: `CA` | core // =============================================================== -| host.geo.country_name -| Country name. +| host.mac +| Host mac address. type: keyword -example: `Canada` + | core // =============================================================== -| host.geo.location -| Longitude and latitude. +| host.name +| Name of the host. +It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. + +type: keyword -type: geo_point -example: `{ "lon": -73.614830, "lat": 45.505918 }` | core // =============================================================== -| host.geo.name -| User-defined description of a location, at the level of granularity they care about. -Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. -Not typically used in automated geolocation. +| host.type +| Type of host. +For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. type: keyword -example: `boston-dc` -| extended -// =============================================================== +| core -| host.geo.region_iso_code -| Region ISO code. +// =============================================================== -type: keyword +|===== -example: `CA-QC` +[[ecs-host-nestings]] +==== Can be nested under Host -| core +[options="header"] +|===== +| Nested fields | Description // =============================================================== -| host.geo.region_name -| Region name. - -type: keyword - -example: `Quebec` -| core +| http://localhost:8000/ecs-geo.html[host.geo.*] +| Fields describing a location. // =============================================================== -| host.hostname -| Hostname of the host. -It normally contains what the `hostname` command returns on the host machine. -type: keyword +| http://localhost:8000/ecs-os.html[host.os.*] +| OS fields contain information about the operating system. +// =============================================================== -| core +| http://localhost:8000/ecs-user.html[host.user.*] +| Fields to describe the user relevant to the event. // =============================================================== -| host.id -| Unique host id. -As hostname is not always unique, use values that are meaningful in your environment. -Example: The current usage of `beat.name`. -type: keyword +|===== +[[ecs-http]] +=== HTTP fields +Fields related to HTTP activity. Use the `url` field set to store the url of the request. -| core +==== HTTP Fields -// =============================================================== +[options="header"] +|===== +| Field | Description | Level -| host.ip -| Host ip address. +// =============================================================== -type: ip +| http.request.body.bytes +| Size in bytes of the request body. +type: long +example: `887` -| core +| extended // =============================================================== -| host.mac -| Host mac address. +| http.request.body.content +| The full HTTP request body. type: keyword +example: `Hello world` - -| core +| extended // =============================================================== -| host.name -| Name of the host. -It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. - -type: keyword +| http.request.bytes +| Total size in bytes of the request (body and headers). +type: long +example: `1437` -| core +| extended // =============================================================== -| host.os.family -| OS family (such as redhat, debian, freebsd, windows). +| http.request.method +| HTTP request method. +The field value must be normalized to lowercase for querying. See the documentation section "Implementing ECS". type: keyword -example: `debian` +example: `get, post, put` | extended // =============================================================== -| host.os.full -| Operating system name, including the version or code name. +| http.request.referrer +| Referrer for this HTTP request. type: keyword -example: `Mac OS Mojave` +example: `https://blog.example.com/` | extended // =============================================================== -| host.os.kernel -| Operating system kernel version as a raw string. +| http.response.body.bytes +| Size in bytes of the response body. -type: keyword +type: long -example: `4.4.0-112-generic` +example: `887` | extended // =============================================================== -| host.os.name -| Operating system name, without the version. +| http.response.body.content +| The full HTTP response body. type: keyword -example: `Mac OS X` +example: `Hello world` | extended // =============================================================== -| host.os.platform -| Operating system platform (such centos, ubuntu, windows). +| http.response.bytes +| Total size in bytes of the response (body and headers). -type: keyword +type: long -example: `darwin` - -| extended - -// =============================================================== - -| host.os.version -| Operating system version as a raw string. - -type: keyword - -example: `10.14.1` - -| extended - -// =============================================================== - -| host.type -| Type of host. -For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. - -type: keyword - - - -| core - -// =============================================================== - -| host.user.email -| User email address. - -type: keyword - - - -| extended - -// =============================================================== - -| host.user.full_name -| User's full name, if available. - -type: keyword - -example: `Albert Einstein` - -| extended - -// =============================================================== - -| host.user.group.id -| Unique identifier for the group on the system/platform. - -type: keyword - - - -| extended - -// =============================================================== - -| host.user.group.name -| Name of the group. - -type: keyword - - - -| extended - -// =============================================================== - -| host.user.hash -| Unique user hash to correlate information for a user in anonymized form. -Useful if `user.id` or `user.name` contain confidential information and cannot be used. - -type: keyword - - - -| extended - -// =============================================================== - -| host.user.id -| One or multiple unique identifiers of the user. - -type: keyword - - - -| core - -// =============================================================== - -| host.user.name -| Short name or login of the user. - -type: keyword - -example: `albert` - -| core - -// =============================================================== - -|===== - -[[ecs-host-nestings]] -==== Can be nested under Host - -[options="header"] -|===== -| Nested fields | Description - -// =============================================================== - - -| http://localhost:8000/ecs-geo.html[host.geo.*] -| Fields describing a location. - -// =============================================================== - - -| http://localhost:8000/ecs-os.html[host.os.*] -| OS fields contain information about the operating system. - -// =============================================================== - - -| http://localhost:8000/ecs-user.html[host.user.*] -| Fields to describe the user relevant to the event. - -// =============================================================== - - -|===== - -[[ecs-http]] -=== HTTP fields - -Fields related to HTTP activity. Use the `url` field set to store the url of the request. - -==== HTTP Fields - -[options="header"] -|===== -| Field | Description | Level - -// =============================================================== - -| http.request.body.bytes -| Size in bytes of the request body. - -type: long - -example: `887` - -| extended - -// =============================================================== - -| http.request.body.content -| The full HTTP request body. - -type: keyword - -example: `Hello world` - -| extended - -// =============================================================== - -| http.request.bytes -| Total size in bytes of the request (body and headers). - -type: long - -example: `1437` - -| extended - -// =============================================================== - -| http.request.method -| HTTP request method. -The field value must be normalized to lowercase for querying. See the documentation section "Implementing ECS". - -type: keyword - -example: `get, post, put` - -| extended - -// =============================================================== - -| http.request.referrer -| Referrer for this HTTP request. - -type: keyword - -example: `https://blog.example.com/` - -| extended - -// =============================================================== - -| http.response.body.bytes -| Size in bytes of the response body. - -type: long - -example: `887` - -| extended - -// =============================================================== - -| http.response.body.content -| The full HTTP response body. - -type: keyword - -example: `Hello world` - -| extended - -// =============================================================== - -| http.response.bytes -| Total size in bytes of the response (body and headers). - -type: long - -example: `1437` +example: `1437` | extended @@ -2307,36 +1425,6 @@ example: `1.1` // =============================================================== -|===== - -[[ecs-http-nestings]] -==== Can be nested under HTTP - -[options="header"] -|===== -| Nested fields | Description - -// =============================================================== - - -| http://localhost:8000/ecs-geo.html[host.geo.*] -| Fields describing a location. - -// =============================================================== - - -| http://localhost:8000/ecs-os.html[host.os.*] -| OS fields contain information about the operating system. - -// =============================================================== - - -| http://localhost:8000/ecs-user.html[host.user.*] -| Fields to describe the user relevant to the event. - -// =============================================================== - - |===== [[ecs-log]] @@ -2352,861 +1440,40 @@ Fields which are specific to log events. // =============================================================== -| log.level -| Original log level of the log event. -Some examples are `warn`, `error`, `i`. - -type: keyword - -example: `err` - -| core - -// =============================================================== - -| log.original -| This is the original log message and contains the full log message before splitting it up in multiple parts. -In contrast to the `message` field which can contain an extracted part of the log message, this field contains the original, full log message. It can have already some modifications applied like encoding or new lines removed to clean up the log message. -This field is not indexed and doc_values are disabled so it can't be queried but the value can be retrieved from `_source`. - -type: keyword - -example: `Sep 19 08:26:10 localhost My log` - -| core - -// =============================================================== - -|===== - -[[ecs-log-nestings]] -==== Can be nested under Log - -[options="header"] -|===== -| Nested fields | Description - -// =============================================================== - - -| http://localhost:8000/ecs-geo.html[host.geo.*] -| Fields describing a location. - -// =============================================================== - - -| http://localhost:8000/ecs-os.html[host.os.*] -| OS fields contain information about the operating system. - -// =============================================================== - - -| http://localhost:8000/ecs-user.html[host.user.*] -| Fields to describe the user relevant to the event. - -// =============================================================== - - -|===== - -[[ecs-network]] -=== Network fields - -The network is defined as the communication path over which a host or network event happens. -The network.* fields should be populated with details about the network activity associated with an event. - -==== Network Fields - -[options="header"] -|===== -| Field | Description | Level - -// =============================================================== - -| network.application -| A name given to an application level protocol. This can be arbitrarily assigned for things like microservices, but also apply to things like skype, icq, facebook, twitter. This would be used in situations where the vendor or service can be decoded such as from the source/dest IP owners, ports, or wire format. -The field value must be normalized to lowercase for querying. See the documentation section "Implementing ECS". - -type: keyword - -example: `aim` - -| extended - -// =============================================================== - -| network.bytes -| Total bytes transferred in both directions. -If `source.bytes` and `destination.bytes` are known, `network.bytes` is their sum. - -type: long - -example: `368` - -| core - -// =============================================================== - -| network.community_id -| A hash of source and destination IPs and ports, as well as the protocol used in a communication. This is a tool-agnostic standard to identify flows. -Learn more at https://github.com/corelight/community-id-spec. - -type: keyword - -example: `1:hO+sN4H+MG5MY/8hIrXPqc4ZQz0=` - -| extended - -// =============================================================== - -| network.direction -| Direction of the network traffic. -Recommended values are: - * inbound - * outbound - * internal - * external - * unknown - -When mapping events from a host-based monitoring context, populate this field from the host's point of view. -When mapping events from a network or perimeter-based monitoring context, populate this field from the point of view of your network perimeter. - -type: keyword - -example: `inbound` - -| core - -// =============================================================== - -| network.forwarded_ip -| Host IP address when the source IP address is the proxy. - -type: ip - -example: `192.1.1.2` - -| core - -// =============================================================== - -| network.iana_number -| IANA Protocol Number (https://www.iana.org/assignments/protocol-numbers/protocol-numbers.xhtml). Standardized list of protocols. This aligns well with NetFlow and sFlow related logs which use the IANA Protocol Number. - -type: keyword - -example: `6` - -| extended - -// =============================================================== - -| network.name -| Name given by operators to sections of their network. - -type: keyword - -example: `Guest Wifi` - -| extended - -// =============================================================== - -| network.packets -| Total packets transferred in both directions. -If `source.packets` and `destination.packets` are known, `network.packets` is their sum. - -type: long - -example: `24` - -| core - -// =============================================================== - -| network.protocol -| L7 Network protocol name. ex. http, lumberjack, transport protocol. -The field value must be normalized to lowercase for querying. See the documentation section "Implementing ECS". - -type: keyword - -example: `http` - -| core - -// =============================================================== - -| network.transport -| Same as network.iana_number, but instead using the Keyword name of the transport layer (udp, tcp, ipv6-icmp, etc.) -The field value must be normalized to lowercase for querying. See the documentation section "Implementing ECS". - -type: keyword - -example: `tcp` - -| core - -// =============================================================== - -| network.type -| In the OSI Model this would be the Network Layer. ipv4, ipv6, ipsec, pim, etc -The field value must be normalized to lowercase for querying. See the documentation section "Implementing ECS". - -type: keyword - -example: `ipv4` - -| core - -// =============================================================== - -|===== - -[[ecs-network-nestings]] -==== Can be nested under Network - -[options="header"] -|===== -| Nested fields | Description - -// =============================================================== - - -| http://localhost:8000/ecs-geo.html[host.geo.*] -| Fields describing a location. - -// =============================================================== - - -| http://localhost:8000/ecs-os.html[host.os.*] -| OS fields contain information about the operating system. - -// =============================================================== - - -| http://localhost:8000/ecs-user.html[host.user.*] -| Fields to describe the user relevant to the event. - -// =============================================================== - - -|===== - -[[ecs-observer]] -=== Observer fields - -An observer is defined as a special network, security, or application device used to detect, observe, or create network, security, or application-related events and metrics. -This could be a custom hardware appliance or a server that has been configured to run special network, security, or application software. Examples include firewalls, intrusion detection/prevention systems, network monitoring sensors, web application firewalls, data loss prevention systems, and APM servers. The observer.* fields shall be populated with details of the system, if any, that detects, observes and/or creates a network, security, or application event or metric. Message queues and ETL components used in processing events or metrics are not considered observers in ECS. - -==== Observer Fields - -[options="header"] -|===== -| Field | Description | Level - -// =============================================================== - -| observer.geo.city_name -| City name. - -type: keyword - -example: `Montreal` - -| core - -// =============================================================== - -| observer.geo.continent_name -| Name of the continent. - -type: keyword - -example: `North America` - -| core - -// =============================================================== - -| observer.geo.country_iso_code -| Country ISO code. - -type: keyword - -example: `CA` - -| core - -// =============================================================== - -| observer.geo.country_name -| Country name. - -type: keyword - -example: `Canada` - -| core - -// =============================================================== - -| observer.geo.location -| Longitude and latitude. - -type: geo_point - -example: `{ "lon": -73.614830, "lat": 45.505918 }` - -| core - -// =============================================================== - -| observer.geo.name -| User-defined description of a location, at the level of granularity they care about. -Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. -Not typically used in automated geolocation. - -type: keyword - -example: `boston-dc` - -| extended - -// =============================================================== - -| observer.geo.region_iso_code -| Region ISO code. - -type: keyword - -example: `CA-QC` - -| core - -// =============================================================== - -| observer.geo.region_name -| Region name. - -type: keyword - -example: `Quebec` - -| core - -// =============================================================== - -| observer.hostname -| Hostname of the observer. - -type: keyword - - - -| core - -// =============================================================== - -| observer.ip -| IP address of the observer. - -type: ip - - - -| core - -// =============================================================== - -| observer.mac -| MAC address of the observer - -type: keyword - - - -| core - -// =============================================================== - -| observer.os.family -| OS family (such as redhat, debian, freebsd, windows). - -type: keyword - -example: `debian` - -| extended - -// =============================================================== - -| observer.os.full -| Operating system name, including the version or code name. - -type: keyword - -example: `Mac OS Mojave` - -| extended - -// =============================================================== - -| observer.os.kernel -| Operating system kernel version as a raw string. - -type: keyword - -example: `4.4.0-112-generic` - -| extended - -// =============================================================== - -| observer.os.name -| Operating system name, without the version. - -type: keyword - -example: `Mac OS X` - -| extended - -// =============================================================== - -| observer.os.platform -| Operating system platform (such centos, ubuntu, windows). - -type: keyword - -example: `darwin` - -| extended - -// =============================================================== - -| observer.os.version -| Operating system version as a raw string. - -type: keyword - -example: `10.14.1` - -| extended - -// =============================================================== - -| observer.serial_number -| Observer serial number. - -type: keyword - - - -| extended - -// =============================================================== - -| observer.type -| The type of the observer the data is coming from. -There is no predefined list of observer types. Some examples are `forwarder`, `firewall`, `ids`, `ips`, `proxy`, `poller`, `sensor`, `APM server`. - -type: keyword - -example: `firewall` - -| core - -// =============================================================== - -| observer.vendor -| observer vendor information. - -type: keyword - - - -| core - -// =============================================================== - -| observer.version -| Observer version. - -type: keyword - - - -| core - -// =============================================================== - -|===== - -[[ecs-observer-nestings]] -==== Can be nested under Observer - -[options="header"] -|===== -| Nested fields | Description - -// =============================================================== - - -| http://localhost:8000/ecs-geo.html[host.geo.*] -| Fields describing a location. - -// =============================================================== - - -| http://localhost:8000/ecs-os.html[host.os.*] -| OS fields contain information about the operating system. - -// =============================================================== - - -| http://localhost:8000/ecs-user.html[host.user.*] -| Fields to describe the user relevant to the event. - -// =============================================================== - - -|===== - -[[ecs-organization]] -=== Organization fields - -The organization fields enrich data with information about the company or entity the data is associated with. -These fields help you arrange or filter data stored in an index by one or multiple organizations. - -==== Organization Fields - -[options="header"] -|===== -| Field | Description | Level - -// =============================================================== - -| organization.id -| Unique identifier for the organization. - -type: keyword - - - -| extended - -// =============================================================== - -| organization.name -| Organization name. - -type: keyword - - - -| extended - -// =============================================================== - -|===== - -[[ecs-organization-nestings]] -==== Can be nested under Organization - -[options="header"] -|===== -| Nested fields | Description - -// =============================================================== - - -| http://localhost:8000/ecs-geo.html[host.geo.*] -| Fields describing a location. - -// =============================================================== - - -| http://localhost:8000/ecs-os.html[host.os.*] -| OS fields contain information about the operating system. - -// =============================================================== - - -| http://localhost:8000/ecs-user.html[host.user.*] -| Fields to describe the user relevant to the event. - -// =============================================================== - - -|===== - -[[ecs-os]] -=== Operating System fields - -The OS fields contain information about the operating system. - -==== Operating System Fields - -[options="header"] -|===== -| Field | Description | Level - -// =============================================================== - -| os.family -| OS family (such as redhat, debian, freebsd, windows). - -type: keyword - -example: `debian` - -| extended - -// =============================================================== - -| os.full -| Operating system name, including the version or code name. - -type: keyword - -example: `Mac OS Mojave` - -| extended - -// =============================================================== - -| os.kernel -| Operating system kernel version as a raw string. - -type: keyword - -example: `4.4.0-112-generic` - -| extended - -// =============================================================== - -| os.name -| Operating system name, without the version. - -type: keyword - -example: `Mac OS X` - -| extended - -// =============================================================== - -| os.platform -| Operating system platform (such centos, ubuntu, windows). - -type: keyword - -example: `darwin` - -| extended - -// =============================================================== - -| os.version -| Operating system version as a raw string. - -type: keyword - -example: `10.14.1` - -| extended - -// =============================================================== - -|===== - -[[ecs-os-nestings]] -==== Can be nested under Operating System - -[options="header"] -|===== -| Nested fields | Description - -// =============================================================== - - -| http://localhost:8000/ecs-geo.html[host.geo.*] -| Fields describing a location. - -// =============================================================== - - -| http://localhost:8000/ecs-os.html[host.os.*] -| OS fields contain information about the operating system. - -// =============================================================== - - -| http://localhost:8000/ecs-user.html[host.user.*] -| Fields to describe the user relevant to the event. - -// =============================================================== - - -|===== - -[[ecs-process]] -=== Process fields - -These fields contain information about a process. -These fields can help you correlate metrics information with a process id/name from a log message. The `process.pid` often stays in the metric itself and is copied to the global field for correlation. - -==== Process Fields - -[options="header"] -|===== -| Field | Description | Level - -// =============================================================== - -| process.args -| Array of process arguments. -May be filtered to protect sensitive information. - -type: keyword - -example: `['ssh', '-l', 'user', '10.0.0.16']` - -| extended - -// =============================================================== - -| process.executable -| Absolute path to the process executable. - -type: keyword - -example: `/usr/bin/ssh` - -| extended - -// =============================================================== - -| process.name -| Process name. -Sometimes called program name or similar. - -type: keyword - -example: `ssh` - -| extended - -// =============================================================== - -| process.pid -| Process id. - -type: long - - - -| core - -// =============================================================== - -| process.ppid -| Process parent id. - -type: long - - - -| extended - -// =============================================================== - -| process.start -| The time the process started. - -type: date - -example: `2016-05-23T08:05:34.853Z` - -| extended - -// =============================================================== - -| process.thread.id -| Thread ID. - -type: long - -example: `4242` - -| extended - -// =============================================================== - -| process.title -| Process title. -The proctitle, some times the same as process name. Can also be different: for example a browser setting its title to the web page currently opened. - -type: keyword - - - -| extended - -// =============================================================== - -| process.working_directory -| The working directory of the process. - -type: keyword - -example: `/home/alice` - -| extended - -// =============================================================== - -|===== - -[[ecs-process-nestings]] -==== Can be nested under Process - -[options="header"] -|===== -| Nested fields | Description +| log.level +| Original log level of the log event. +Some examples are `warn`, `error`, `i`. -// =============================================================== +type: keyword +example: `err` -| http://localhost:8000/ecs-geo.html[host.geo.*] -| Fields describing a location. +| core // =============================================================== +| log.original +| This is the original log message and contains the full log message before splitting it up in multiple parts. +In contrast to the `message` field which can contain an extracted part of the log message, this field contains the original, full log message. It can have already some modifications applied like encoding or new lines removed to clean up the log message. +This field is not indexed and doc_values are disabled so it can't be queried but the value can be retrieved from `_source`. -| http://localhost:8000/ecs-os.html[host.os.*] -| OS fields contain information about the operating system. - -// =============================================================== +type: keyword +example: `Sep 19 08:26:10 localhost My log` -| http://localhost:8000/ecs-user.html[host.user.*] -| Fields to describe the user relevant to the event. +| core // =============================================================== - |===== -[[ecs-related]] -=== Related fields +[[ecs-network]] +=== Network fields -This field set is meant to facilitate pivoting around a piece of data. -Some pieces of information can be seen in many places in an ECS event. To facilitate searching for them, store an array of all seen values to their corresponding field in `related.`. -A concrete example is IP addresses, which can be under host, observer, source, destination, client, server, and network.forwarded_ip. If you append all IPs to `related.ip`, you can then search for a given IP trivially, no matter where it appeared, by querying `related.ip:a.b.c.d`. +The network is defined as the communication path over which a host or network event happens. +The network.* fields should be populated with details about the network activity associated with an event. -==== Related Fields +==== Network Fields [options="header"] |===== @@ -3214,193 +1481,230 @@ A concrete example is IP addresses, which can be under host, observer, source, d // =============================================================== -| related.ip -| All of the IPs seen on your event. - -type: ip +| network.application +| A name given to an application level protocol. This can be arbitrarily assigned for things like microservices, but also apply to things like skype, icq, facebook, twitter. This would be used in situations where the vendor or service can be decoded such as from the source/dest IP owners, ports, or wire format. +The field value must be normalized to lowercase for querying. See the documentation section "Implementing ECS". +type: keyword +example: `aim` | extended // =============================================================== -|===== +| network.bytes +| Total bytes transferred in both directions. +If `source.bytes` and `destination.bytes` are known, `network.bytes` is their sum. + +type: long -[[ecs-related-nestings]] -==== Can be nested under Related +example: `368` -[options="header"] -|===== -| Nested fields | Description +| core // =============================================================== +| network.community_id +| A hash of source and destination IPs and ports, as well as the protocol used in a communication. This is a tool-agnostic standard to identify flows. +Learn more at https://github.com/corelight/community-id-spec. + +type: keyword -| http://localhost:8000/ecs-geo.html[host.geo.*] -| Fields describing a location. +example: `1:hO+sN4H+MG5MY/8hIrXPqc4ZQz0=` + +| extended // =============================================================== +| network.direction +| Direction of the network traffic. +Recommended values are: + * inbound + * outbound + * internal + * external + * unknown -| http://localhost:8000/ecs-os.html[host.os.*] -| OS fields contain information about the operating system. +When mapping events from a host-based monitoring context, populate this field from the host's point of view. +When mapping events from a network or perimeter-based monitoring context, populate this field from the point of view of your network perimeter. -// =============================================================== +type: keyword +example: `inbound` -| http://localhost:8000/ecs-user.html[host.user.*] -| Fields to describe the user relevant to the event. +| core // =============================================================== +| network.forwarded_ip +| Host IP address when the source IP address is the proxy. -|===== +type: ip -[[ecs-server]] -=== Server fields +example: `192.1.1.2` -A Server is defined as the responder in a network connection for events regarding sessions, connections, or bidirectional flow records. -For TCP events, the server is the receiver of the initial SYN packet(s) of the TCP connection. For other protocols, the server is generally the responder in the network transaction. Some systems actually use the term "responder" to refer the server in TCP connections. The server fields describe details about the system acting as the server in the network event. Server fields are usually populated in conjunction with client fields. Server fields are generally not populated for packet-level events. -Client / server representations can add semantic context to an exchange, which is helpful to visualize the data in certain situations. If your context falls in that category, you should still ensure that source and destination are filled appropriately. +| core -==== Server Fields +// =============================================================== -[options="header"] -|===== -| Field | Description | Level +| network.iana_number +| IANA Protocol Number (https://www.iana.org/assignments/protocol-numbers/protocol-numbers.xhtml). Standardized list of protocols. This aligns well with NetFlow and sFlow related logs which use the IANA Protocol Number. + +type: keyword + +example: `6` + +| extended // =============================================================== -| server.address -| Some event server addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. -Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. +| network.name +| Name given by operators to sections of their network. type: keyword - +example: `Guest Wifi` | extended // =============================================================== -| server.bytes -| Bytes sent from the server to the client. +| network.packets +| Total packets transferred in both directions. +If `source.packets` and `destination.packets` are known, `network.packets` is their sum. type: long -example: `184` +example: `24` | core // =============================================================== -| server.domain -| Server domain. +| network.protocol +| L7 Network protocol name. ex. http, lumberjack, transport protocol. +The field value must be normalized to lowercase for querying. See the documentation section "Implementing ECS". type: keyword - +example: `http` | core // =============================================================== -| server.geo.city_name -| City name. +| network.transport +| Same as network.iana_number, but instead using the Keyword name of the transport layer (udp, tcp, ipv6-icmp, etc.) +The field value must be normalized to lowercase for querying. See the documentation section "Implementing ECS". type: keyword -example: `Montreal` +example: `tcp` | core // =============================================================== -| server.geo.continent_name -| Name of the continent. +| network.type +| In the OSI Model this would be the Network Layer. ipv4, ipv6, ipsec, pim, etc +The field value must be normalized to lowercase for querying. See the documentation section "Implementing ECS". type: keyword -example: `North America` +example: `ipv4` | core // =============================================================== -| server.geo.country_iso_code -| Country ISO code. +|===== + +[[ecs-observer]] +=== Observer fields + +An observer is defined as a special network, security, or application device used to detect, observe, or create network, security, or application-related events and metrics. +This could be a custom hardware appliance or a server that has been configured to run special network, security, or application software. Examples include firewalls, intrusion detection/prevention systems, network monitoring sensors, web application firewalls, data loss prevention systems, and APM servers. The observer.* fields shall be populated with details of the system, if any, that detects, observes and/or creates a network, security, or application event or metric. Message queues and ETL components used in processing events or metrics are not considered observers in ECS. + +==== Observer Fields + +[options="header"] +|===== +| Field | Description | Level + +// =============================================================== + +| observer.hostname +| Hostname of the observer. type: keyword -example: `CA` + | core // =============================================================== -| server.geo.country_name -| Country name. +| observer.ip +| IP address of the observer. + +type: ip -type: keyword -example: `Canada` | core // =============================================================== -| server.geo.location -| Longitude and latitude. +| observer.mac +| MAC address of the observer + +type: keyword -type: geo_point -example: `{ "lon": -73.614830, "lat": 45.505918 }` | core // =============================================================== -| server.geo.name -| User-defined description of a location, at the level of granularity they care about. -Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. -Not typically used in automated geolocation. +| observer.serial_number +| Observer serial number. type: keyword -example: `boston-dc` + | extended // =============================================================== -| server.geo.region_iso_code -| Region ISO code. +| observer.type +| The type of the observer the data is coming from. +There is no predefined list of observer types. Some examples are `forwarder`, `firewall`, `ids`, `ips`, `proxy`, `poller`, `sensor`, `APM server`. type: keyword -example: `CA-QC` +example: `firewall` | core // =============================================================== -| server.geo.region_name -| Region name. +| observer.vendor +| observer vendor information. type: keyword -example: `Quebec` + | core // =============================================================== -| server.ip -| IP address of the server. -Can be one or multiple IPv4 or IPv6 addresses. +| observer.version +| Observer version. -type: ip +type: keyword @@ -3408,41 +1712,59 @@ type: ip // =============================================================== -| server.mac -| MAC address of the server. +|===== -type: keyword +[[ecs-observer-nestings]] +==== Can be nested under Observer + +[options="header"] +|===== +| Nested fields | Description +// =============================================================== -| core +| http://localhost:8000/ecs-geo.html[observer.geo.*] +| Fields describing a location. // =============================================================== -| server.packets -| Packets sent from the server to the client. -type: long +| http://localhost:8000/ecs-os.html[observer.os.*] +| OS fields contain information about the operating system. -example: `12` +// =============================================================== -| core + +|===== + +[[ecs-organization]] +=== Organization fields + +The organization fields enrich data with information about the company or entity the data is associated with. +These fields help you arrange or filter data stored in an index by one or multiple organizations. + +==== Organization Fields + +[options="header"] +|===== +| Field | Description | Level // =============================================================== -| server.port -| Port of the server. +| organization.id +| Unique identifier for the organization. -type: long +type: keyword -| core +| extended // =============================================================== -| server.user.email -| User email address. +| organization.name +| Organization name. type: keyword @@ -3452,233 +1774,243 @@ type: keyword // =============================================================== -| server.user.full_name -| User's full name, if available. +|===== + +[[ecs-os]] +=== Operating System fields + +The OS fields contain information about the operating system. + +==== Operating System Fields + +[options="header"] +|===== +| Field | Description | Level + +// =============================================================== + +| os.family +| OS family (such as redhat, debian, freebsd, windows). type: keyword -example: `Albert Einstein` +example: `debian` | extended // =============================================================== -| server.user.group.id -| Unique identifier for the group on the system/platform. +| os.full +| Operating system name, including the version or code name. type: keyword - +example: `Mac OS Mojave` | extended // =============================================================== -| server.user.group.name -| Name of the group. +| os.kernel +| Operating system kernel version as a raw string. type: keyword - +example: `4.4.0-112-generic` | extended // =============================================================== -| server.user.hash -| Unique user hash to correlate information for a user in anonymized form. -Useful if `user.id` or `user.name` contain confidential information and cannot be used. +| os.name +| Operating system name, without the version. type: keyword - +example: `Mac OS X` | extended // =============================================================== -| server.user.id -| One or multiple unique identifiers of the user. +| os.platform +| Operating system platform (such centos, ubuntu, windows). type: keyword +example: `darwin` - -| core +| extended // =============================================================== -| server.user.name -| Short name or login of the user. +| os.version +| Operating system version as a raw string. type: keyword -example: `albert` +example: `10.14.1` -| core +| extended // =============================================================== |===== -[[ecs-server-nestings]] -==== Can be nested under Server +[[ecs-process]] +=== Process fields + +These fields contain information about a process. +These fields can help you correlate metrics information with a process id/name from a log message. The `process.pid` often stays in the metric itself and is copied to the global field for correlation. + +==== Process Fields [options="header"] |===== -| Nested fields | Description +| Field | Description | Level // =============================================================== +| process.args +| Array of process arguments. +May be filtered to protect sensitive information. -| http://localhost:8000/ecs-geo.html[host.geo.*] -| Fields describing a location. - -// =============================================================== +type: keyword +example: `['ssh', '-l', 'user', '10.0.0.16']` -| http://localhost:8000/ecs-os.html[host.os.*] -| OS fields contain information about the operating system. +| extended // =============================================================== +| process.executable +| Absolute path to the process executable. -| http://localhost:8000/ecs-user.html[host.user.*] -| Fields to describe the user relevant to the event. +type: keyword -// =============================================================== +example: `/usr/bin/ssh` +| extended -|===== +// =============================================================== -[[ecs-service]] -=== Service fields +| process.name +| Process name. +Sometimes called program name or similar. -The service fields describe the service for or from which the data was collected. -These fields help you find and correlate logs for a specific service and version. +type: keyword -==== Service Fields +example: `ssh` -[options="header"] -|===== -| Field | Description | Level +| extended // =============================================================== -| service.ephemeral_id -| Ephemeral identifier of this service (if one exists). -This id normally changes across restarts, but `service.id` does not. +| process.pid +| Process id. -type: keyword +type: long -example: `8a4f500f` -| extended + +| core // =============================================================== -| service.id -| Unique identifier of the running service. -This id should uniquely identify this service. This makes it possible to correlate logs and metrics for one specific service. -Example: If you are experiencing issues with one redis instance, you can filter on that id to see metrics and logs for that single instance. +| process.ppid +| Process parent id. -type: keyword +type: long -example: `d37e5ebfe0ae6c4972dbe9f0174a1637bb8247f6` -| core + +| extended // =============================================================== -| service.name -| Name of the service data is collected from. -The name of the service is normally user given. This allows if two instances of the same service are running on the same machine they can be differentiated by the `service.name`. -Also it allows for distributed services that run on multiple hosts to correlate the related instances based on the name. -In the case of Elasticsearch the service.name could contain the cluster name. For Beats the service.name is by default a copy of the `service.type` field if no name is specified. +| process.start +| The time the process started. -type: keyword +type: date -example: `elasticsearch-metrics` +example: `2016-05-23T08:05:34.853Z` -| core +| extended // =============================================================== -| service.state -| Current state of the service. - -type: keyword +| process.thread.id +| Thread ID. +type: long +example: `4242` -| core +| extended // =============================================================== -| service.type -| The type of the service data is collected from. -The type can be used to group and correlate logs and metrics from one service type. -Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. +| process.title +| Process title. +The proctitle, some times the same as process name. Can also be different: for example a browser setting its title to the web page currently opened. type: keyword -example: `elasticsearch` -| core + +| extended // =============================================================== -| service.version -| Version of the service the data was collected from. -This allows to look at a data set only for a specific version of a service. +| process.working_directory +| The working directory of the process. type: keyword -example: `3.2.4` +example: `/home/alice` -| core +| extended // =============================================================== |===== -[[ecs-service-nestings]] -==== Can be nested under Service - -[options="header"] -|===== -| Nested fields | Description +[[ecs-related]] +=== Related fields -// =============================================================== +This field set is meant to facilitate pivoting around a piece of data. +Some pieces of information can be seen in many places in an ECS event. To facilitate searching for them, store an array of all seen values to their corresponding field in `related.`. +A concrete example is IP addresses, which can be under host, observer, source, destination, client, server, and network.forwarded_ip. If you append all IPs to `related.ip`, you can then search for a given IP trivially, no matter where it appeared, by querying `related.ip:a.b.c.d`. +==== Related Fields -| http://localhost:8000/ecs-geo.html[host.geo.*] -| Fields describing a location. +[options="header"] +|===== +| Field | Description | Level // =============================================================== +| related.ip +| All of the IPs seen on your event. -| http://localhost:8000/ecs-os.html[host.os.*] -| OS fields contain information about the operating system. +type: ip -// =============================================================== -| http://localhost:8000/ecs-user.html[host.user.*] -| Fields to describe the user relevant to the event. +| extended // =============================================================== - |===== -[[ecs-source]] -=== Source fields +[[ecs-server]] +=== Server fields -Source fields describe details about the source of a packet/event. -Source fields are usually populated in conjunction with destination fields. +A Server is defined as the responder in a network connection for events regarding sessions, connections, or bidirectional flow records. +For TCP events, the server is the receiver of the initial SYN packet(s) of the TCP connection. For other protocols, the server is generally the responder in the network transaction. Some systems actually use the term "responder" to refer the server in TCP connections. The server fields describe details about the system acting as the server in the network event. Server fields are usually populated in conjunction with client fields. Server fields are generally not populated for packet-level events. +Client / server representations can add semantic context to an exchange, which is helpful to visualize the data in certain situations. If your context falls in that category, you should still ensure that source and destination are filled appropriately. -==== Source Fields +==== Server Fields [options="header"] |===== @@ -3686,8 +2018,8 @@ Source fields are usually populated in conjunction with destination fields. // =============================================================== -| source.address -| Some event source addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. +| server.address +| Some event server addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. type: keyword @@ -3698,8 +2030,8 @@ type: keyword // =============================================================== -| source.bytes -| Bytes sent from the source to the destination. +| server.bytes +| Bytes sent from the server to the client. type: long @@ -3709,8 +2041,8 @@ example: `184` // =============================================================== -| source.domain -| Source domain. +| server.domain +| Server domain. type: keyword @@ -3720,101 +2052,134 @@ type: keyword // =============================================================== -| source.geo.city_name -| City name. +| server.ip +| IP address of the server. +Can be one or multiple IPv4 or IPv6 addresses. + +type: ip -type: keyword -example: `Montreal` | core // =============================================================== -| source.geo.continent_name -| Name of the continent. +| server.mac +| MAC address of the server. type: keyword -example: `North America` + | core // =============================================================== -| source.geo.country_iso_code -| Country ISO code. +| server.packets +| Packets sent from the server to the client. -type: keyword +type: long -example: `CA` +example: `12` | core // =============================================================== -| source.geo.country_name -| Country name. +| server.port +| Port of the server. + +type: long -type: keyword -example: `Canada` | core // =============================================================== -| source.geo.location -| Longitude and latitude. +|===== -type: geo_point +[[ecs-server-nestings]] +==== Can be nested under Server + +[options="header"] +|===== +| Nested fields | Description + +// =============================================================== + + +| http://localhost:8000/ecs-geo.html[server.geo.*] +| Fields describing a location. + +// =============================================================== + + +| http://localhost:8000/ecs-user.html[server.user.*] +| Fields to describe the user relevant to the event. + +// =============================================================== + + +|===== + +[[ecs-service]] +=== Service fields + +The service fields describe the service for or from which the data was collected. +These fields help you find and correlate logs for a specific service and version. -example: `{ "lon": -73.614830, "lat": 45.505918 }` +==== Service Fields -| core +[options="header"] +|===== +| Field | Description | Level // =============================================================== -| source.geo.name -| User-defined description of a location, at the level of granularity they care about. -Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. -Not typically used in automated geolocation. +| service.ephemeral_id +| Ephemeral identifier of this service (if one exists). +This id normally changes across restarts, but `service.id` does not. type: keyword -example: `boston-dc` +example: `8a4f500f` | extended // =============================================================== -| source.geo.region_iso_code -| Region ISO code. +| service.id +| Unique identifier of the running service. +This id should uniquely identify this service. This makes it possible to correlate logs and metrics for one specific service. +Example: If you are experiencing issues with one redis instance, you can filter on that id to see metrics and logs for that single instance. type: keyword -example: `CA-QC` +example: `d37e5ebfe0ae6c4972dbe9f0174a1637bb8247f6` | core // =============================================================== -| source.geo.region_name -| Region name. +| service.name +| Name of the service data is collected from. +The name of the service is normally user given. This allows if two instances of the same service are running on the same machine they can be differentiated by the `service.name`. +Also it allows for distributed services that run on multiple hosts to correlate the related instances based on the name. +In the case of Elasticsearch the service.name could contain the cluster name. For Beats the service.name is by default a copy of the `service.type` field if no name is specified. type: keyword -example: `Quebec` +example: `elasticsearch-metrics` | core // =============================================================== -| source.ip -| IP address of the source. -Can be one or multiple IPv4 or IPv6 addresses. +| service.state +| Current state of the service. -type: ip +type: keyword @@ -3822,41 +2187,50 @@ type: ip // =============================================================== -| source.mac -| MAC address of the source. +| service.type +| The type of the service data is collected from. +The type can be used to group and correlate logs and metrics from one service type. +Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. type: keyword - +example: `elasticsearch` | core // =============================================================== -| source.packets -| Packets sent from the source to the destination. +| service.version +| Version of the service the data was collected from. +This allows to look at a data set only for a specific version of a service. -type: long +type: keyword -example: `12` +example: `3.2.4` | core // =============================================================== -| source.port -| Port of the source. +|===== -type: long +[[ecs-source]] +=== Source fields +Source fields describe details about the source of a packet/event. +Source fields are usually populated in conjunction with destination fields. +==== Source Fields -| core +[options="header"] +|===== +| Field | Description | Level // =============================================================== -| source.user.email -| User email address. +| source.address +| Some event source addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. +Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. type: keyword @@ -3866,68 +2240,68 @@ type: keyword // =============================================================== -| source.user.full_name -| User's full name, if available. +| source.bytes +| Bytes sent from the source to the destination. -type: keyword +type: long -example: `Albert Einstein` +example: `184` -| extended +| core // =============================================================== -| source.user.group.id -| Unique identifier for the group on the system/platform. +| source.domain +| Source domain. type: keyword -| extended +| core // =============================================================== -| source.user.group.name -| Name of the group. +| source.ip +| IP address of the source. +Can be one or multiple IPv4 or IPv6 addresses. -type: keyword +type: ip -| extended +| core // =============================================================== -| source.user.hash -| Unique user hash to correlate information for a user in anonymized form. -Useful if `user.id` or `user.name` contain confidential information and cannot be used. +| source.mac +| MAC address of the source. type: keyword -| extended +| core // =============================================================== -| source.user.id -| One or multiple unique identifiers of the user. - -type: keyword +| source.packets +| Packets sent from the source to the destination. +type: long +example: `12` | core // =============================================================== -| source.user.name -| Short name or login of the user. +| source.port +| Port of the source. + +type: long -type: keyword -example: `albert` | core @@ -3945,19 +2319,13 @@ example: `albert` // =============================================================== -| http://localhost:8000/ecs-geo.html[host.geo.*] +| http://localhost:8000/ecs-geo.html[source.geo.*] | Fields describing a location. // =============================================================== -| http://localhost:8000/ecs-os.html[host.os.*] -| OS fields contain information about the operating system. - -// =============================================================== - - -| http://localhost:8000/ecs-user.html[host.user.*] +| http://localhost:8000/ecs-user.html[source.user.*] | Fields to describe the user relevant to the event. // =============================================================== @@ -4094,36 +2462,6 @@ type: keyword // =============================================================== -|===== - -[[ecs-url-nestings]] -==== Can be nested under URL - -[options="header"] -|===== -| Nested fields | Description - -// =============================================================== - - -| http://localhost:8000/ecs-geo.html[host.geo.*] -| Fields describing a location. - -// =============================================================== - - -| http://localhost:8000/ecs-os.html[host.os.*] -| OS fields contain information about the operating system. - -// =============================================================== - - -| http://localhost:8000/ecs-user.html[host.user.*] -| Fields to describe the user relevant to the event. - -// =============================================================== - - |===== [[ecs-user]] @@ -4158,28 +2496,6 @@ type: keyword example: `Albert Einstein` -| extended - -// =============================================================== - -| user.group.id -| Unique identifier for the group on the system/platform. - -type: keyword - - - -| extended - -// =============================================================== - -| user.group.name -| Name of the group. - -type: keyword - - - | extended // =============================================================== @@ -4230,20 +2546,8 @@ example: `albert` // =============================================================== -| http://localhost:8000/ecs-geo.html[host.geo.*] -| Fields describing a location. - -// =============================================================== - - -| http://localhost:8000/ecs-os.html[host.os.*] -| OS fields contain information about the operating system. - -// =============================================================== - - -| http://localhost:8000/ecs-user.html[host.user.*] -| Fields to describe the user relevant to the event. +| http://localhost:8000/ecs-group.html[user.group.*] +| User's group relevant to the event. // =============================================================== @@ -4297,72 +2601,6 @@ example: `Mozilla/5.0 (iPhone; CPU iPhone OS 12_1 like Mac OS X) AppleWebKit/605 // =============================================================== -| user_agent.os.family -| OS family (such as redhat, debian, freebsd, windows). - -type: keyword - -example: `debian` - -| extended - -// =============================================================== - -| user_agent.os.full -| Operating system name, including the version or code name. - -type: keyword - -example: `Mac OS Mojave` - -| extended - -// =============================================================== - -| user_agent.os.kernel -| Operating system kernel version as a raw string. - -type: keyword - -example: `4.4.0-112-generic` - -| extended - -// =============================================================== - -| user_agent.os.name -| Operating system name, without the version. - -type: keyword - -example: `Mac OS X` - -| extended - -// =============================================================== - -| user_agent.os.platform -| Operating system platform (such centos, ubuntu, windows). - -type: keyword - -example: `darwin` - -| extended - -// =============================================================== - -| user_agent.os.version -| Operating system version as a raw string. - -type: keyword - -example: `10.14.1` - -| extended - -// =============================================================== - | user_agent.version | Version of the user agent. @@ -4386,22 +2624,10 @@ example: `12.0` // =============================================================== -| http://localhost:8000/ecs-geo.html[host.geo.*] -| Fields describing a location. - -// =============================================================== - - -| http://localhost:8000/ecs-os.html[host.os.*] +| http://localhost:8000/ecs-os.html[user_agent.os.*] | OS fields contain information about the operating system. // =============================================================== -| http://localhost:8000/ecs-user.html[host.user.*] -| Fields to describe the user relevant to the event. - -// =============================================================== - - |===== diff --git a/generated/ecs/fields_nested.yml b/generated/ecs/fields_nested.yml index 9d62f21376..27a7d0e977 100644 --- a/generated/ecs/fields_nested.yml +++ b/generated/ecs/fields_nested.yml @@ -410,6 +410,9 @@ client: type: keyword group: 2 name: client + nestings: + - geo + - user prefix: client. short: Fields about the client side of a network connection, used with server. title: Client @@ -814,6 +817,9 @@ destination: type: keyword group: 2 name: destination + nestings: + - geo + - user prefix: destination. short: Fields about the destination side of a network connection, used with source. title: Destination @@ -1736,6 +1742,10 @@ host: type: keyword group: 2 name: host + nestings: + - geo + - os + - user prefix: host. short: Fields describing the relevant computing instance. title: Host @@ -2292,6 +2302,9 @@ observer: type: keyword group: 2 name: observer + nestings: + - geo + - os prefix: observer. short: Fields describing an entity observing the event from outside the host. title: Observer @@ -2793,6 +2806,9 @@ server: type: keyword group: 2 name: server + nestings: + - geo + - user prefix: server. short: Fields about the server side of a network connection, used with client. title: Server @@ -3140,6 +3156,9 @@ source: type: keyword group: 2 name: source + nestings: + - geo + - user prefix: source. short: Fields about the source side of a network connection, used with destination. title: Source @@ -3348,6 +3367,8 @@ user: type: keyword group: 2 name: user + nestings: + - group prefix: user. reusable: expected: @@ -3474,6 +3495,8 @@ user_agent: type: keyword group: 2 name: user_agent + nestings: + - os prefix: user_agent. short: Fields to describe a browser user_agent string. title: User agent diff --git a/scripts/generators/asciidoc_fields.py b/scripts/generators/asciidoc_fields.py index 74231ce873..116f980ea4 100644 --- a/scripts/generators/asciidoc_fields.py +++ b/scripts/generators/asciidoc_fields.py @@ -45,11 +45,11 @@ def render_field_index_row(fieldset): def page_field_details(ecs_nested): page_text = '' for fieldset in ecs_helpers.dict_sorted_by_keys(ecs_nested, ['group', 'name']): - page_text += render_fieldset(fieldset) + page_text += render_fieldset(fieldset, ecs_nested) return page_text -def render_fieldset(fieldset): +def render_fieldset(fieldset, ecs_nested): text = field_details_table_header().format( fieldset_name=fieldset['name'], fieldset_description=fieldset['description'], @@ -57,36 +57,25 @@ def render_fieldset(fieldset): ) for field in ecs_helpers.dict_sorted_by_keys(fieldset['fields'], 'flat_name'): - text += render_field_details_row(field) + if 'original_fieldset' not in field: + text += render_field_details_row(field) text += table_footer() - text += nestings_table_header().format( - fieldset_name=fieldset['name'], - fieldset_title=fieldset['title'] - ) - - # TODO Un-hardcode - fieldset['nestings'] = [ - { - 'flat_nesting': 'host.geo.*', - 'name': 'geo', - 'short': 'Fields describing a location.' - }, - { - 'flat_nesting': 'host.os.*', - 'name': 'os', - 'short': 'OS fields contain information about the operating system.' - }, - { - 'flat_nesting': 'host.user.*', - 'name': 'user', - 'short': 'Fields to describe the user relevant to the event.' - } - ] - for nesting in fieldset['nestings']: - text += render_nesting_row(nesting) - text += table_footer() + if 'nestings' in fieldset: + text += nestings_table_header().format( + fieldset_name=fieldset['name'], + fieldset_title=fieldset['title'] + ) + + nestings = [] + for nested_fs_name in sorted(fieldset['nestings']): + text += render_nesting_row({ + 'flat_nesting': "{}.{}.*".format(fieldset['name'], nested_fs_name), + 'name': nested_fs_name, + 'short': ecs_nested[nested_fs_name]['short'] + }) + text += table_footer() return text diff --git a/scripts/schema_reader.py b/scripts/schema_reader.py index f972927cef..9090328e55 100644 --- a/scripts/schema_reader.py +++ b/scripts/schema_reader.py @@ -126,6 +126,13 @@ def duplicate_reusable_fieldsets(schema, fields_flat, fields_nested): if 'reusable' in schema: for new_nesting in schema['reusable']['expected']: + # List field set names expected under another field set. + # E.g. host.nestings = [ 'geo', 'os', 'user' ] + if 'nestings' not in fields_nested[new_nesting]: + fields_nested[new_nesting]['nestings'] = [] + fields_nested[new_nesting]['nestings'].append(schema['name']) + + # Explicitly list all leaf fields coming from field set reuse. for (name, field) in schema['fields'].items(): # Poor folks deepcopy, sorry -- A Rubyist copied_field = field.copy() From ad4abc34a64cacbbe5773a2afedd3dcfd9bf695c Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Tue, 19 Mar 2019 15:58:08 -0400 Subject: [PATCH 26/28] Incorporate sentences on fieldset reuse like in the readme --- docs/field-details.asciidoc | 178 ++++++++++++++++++-------- scripts/generators/asciidoc_fields.py | 28 +++- 2 files changed, 149 insertions(+), 57 deletions(-) diff --git a/docs/field-details.asciidoc b/docs/field-details.asciidoc index fb1ea3b8f0..16767f0db0 100644 --- a/docs/field-details.asciidoc +++ b/docs/field-details.asciidoc @@ -1,10 +1,12 @@ [[ecs-base]] -=== Base fields +=== Base Fields The `base` field set contains all fields which are on the top level. These fields are common across all types of events. -==== Base Fields + + +==== Base Field Details [options="header"] |===== @@ -66,12 +68,14 @@ example: `["production", "env2"]` |===== [[ecs-agent]] -=== Agent fields +=== Agent Fields The agent fields contain the data about the software entity, if any, that collects, detects, or observes events on a host, or takes measurements on a host. Examples include Beats. Agents may also run on observers. ECS agent.* fields shall be populated with details of the agent running on the host or observer where the event happened or the measurement was taken. -==== Agent Fields + + +==== Agent Field Details [options="header"] |===== @@ -142,13 +146,15 @@ example: `6.0.0-rc2` |===== [[ecs-client]] -=== Client fields +=== Client Fields A client is defined as the initiator of a network connection for events regarding sessions, connections, or bidirectional flow records. For TCP events, the client is the initiator of the TCP connection that sends the SYN packet(s). For other protocols, the client is generally the initiator or requestor in the network transaction. Some systems use the term "originator" to refer the client in TCP connections. The client fields describe details about the system acting as the client in the network event. Client fields are usually populated in conjunction with server fields. Client fields are generally not populated for packet-level events. Client / server representations can add semantic context to an exchange, which is helpful to visualize the data in certain situations. If your context falls in that category, you should still ensure that source and destination are filled appropriately. -==== Client Fields + + +==== Client Field Details [options="header"] |===== @@ -262,11 +268,13 @@ type: long |===== [[ecs-cloud]] -=== Cloud fields +=== Cloud Fields Fields related to the cloud or infrastructure the events are coming from. -==== Cloud Fields + + +==== Cloud Field Details [options="header"] |===== @@ -355,12 +363,14 @@ example: `us-east-1` |===== [[ecs-container]] -=== Container fields +=== Container Fields Container fields are used for meta information about the specific container that is the source of information. These fields help correlate data based containers from any runtime. -==== Container Fields + + +==== Container Field Details [options="header"] |===== @@ -437,12 +447,14 @@ example: `docker` |===== [[ecs-destination]] -=== Destination fields +=== Destination Fields Destination fields describe details about the destination of a packet/event. Destination fields are usually populated in conjunction with source fields. -==== Destination Fields + + +==== Destination Field Details [options="header"] |===== @@ -556,11 +568,13 @@ type: long |===== [[ecs-ecs]] -=== ECS fields +=== ECS Fields Meta-information specific to ECS. -==== ECS Fields + + +==== ECS Field Details [options="header"] |===== @@ -583,12 +597,14 @@ example: `1.0.0` |===== [[ecs-error]] -=== Error fields +=== Error Fields These fields can represent errors of any kind. Use them for errors that happen while fetching events or in cases where the event itself contains an error. -==== Error Fields + + +==== Error Field Details [options="header"] |===== @@ -632,12 +648,14 @@ type: text |===== [[ecs-event]] -=== Event fields +=== Event Fields The event fields are used for context information about the log or metric event itself. A log is defined as an event containing details of something that happened. Log events must include the time at which the thing happened. Examples of log events include a process starting on a host, a network packet being sent from a source to a destination, or a network connection between a client and a server being initiated or closed. A metric is defined as an event containing one or more numerical or categorical measurements and the time at which the measurement was taken. Examples of metric events include memory pressure measured on a host, or vulnerabilities measured on a scanned host. -==== Event Fields + + +==== Event Field Details [options="header"] |===== @@ -860,12 +878,14 @@ type: keyword |===== [[ecs-file]] -=== File fields +=== File Fields A file is defined as a set of information that has been created on, or has existed on a filesystem. File objects can be associated with host events, network events, and/or file events (e.g., those produced by File Integrity Monitoring [FIM] products or services). File fields provide details about the affected file associated with the event or metric. -==== File Fields + + +==== File Field Details [options="header"] |===== @@ -1031,12 +1051,18 @@ type: keyword |===== [[ecs-geo]] -=== Geo fields +=== Geo Fields Geo fields can carry data about a specific location related to an event. This geolocation information can be derived from techniques such as Geo IP, or be user-supplied. -==== Geo Fields +The `geo` fields are expected to be nested at: `client.geo`, `destination.geo`, `host.geo`, `observer.geo`, `server.geo`, `source.geo`. + +Note also that the `geo` fields are not expected to be used directly at the top level. + + + +==== Geo Field Details [options="header"] |===== @@ -1137,11 +1163,17 @@ example: `Quebec` |===== [[ecs-group]] -=== Group fields +=== Group Fields The group fields are meant to represent groups that are relevant to the event. -==== Group Fields +The `group` fields are expected to be nested at: `user.group`. + +Note also that the `group` fields may be used directly at the top level. + + + +==== Group Field Details [options="header"] |===== @@ -1174,12 +1206,14 @@ type: keyword |===== [[ecs-host]] -=== Host fields +=== Host Fields A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes. -==== Host Fields + + +==== Host Field Details [options="header"] |===== @@ -1302,11 +1336,13 @@ type: keyword |===== [[ecs-http]] -=== HTTP fields +=== HTTP Fields Fields related to HTTP activity. Use the `url` field set to store the url of the request. -==== HTTP Fields + + +==== HTTP Field Details [options="header"] |===== @@ -1428,11 +1464,13 @@ example: `1.1` |===== [[ecs-log]] -=== Log fields +=== Log Fields Fields which are specific to log events. -==== Log Fields + + +==== Log Field Details [options="header"] |===== @@ -1468,12 +1506,14 @@ example: `Sep 19 08:26:10 localhost My log` |===== [[ecs-network]] -=== Network fields +=== Network Fields The network is defined as the communication path over which a host or network event happens. The network.* fields should be populated with details about the network activity associated with an event. -==== Network Fields + + +==== Network Field Details [options="header"] |===== @@ -1621,12 +1661,14 @@ example: `ipv4` |===== [[ecs-observer]] -=== Observer fields +=== Observer Fields An observer is defined as a special network, security, or application device used to detect, observe, or create network, security, or application-related events and metrics. This could be a custom hardware appliance or a server that has been configured to run special network, security, or application software. Examples include firewalls, intrusion detection/prevention systems, network monitoring sensors, web application firewalls, data loss prevention systems, and APM servers. The observer.* fields shall be populated with details of the system, if any, that detects, observes and/or creates a network, security, or application event or metric. Message queues and ETL components used in processing events or metrics are not considered observers in ECS. -==== Observer Fields + + +==== Observer Field Details [options="header"] |===== @@ -1739,12 +1781,14 @@ type: keyword |===== [[ecs-organization]] -=== Organization fields +=== Organization Fields The organization fields enrich data with information about the company or entity the data is associated with. These fields help you arrange or filter data stored in an index by one or multiple organizations. -==== Organization Fields + + +==== Organization Field Details [options="header"] |===== @@ -1777,11 +1821,17 @@ type: keyword |===== [[ecs-os]] -=== Operating System fields +=== Operating System Fields The OS fields contain information about the operating system. -==== Operating System Fields +The `os` fields are expected to be nested at: `host.os`, `observer.os`, `user_agent.os`. + +Note also that the `os` fields are not expected to be used directly at the top level. + + + +==== Operating System Field Details [options="header"] |===== @@ -1858,12 +1908,14 @@ example: `10.14.1` |===== [[ecs-process]] -=== Process fields +=== Process Fields These fields contain information about a process. These fields can help you correlate metrics information with a process id/name from a log message. The `process.pid` often stays in the metric itself and is copied to the global field for correlation. -==== Process Fields + + +==== Process Field Details [options="header"] |===== @@ -1976,13 +2028,15 @@ example: `/home/alice` |===== [[ecs-related]] -=== Related fields +=== Related Fields This field set is meant to facilitate pivoting around a piece of data. Some pieces of information can be seen in many places in an ECS event. To facilitate searching for them, store an array of all seen values to their corresponding field in `related.`. A concrete example is IP addresses, which can be under host, observer, source, destination, client, server, and network.forwarded_ip. If you append all IPs to `related.ip`, you can then search for a given IP trivially, no matter where it appeared, by querying `related.ip:a.b.c.d`. -==== Related Fields + + +==== Related Field Details [options="header"] |===== @@ -2004,13 +2058,15 @@ type: ip |===== [[ecs-server]] -=== Server fields +=== Server Fields A Server is defined as the responder in a network connection for events regarding sessions, connections, or bidirectional flow records. For TCP events, the server is the receiver of the initial SYN packet(s) of the TCP connection. For other protocols, the server is generally the responder in the network transaction. Some systems actually use the term "responder" to refer the server in TCP connections. The server fields describe details about the system acting as the server in the network event. Server fields are usually populated in conjunction with client fields. Server fields are generally not populated for packet-level events. Client / server representations can add semantic context to an exchange, which is helpful to visualize the data in certain situations. If your context falls in that category, you should still ensure that source and destination are filled appropriately. -==== Server Fields + + +==== Server Field Details [options="header"] |===== @@ -2124,12 +2180,14 @@ type: long |===== [[ecs-service]] -=== Service fields +=== Service Fields The service fields describe the service for or from which the data was collected. These fields help you find and correlate logs for a specific service and version. -==== Service Fields + + +==== Service Field Details [options="header"] |===== @@ -2215,12 +2273,14 @@ example: `3.2.4` |===== [[ecs-source]] -=== Source fields +=== Source Fields Source fields describe details about the source of a packet/event. Source fields are usually populated in conjunction with destination fields. -==== Source Fields + + +==== Source Field Details [options="header"] |===== @@ -2334,11 +2394,13 @@ type: long |===== [[ecs-url]] -=== URL fields +=== URL Fields URL fields provide support for complete or partial URLs, and supports the breaking down into scheme, domain, path, and so on. -==== URL Fields + + +==== URL Field Details [options="header"] |===== @@ -2465,12 +2527,18 @@ type: keyword |===== [[ecs-user]] -=== User fields +=== User Fields The user fields describe information about the user that is relevant to the event. Fields can have one entry or multiple entries. If a user has more than one id, provide an array that includes all of them. -==== User Fields +The `user` fields are expected to be nested at: `client.user`, `destination.user`, `host.user`, `server.user`, `source.user`. + +Note also that the `user` fields may be used directly at the top level. + + + +==== User Field Details [options="header"] |===== @@ -2555,12 +2623,14 @@ example: `albert` |===== [[ecs-user_agent]] -=== User agent fields +=== User agent Fields The user_agent fields normally come from a browser request. They often show up in web service logs coming from the parsed user agent string. -==== User agent Fields + + +==== User agent Field Details [options="header"] |===== diff --git a/scripts/generators/asciidoc_fields.py b/scripts/generators/asciidoc_fields.py index 116f980ea4..3d6d1036a0 100644 --- a/scripts/generators/asciidoc_fields.py +++ b/scripts/generators/asciidoc_fields.py @@ -51,9 +51,10 @@ def page_field_details(ecs_nested): def render_fieldset(fieldset, ecs_nested): text = field_details_table_header().format( + fieldset_title=fieldset['title'], fieldset_name=fieldset['name'], fieldset_description=fieldset['description'], - fieldset_title=fieldset['title'] + fieldset_reuses=render_fieldset_reuses_text(fieldset) ) for field in ecs_helpers.dict_sorted_by_keys(fieldset['fields'], 'flat_name'): @@ -94,6 +95,25 @@ def render_field_details_row(field): return text +def render_fieldset_reuses_text(fieldset): + if 'reusable' not in fieldset: + return '' + + section_name = fieldset['name'] + sorted_fields = sorted(fieldset['reusable']['expected']) + rendered_fields = map(lambda f: "`{}.{}`".format(f, section_name), sorted_fields) + text = "The `{}` fields are expected to be nested at: {}.\n\n".format( + section_name, ', '.join(rendered_fields)) + + if 'top_level' in fieldset['reusable'] and fieldset['reusable']['top_level']: + template = "Note also that the `{}` fields may be used directly at the top level.\n\n" + else: + template = "Note also that the `{}` fields are not expected to " + \ + "be used directly at the top level.\n\n" + text += template.format(section_name) + return text + + def render_nesting_row(nesting): text = nestings_row().format( nesting_name=nesting['name'], @@ -149,11 +169,13 @@ def index_footer(): def field_details_table_header(): return ''' [[ecs-{fieldset_name}]] -=== {fieldset_title} fields +=== {fieldset_title} Fields {fieldset_description} -==== {fieldset_title} Fields +{fieldset_reuses} + +==== {fieldset_title} Field Details [options="header"] |===== From e5ee3b71ebd16f55163437ab6b9fdf87d85c3d2d Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Tue, 19 Mar 2019 16:19:18 -0400 Subject: [PATCH 27/28] Group both sides of field reuse in the same section under main field details --- docs/field-details.asciidoc | 155 +++++++++++++------------- scripts/generators/asciidoc_fields.py | 58 ++++++---- 2 files changed, 117 insertions(+), 96 deletions(-) diff --git a/docs/field-details.asciidoc b/docs/field-details.asciidoc index 16767f0db0..ae8b0162f8 100644 --- a/docs/field-details.asciidoc +++ b/docs/field-details.asciidoc @@ -4,8 +4,6 @@ The `base` field set contains all fields which are on the top level. These fields are common across all types of events. - - ==== Base Field Details [options="header"] @@ -73,8 +71,6 @@ example: `["production", "env2"]` The agent fields contain the data about the software entity, if any, that collects, detects, or observes events on a host, or takes measurements on a host. Examples include Beats. Agents may also run on observers. ECS agent.* fields shall be populated with details of the agent running on the host or observer where the event happened or the measurement was taken. - - ==== Agent Field Details [options="header"] @@ -152,8 +148,6 @@ A client is defined as the initiator of a network connection for events regardin For TCP events, the client is the initiator of the TCP connection that sends the SYN packet(s). For other protocols, the client is generally the initiator or requestor in the network transaction. Some systems use the term "originator" to refer the client in TCP connections. The client fields describe details about the system acting as the client in the network event. Client fields are usually populated in conjunction with server fields. Client fields are generally not populated for packet-level events. Client / server representations can add semantic context to an exchange, which is helpful to visualize the data in certain situations. If your context falls in that category, you should still ensure that source and destination are filled appropriately. - - ==== Client Field Details [options="header"] @@ -243,8 +237,13 @@ type: long |===== +==== Field Reuse + + + + [[ecs-client-nestings]] -==== Can be nested under Client +===== Field sets that can be nested under Client [options="header"] |===== @@ -272,8 +271,6 @@ type: long Fields related to the cloud or infrastructure the events are coming from. - - ==== Cloud Field Details [options="header"] @@ -368,8 +365,6 @@ example: `us-east-1` Container fields are used for meta information about the specific container that is the source of information. These fields help correlate data based containers from any runtime. - - ==== Container Field Details [options="header"] @@ -452,8 +447,6 @@ example: `docker` Destination fields describe details about the destination of a packet/event. Destination fields are usually populated in conjunction with source fields. - - ==== Destination Field Details [options="header"] @@ -543,8 +536,13 @@ type: long |===== +==== Field Reuse + + + + [[ecs-destination-nestings]] -==== Can be nested under Destination +===== Field sets that can be nested under Destination [options="header"] |===== @@ -572,8 +570,6 @@ type: long Meta-information specific to ECS. - - ==== ECS Field Details [options="header"] @@ -602,8 +598,6 @@ example: `1.0.0` These fields can represent errors of any kind. Use them for errors that happen while fetching events or in cases where the event itself contains an error. - - ==== Error Field Details [options="header"] @@ -653,8 +647,6 @@ type: text The event fields are used for context information about the log or metric event itself. A log is defined as an event containing details of something that happened. Log events must include the time at which the thing happened. Examples of log events include a process starting on a host, a network packet being sent from a source to a destination, or a network connection between a client and a server being initiated or closed. A metric is defined as an event containing one or more numerical or categorical measurements and the time at which the measurement was taken. Examples of metric events include memory pressure measured on a host, or vulnerabilities measured on a scanned host. - - ==== Event Field Details [options="header"] @@ -883,8 +875,6 @@ type: keyword A file is defined as a set of information that has been created on, or has existed on a filesystem. File objects can be associated with host events, network events, and/or file events (e.g., those produced by File Integrity Monitoring [FIM] products or services). File fields provide details about the affected file associated with the event or metric. - - ==== File Field Details [options="header"] @@ -1056,12 +1046,6 @@ type: keyword Geo fields can carry data about a specific location related to an event. This geolocation information can be derived from techniques such as Geo IP, or be user-supplied. -The `geo` fields are expected to be nested at: `client.geo`, `destination.geo`, `host.geo`, `observer.geo`, `server.geo`, `source.geo`. - -Note also that the `geo` fields are not expected to be used directly at the top level. - - - ==== Geo Field Details [options="header"] @@ -1162,16 +1146,19 @@ example: `Quebec` |===== -[[ecs-group]] -=== Group Fields +==== Field Reuse -The group fields are meant to represent groups that are relevant to the event. +The `geo` fields are expected to be nested at: `client.geo`, `destination.geo`, `host.geo`, `observer.geo`, `server.geo`, `source.geo`. + +Note also that the `geo` fields are not expected to be used directly at the top level. -The `group` fields are expected to be nested at: `user.group`. -Note also that the `group` fields may be used directly at the top level. +[[ecs-group]] +=== Group Fields + +The group fields are meant to represent groups that are relevant to the event. ==== Group Field Details @@ -1205,14 +1192,21 @@ type: keyword |===== +==== Field Reuse + +The `group` fields are expected to be nested at: `user.group`. + +Note also that the `group` fields may be used directly at the top level. + + + + [[ecs-host]] === Host Fields A host is defined as a general computing instance. ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes. - - ==== Host Field Details [options="header"] @@ -1305,8 +1299,13 @@ type: keyword |===== +==== Field Reuse + + + + [[ecs-host-nestings]] -==== Can be nested under Host +===== Field sets that can be nested under Host [options="header"] |===== @@ -1340,8 +1339,6 @@ type: keyword Fields related to HTTP activity. Use the `url` field set to store the url of the request. - - ==== HTTP Field Details [options="header"] @@ -1468,8 +1465,6 @@ example: `1.1` Fields which are specific to log events. - - ==== Log Field Details [options="header"] @@ -1511,8 +1506,6 @@ example: `Sep 19 08:26:10 localhost My log` The network is defined as the communication path over which a host or network event happens. The network.* fields should be populated with details about the network activity associated with an event. - - ==== Network Field Details [options="header"] @@ -1666,8 +1659,6 @@ example: `ipv4` An observer is defined as a special network, security, or application device used to detect, observe, or create network, security, or application-related events and metrics. This could be a custom hardware appliance or a server that has been configured to run special network, security, or application software. Examples include firewalls, intrusion detection/prevention systems, network monitoring sensors, web application firewalls, data loss prevention systems, and APM servers. The observer.* fields shall be populated with details of the system, if any, that detects, observes and/or creates a network, security, or application event or metric. Message queues and ETL components used in processing events or metrics are not considered observers in ECS. - - ==== Observer Field Details [options="header"] @@ -1756,8 +1747,13 @@ type: keyword |===== +==== Field Reuse + + + + [[ecs-observer-nestings]] -==== Can be nested under Observer +===== Field sets that can be nested under Observer [options="header"] |===== @@ -1786,8 +1782,6 @@ type: keyword The organization fields enrich data with information about the company or entity the data is associated with. These fields help you arrange or filter data stored in an index by one or multiple organizations. - - ==== Organization Field Details [options="header"] @@ -1825,12 +1819,6 @@ type: keyword The OS fields contain information about the operating system. -The `os` fields are expected to be nested at: `host.os`, `observer.os`, `user_agent.os`. - -Note also that the `os` fields are not expected to be used directly at the top level. - - - ==== Operating System Field Details [options="header"] @@ -1907,14 +1895,21 @@ example: `10.14.1` |===== +==== Field Reuse + +The `os` fields are expected to be nested at: `host.os`, `observer.os`, `user_agent.os`. + +Note also that the `os` fields are not expected to be used directly at the top level. + + + + [[ecs-process]] === Process Fields These fields contain information about a process. These fields can help you correlate metrics information with a process id/name from a log message. The `process.pid` often stays in the metric itself and is copied to the global field for correlation. - - ==== Process Field Details [options="header"] @@ -2034,8 +2029,6 @@ This field set is meant to facilitate pivoting around a piece of data. Some pieces of information can be seen in many places in an ECS event. To facilitate searching for them, store an array of all seen values to their corresponding field in `related.`. A concrete example is IP addresses, which can be under host, observer, source, destination, client, server, and network.forwarded_ip. If you append all IPs to `related.ip`, you can then search for a given IP trivially, no matter where it appeared, by querying `related.ip:a.b.c.d`. - - ==== Related Field Details [options="header"] @@ -2064,8 +2057,6 @@ A Server is defined as the responder in a network connection for events regardin For TCP events, the server is the receiver of the initial SYN packet(s) of the TCP connection. For other protocols, the server is generally the responder in the network transaction. Some systems actually use the term "responder" to refer the server in TCP connections. The server fields describe details about the system acting as the server in the network event. Server fields are usually populated in conjunction with client fields. Server fields are generally not populated for packet-level events. Client / server representations can add semantic context to an exchange, which is helpful to visualize the data in certain situations. If your context falls in that category, you should still ensure that source and destination are filled appropriately. - - ==== Server Field Details [options="header"] @@ -2155,8 +2146,13 @@ type: long |===== +==== Field Reuse + + + + [[ecs-server-nestings]] -==== Can be nested under Server +===== Field sets that can be nested under Server [options="header"] |===== @@ -2185,8 +2181,6 @@ type: long The service fields describe the service for or from which the data was collected. These fields help you find and correlate logs for a specific service and version. - - ==== Service Field Details [options="header"] @@ -2278,8 +2272,6 @@ example: `3.2.4` Source fields describe details about the source of a packet/event. Source fields are usually populated in conjunction with destination fields. - - ==== Source Field Details [options="header"] @@ -2369,8 +2361,13 @@ type: long |===== +==== Field Reuse + + + + [[ecs-source-nestings]] -==== Can be nested under Source +===== Field sets that can be nested under Source [options="header"] |===== @@ -2398,8 +2395,6 @@ type: long URL fields provide support for complete or partial URLs, and supports the breaking down into scheme, domain, path, and so on. - - ==== URL Field Details [options="header"] @@ -2532,12 +2527,6 @@ type: keyword The user fields describe information about the user that is relevant to the event. Fields can have one entry or multiple entries. If a user has more than one id, provide an array that includes all of them. -The `user` fields are expected to be nested at: `client.user`, `destination.user`, `host.user`, `server.user`, `source.user`. - -Note also that the `user` fields may be used directly at the top level. - - - ==== User Field Details [options="header"] @@ -2604,8 +2593,17 @@ example: `albert` |===== +==== Field Reuse + +The `user` fields are expected to be nested at: `client.user`, `destination.user`, `host.user`, `server.user`, `source.user`. + +Note also that the `user` fields may be used directly at the top level. + + + + [[ecs-user-nestings]] -==== Can be nested under User +===== Field sets that can be nested under User [options="header"] |===== @@ -2628,8 +2626,6 @@ example: `albert` The user_agent fields normally come from a browser request. They often show up in web service logs coming from the parsed user agent string. - - ==== User agent Field Details [options="header"] @@ -2684,8 +2680,13 @@ example: `12.0` |===== +==== Field Reuse + + + + [[ecs-user_agent-nestings]] -==== Can be nested under User agent +===== Field sets that can be nested under User agent [options="header"] |===== diff --git a/scripts/generators/asciidoc_fields.py b/scripts/generators/asciidoc_fields.py index 3d6d1036a0..6d0bd0c8a7 100644 --- a/scripts/generators/asciidoc_fields.py +++ b/scripts/generators/asciidoc_fields.py @@ -53,8 +53,7 @@ def render_fieldset(fieldset, ecs_nested): text = field_details_table_header().format( fieldset_title=fieldset['title'], fieldset_name=fieldset['name'], - fieldset_description=fieldset['description'], - fieldset_reuses=render_fieldset_reuses_text(fieldset) + fieldset_description=fieldset['description'] ) for field in ecs_helpers.dict_sorted_by_keys(fieldset['fields'], 'flat_name'): @@ -63,20 +62,7 @@ def render_fieldset(fieldset, ecs_nested): text += table_footer() - if 'nestings' in fieldset: - text += nestings_table_header().format( - fieldset_name=fieldset['name'], - fieldset_title=fieldset['title'] - ) - - nestings = [] - for nested_fs_name in sorted(fieldset['nestings']): - text += render_nesting_row({ - 'flat_nesting': "{}.{}.*".format(fieldset['name'], nested_fs_name), - 'name': nested_fs_name, - 'short': ecs_nested[nested_fs_name]['short'] - }) - text += table_footer() + text += render_fieldset_reuse_section(fieldset, ecs_nested) return text @@ -95,7 +81,32 @@ def render_field_details_row(field): return text +def render_fieldset_reuse_section(fieldset, ecs_nested): + '''Render the section on where field set can be nested, and which field sets can be nested here''' + if not ('nestings' in fieldset or 'reusable' in fieldset): + return '' + + text = field_reuse_section().format( + reuse_of_fieldset=render_fieldset_reuses_text(fieldset) + ) + if 'nestings' in fieldset: + text += nestings_table_header().format( + fieldset_name=fieldset['name'], + fieldset_title=fieldset['title'] + ) + nestings = [] + for nested_fs_name in sorted(fieldset['nestings']): + text += render_nesting_row({ + 'flat_nesting': "{}.{}.*".format(fieldset['name'], nested_fs_name), + 'name': nested_fs_name, + 'short': ecs_nested[nested_fs_name]['short'] + }) + text += table_footer() + return text + + def render_fieldset_reuses_text(fieldset): + '''Render where a given field set is expected to be reused''' if 'reusable' not in fieldset: return '' @@ -173,8 +184,6 @@ def field_details_table_header(): {fieldset_description} -{fieldset_reuses} - ==== {fieldset_title} Field Details [options="header"] @@ -200,12 +209,23 @@ def field_details_row(): ''' +# Field reuse + +def field_reuse_section(): + return ''' +==== Field Reuse + +{reuse_of_fieldset} + +''' + + # Nestings table def nestings_table_header(): return ''' [[ecs-{fieldset_name}-nestings]] -==== Can be nested under {fieldset_title} +===== Field sets that can be nested under {fieldset_title} [options="header"] |===== From b1ee4b5ee21682775c3356d73253ec8a1016d232 Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Tue, 19 Mar 2019 23:35:55 -0400 Subject: [PATCH 28/28] Remove obsolete comment --- scripts/schema_reader.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/schema_reader.py b/scripts/schema_reader.py index 9090328e55..8c01756621 100644 --- a/scripts/schema_reader.py +++ b/scripts/schema_reader.py @@ -60,7 +60,6 @@ def schema_set_default_values(schema): dict_set_default(schema, 'short', schema['description']) if "\n" in schema['short']: raise ValueError("Short descriptions must be single line.\nFieldset: {}\n{}".format(schema['name'], schema)) - # print("Short descriptions must be single line. Fieldset: {}".format(schema['name'])) def schema_set_fieldset_prefix(schema):