diff --git a/docs/reference/ingest/processors/user-agent.asciidoc b/docs/reference/ingest/processors/user-agent.asciidoc index 942ba9f148799..152989e956de9 100644 --- a/docs/reference/ingest/processors/user-agent.asciidoc +++ b/docs/reference/ingest/processors/user-agent.asciidoc @@ -19,6 +19,7 @@ The ingest-user-agent module ships by default with the regexes.yaml made availab | `regex_file` | no | - | The name of the file in the `config/ingest-user-agent` directory containing the regular expressions for parsing the user agent string. Both the directory and the file have to be created before starting Elasticsearch. If not specified, ingest-user-agent will use the regexes.yaml from uap-core it ships with (see below). | `properties` | no | [`name`, `major`, `minor`, `patch`, `build`, `os`, `os_name`, `os_major`, `os_minor`, `device`] | Controls what properties are added to `target_field`. | `ignore_missing` | no | `false` | If `true` and `field` does not exist, the processor quietly exits without modifying the document +| `ecs` | no | `true` | Whether to return the output in Elastic Common Schema format. NOTE: This setting is deprecated and will be removed in a future version. |====== Here is an example that adds the user agent details to the `user_agent` field based on the `agent` field: diff --git a/docs/reference/migration/migrate_7_0/settings.asciidoc b/docs/reference/migration/migrate_7_0/settings.asciidoc index 2e5631b378652..0c8d26aae9d53 100644 --- a/docs/reference/migration/migrate_7_0/settings.asciidoc +++ b/docs/reference/migration/migrate_7_0/settings.asciidoc @@ -201,6 +201,6 @@ Elastic Stack to handle the indexing part. [float] [[ingest-user-agent-ecs-always]] -==== Ingest User Agent processor always uses `ecs` output format -The deprecated `ecs` setting for the user agent ingest processor has been -removed. https://github.com/elastic/ecs[ECS] format is now the default. +==== Ingest User Agent processor defaults uses `ecs` output format +https://github.com/elastic/ecs[ECS] format is now the default. +The `ecs` setting for the user agent ingest processor now defaults to true. diff --git a/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java b/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java index d83762a5e47ab..1d5b1e0fbc503 100644 --- a/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java +++ b/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java @@ -27,9 +27,11 @@ import org.elasticsearch.ingest.useragent.UserAgentParser.Details; import org.elasticsearch.ingest.useragent.UserAgentParser.VersionedName; +import java.lang.reflect.Field; import java.util.Arrays; import java.util.EnumSet; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Locale; import java.util.Map; @@ -51,15 +53,17 @@ public class UserAgentProcessor extends AbstractProcessor { private final Set properties; private final UserAgentParser parser; private final boolean ignoreMissing; + private final boolean useECS; public UserAgentProcessor(String tag, String field, String targetField, UserAgentParser parser, Set properties, - boolean ignoreMissing) { + boolean ignoreMissing, boolean useECS) { super(tag); this.field = field; this.targetField = targetField; this.parser = parser; this.properties = properties; this.ignoreMissing = ignoreMissing; + this.useECS = useECS; } boolean isIgnoreMissing() { @@ -80,68 +84,135 @@ public IngestDocument execute(IngestDocument ingestDocument) { Map uaDetails = new HashMap<>(); - // Parse the user agent in the ECS (Elastic Common Schema) format - for (Property property : this.properties) { - switch (property) { - case ORIGINAL: - uaDetails.put("original", userAgent); - break; - case NAME: - if (uaClient.userAgent != null && uaClient.userAgent.name != null) { - uaDetails.put("name", uaClient.userAgent.name); - } else { - uaDetails.put("name", "Other"); - } - break; - case VERSION: - StringBuilder version = new StringBuilder(); - if (uaClient.userAgent != null && uaClient.userAgent.major != null) { - version.append(uaClient.userAgent.major); - if (uaClient.userAgent.minor != null) { - version.append(".").append(uaClient.userAgent.minor); - if (uaClient.userAgent.patch != null) { - version.append(".").append(uaClient.userAgent.patch); - if (uaClient.userAgent.build != null) { - version.append(".").append(uaClient.userAgent.build); + if (useECS) { + // Parse the user agent in the ECS (Elastic Common Schema) format + for (Property property : this.properties) { + switch (property) { + case ORIGINAL: + uaDetails.put("original", userAgent); + break; + case NAME: + if (uaClient.userAgent != null && uaClient.userAgent.name != null) { + uaDetails.put("name", uaClient.userAgent.name); + } else { + uaDetails.put("name", "Other"); + } + break; + case VERSION: + StringBuilder version = new StringBuilder(); + if (uaClient.userAgent != null && uaClient.userAgent.major != null) { + version.append(uaClient.userAgent.major); + if (uaClient.userAgent.minor != null) { + version.append(".").append(uaClient.userAgent.minor); + if (uaClient.userAgent.patch != null) { + version.append(".").append(uaClient.userAgent.patch); + if (uaClient.userAgent.build != null) { + version.append(".").append(uaClient.userAgent.build); + } } } + uaDetails.put("version", version.toString()); } - uaDetails.put("version", version.toString()); - } - break; - case OS: - if (uaClient.operatingSystem != null) { - Map osDetails = new HashMap<>(3); - if (uaClient.operatingSystem.name != null) { - osDetails.put("name", uaClient.operatingSystem.name); - StringBuilder sb = new StringBuilder(); - if (uaClient.operatingSystem.major != null) { - sb.append(uaClient.operatingSystem.major); - if (uaClient.operatingSystem.minor != null) { - sb.append(".").append(uaClient.operatingSystem.minor); - if (uaClient.operatingSystem.patch != null) { - sb.append(".").append(uaClient.operatingSystem.patch); - if (uaClient.operatingSystem.build != null) { - sb.append(".").append(uaClient.operatingSystem.build); + break; + case OS: + if (uaClient.operatingSystem != null) { + Map osDetails = new HashMap<>(3); + if (uaClient.operatingSystem.name != null) { + osDetails.put("name", uaClient.operatingSystem.name); + StringBuilder sb = new StringBuilder(); + if (uaClient.operatingSystem.major != null) { + sb.append(uaClient.operatingSystem.major); + if (uaClient.operatingSystem.minor != null) { + sb.append(".").append(uaClient.operatingSystem.minor); + if (uaClient.operatingSystem.patch != null) { + sb.append(".").append(uaClient.operatingSystem.patch); + if (uaClient.operatingSystem.build != null) { + sb.append(".").append(uaClient.operatingSystem.build); + } } } + osDetails.put("version", sb.toString()); + osDetails.put("full", uaClient.operatingSystem.name + " " + sb.toString()); } - osDetails.put("version", sb.toString()); - osDetails.put("full", uaClient.operatingSystem.name + " " + sb.toString()); + uaDetails.put("os", osDetails); } - uaDetails.put("os", osDetails); } - } - break; - case DEVICE: - Map deviceDetails = new HashMap<>(1); - if (uaClient.device != null && uaClient.device.name != null) { - deviceDetails.put("name", uaClient.device.name); - } else { - deviceDetails.put("name", "Other"); - } - uaDetails.put("device", deviceDetails); - break; + break; + case DEVICE: + Map deviceDetails = new HashMap<>(1); + if (uaClient.device != null && uaClient.device.name != null) { + deviceDetails.put("name", uaClient.device.name); + } else { + deviceDetails.put("name", "Other"); + } + uaDetails.put("device", deviceDetails); + break; + } + } + } else { + // Deprecated format, removed in 8.0 + for (Property property : this.properties) { + switch (property) { + case NAME: + if (uaClient.userAgent != null && uaClient.userAgent.name != null) { + uaDetails.put("name", uaClient.userAgent.name); + } else { + uaDetails.put("name", "Other"); + } + break; + case MAJOR: + if (uaClient.userAgent != null && uaClient.userAgent.major != null) { + uaDetails.put("major", uaClient.userAgent.major); + } + break; + case MINOR: + if (uaClient.userAgent != null && uaClient.userAgent.minor != null) { + uaDetails.put("minor", uaClient.userAgent.minor); + } + break; + case PATCH: + if (uaClient.userAgent != null && uaClient.userAgent.patch != null) { + uaDetails.put("patch", uaClient.userAgent.patch); + } + break; + case BUILD: + if (uaClient.userAgent != null && uaClient.userAgent.build != null) { + uaDetails.put("build", uaClient.userAgent.build); + } + break; + case OS: + if (uaClient.operatingSystem != null) { + uaDetails.put("os", buildFullOSName(uaClient.operatingSystem)); + } else { + uaDetails.put("os", "Other"); + } + + break; + case OS_NAME: + if (uaClient.operatingSystem != null && uaClient.operatingSystem.name != null) { + uaDetails.put("os_name", uaClient.operatingSystem.name); + } else { + uaDetails.put("os_name", "Other"); + } + break; + case OS_MAJOR: + if (uaClient.operatingSystem != null && uaClient.operatingSystem.major != null) { + uaDetails.put("os_major", uaClient.operatingSystem.major); + } + break; + case OS_MINOR: + if (uaClient.operatingSystem != null && uaClient.operatingSystem.minor != null) { + uaDetails.put("os_minor", uaClient.operatingSystem.minor); + } + break; + case DEVICE: + if (uaClient.device != null && uaClient.device.name != null) { + uaDetails.put("device", uaClient.device.name); + } else { + uaDetails.put("device", "Other"); + } + break; + } } } @@ -201,6 +272,10 @@ UserAgentParser getUaParser() { return parser; } + public boolean isUseECS() { + return useECS; + } + public static final class Factory implements Processor.Factory { private final Map userAgentParsers; @@ -217,10 +292,7 @@ public UserAgentProcessor create(Map factories, Strin String regexFilename = readStringProperty(TYPE, processorTag, config, "regex_file", IngestUserAgentPlugin.DEFAULT_PARSER_NAME); List propertyNames = readOptionalList(TYPE, processorTag, config, "properties"); boolean ignoreMissing = readBooleanProperty(TYPE, processorTag, config, "ignore_missing", false); - Object ecsValue = config.remove("ecs"); - if (ecsValue != null) { - deprecationLogger.deprecated("setting [ecs] is deprecated as ECS format is the default and only option"); - } + boolean useECS = readBooleanProperty(TYPE, processorTag, config, "ecs", true); UserAgentParser parser = userAgentParsers.get(regexFilename); if (parser == null) { @@ -242,22 +314,53 @@ public UserAgentProcessor create(Map factories, Strin properties = EnumSet.allOf(Property.class); } - return new UserAgentProcessor(processorTag, field, targetField, parser, properties, ignoreMissing); + if (useECS == false) { + deprecationLogger.deprecated("setting [ecs] to false for non-common schema " + + "format is deprecated and will be removed in 8.0, set to true or remove to use the non-deprecated format"); + } + + return new UserAgentProcessor(processorTag, field, targetField, parser, properties, ignoreMissing, useECS); } } enum Property { NAME, + // Deprecated in 6.7 (superceded by VERSION), to be removed in 8.0 + @Deprecated MAJOR, + @Deprecated MINOR, + @Deprecated PATCH, OS, + // Deprecated in 6.7 (superceded by just using OS), to be removed in 8.0 + @Deprecated OS_NAME, + @Deprecated OS_MAJOR, + @Deprecated OS_MINOR, DEVICE, + @Deprecated BUILD, // Same deprecated as OS_* above ORIGINAL, VERSION; + private static Set DEPRECATED_PROPERTIES; + + static { + Set deprecated = new HashSet<>(); + for (Field field : Property.class.getFields()) { + if (field.isEnumConstant() && field.isAnnotationPresent(Deprecated.class)) { + deprecated.add(valueOf(field.getName())); + } + } + DEPRECATED_PROPERTIES = deprecated; + } + public static Property parseProperty(String propertyName) { try { - return valueOf(propertyName.toUpperCase(Locale.ROOT)); - } catch (IllegalArgumentException e) { + Property value = valueOf(propertyName.toUpperCase(Locale.ROOT)); + if (DEPRECATED_PROPERTIES.contains(value)) { + deprecationLogger.deprecated("the [{}] property is deprecated for the user-agent processor", propertyName); + } + return value; + } + catch (IllegalArgumentException e) { throw new IllegalArgumentException("illegal property value [" + propertyName + "]. valid values are " + Arrays.toString(EnumSet.allOf(Property.class).toArray())); } diff --git a/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorFactoryTests.java b/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorFactoryTests.java index f723c13f23022..c25200b457985 100644 --- a/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorFactoryTests.java +++ b/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorFactoryTests.java @@ -20,6 +20,7 @@ package org.elasticsearch.ingest.useragent; import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.common.Strings; import org.elasticsearch.test.ESTestCase; import org.junit.BeforeClass; @@ -27,17 +28,21 @@ import java.io.BufferedWriter; import java.io.IOException; import java.io.InputStreamReader; +import java.lang.reflect.Field; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.EnumSet; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; @@ -84,12 +89,12 @@ public void testBuildDefaults() throws Exception { UserAgentProcessor processor = factory.create(null, processorTag, config); assertThat(processor.getTag(), equalTo(processorTag)); assertThat(processor.getField(), equalTo("_field")); - assertThat(processor.getTargetField(), equalTo("user_agent")); assertThat(processor.getUaParser().getUaPatterns().size(), greaterThan(0)); assertThat(processor.getUaParser().getOsPatterns().size(), greaterThan(0)); assertThat(processor.getUaParser().getDevicePatterns().size(), greaterThan(0)); assertThat(processor.getProperties(), equalTo(EnumSet.allOf(UserAgentProcessor.Property.class))); assertFalse(processor.isIgnoreMissing()); + assertTrue(processor.isUseECS()); } public void testBuildWithIgnoreMissing() throws Exception { @@ -98,6 +103,7 @@ public void testBuildWithIgnoreMissing() throws Exception { Map config = new HashMap<>(); config.put("field", "_field"); config.put("ignore_missing", true); + config.put("ecs", true); String processorTag = randomAlphaOfLength(10); @@ -118,6 +124,7 @@ public void testBuildTargetField() throws Exception { Map config = new HashMap<>(); config.put("field", "_field"); config.put("target_field", "_target_field"); + config.put("ecs", true); UserAgentProcessor processor = factory.create(null, null, config); assertThat(processor.getField(), equalTo("_field")); @@ -130,6 +137,7 @@ public void testBuildRegexFile() throws Exception { Map config = new HashMap<>(); config.put("field", "_field"); config.put("regex_file", regexWithoutDevicesFilename); + config.put("ecs", true); UserAgentProcessor processor = factory.create(null, null, config); assertThat(processor.getField(), equalTo("_field")); @@ -155,8 +163,17 @@ public void testBuildFields() throws Exception { Set properties = EnumSet.noneOf(UserAgentProcessor.Property.class); List fieldNames = new ArrayList<>(); int numFields = scaledRandomIntBetween(1, UserAgentProcessor.Property.values().length); + Set warnings = new HashSet<>(); + Set deprecated = Arrays.stream(UserAgentProcessor.Property.class.getFields()) + .filter(Field::isEnumConstant) + .filter(field -> field.isAnnotationPresent(Deprecated.class)) + .map(field -> UserAgentProcessor.Property.valueOf(field.getName())) + .collect(Collectors.toSet()); for (int i = 0; i < numFields; i++) { UserAgentProcessor.Property property = UserAgentProcessor.Property.values()[i]; + if (deprecated.contains(property)) { + warnings.add("the [" + property.name().toLowerCase(Locale.ROOT) + "] property is deprecated for the user-agent processor"); + } properties.add(property); fieldNames.add(property.name().toLowerCase(Locale.ROOT)); } @@ -164,10 +181,14 @@ public void testBuildFields() throws Exception { Map config = new HashMap<>(); config.put("field", "_field"); config.put("properties", fieldNames); + config.put("ecs", true); UserAgentProcessor processor = factory.create(null, null, config); assertThat(processor.getField(), equalTo("_field")); assertThat(processor.getProperties(), equalTo(properties)); + if (warnings.size() > 0) { + assertWarnings(warnings.toArray(Strings.EMPTY_ARRAY)); + } } public void testInvalidProperty() throws Exception { @@ -178,8 +199,8 @@ public void testInvalidProperty() throws Exception { config.put("properties", Collections.singletonList("invalid")); ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, config)); - assertThat(e.getMessage(), equalTo("[properties] illegal property value [invalid]. valid values are [NAME, OS, DEVICE, " + - "ORIGINAL, VERSION]")); + assertThat(e.getMessage(), equalTo("[properties] illegal property value [invalid]. valid values are [NAME, MAJOR, MINOR, " + + "PATCH, OS, OS_NAME, OS_MAJOR, OS_MINOR, DEVICE, BUILD, ORIGINAL, VERSION]")); } public void testInvalidPropertiesType() throws Exception { diff --git a/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorTests.java b/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorTests.java index f043cc5369a26..4cb270e75a94b 100644 --- a/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorTests.java +++ b/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorTests.java @@ -48,12 +48,12 @@ public static void setupProcessor() throws IOException { UserAgentParser parser = new UserAgentParser(randomAlphaOfLength(10), regexStream, new UserAgentCache(1000)); processor = new UserAgentProcessor(randomAlphaOfLength(10), "source_field", "target_field", parser, - EnumSet.allOf(UserAgentProcessor.Property.class), false); + EnumSet.allOf(UserAgentProcessor.Property.class), false, true); } public void testNullValueWithIgnoreMissing() throws Exception { UserAgentProcessor processor = new UserAgentProcessor(randomAlphaOfLength(10), "source_field", "target_field", null, - EnumSet.allOf(UserAgentProcessor.Property.class), true); + EnumSet.allOf(UserAgentProcessor.Property.class), true, true); IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("source_field", null)); IngestDocument ingestDocument = new IngestDocument(originalIngestDocument); @@ -63,7 +63,7 @@ public void testNullValueWithIgnoreMissing() throws Exception { public void testNonExistentWithIgnoreMissing() throws Exception { UserAgentProcessor processor = new UserAgentProcessor(randomAlphaOfLength(10), "source_field", "target_field", null, - EnumSet.allOf(UserAgentProcessor.Property.class), true); + EnumSet.allOf(UserAgentProcessor.Property.class), true, true); IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.emptyMap()); IngestDocument ingestDocument = new IngestDocument(originalIngestDocument); processor.execute(ingestDocument); @@ -72,7 +72,7 @@ public void testNonExistentWithIgnoreMissing() throws Exception { public void testNullWithoutIgnoreMissing() throws Exception { UserAgentProcessor processor = new UserAgentProcessor(randomAlphaOfLength(10), "source_field", "target_field", null, - EnumSet.allOf(UserAgentProcessor.Property.class), false); + EnumSet.allOf(UserAgentProcessor.Property.class), false, true); IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("source_field", null)); IngestDocument ingestDocument = new IngestDocument(originalIngestDocument); @@ -82,7 +82,7 @@ public void testNullWithoutIgnoreMissing() throws Exception { public void testNonExistentWithoutIgnoreMissing() throws Exception { UserAgentProcessor processor = new UserAgentProcessor(randomAlphaOfLength(10), "source_field", "target_field", null, - EnumSet.allOf(UserAgentProcessor.Property.class), false); + EnumSet.allOf(UserAgentProcessor.Property.class), false, true); IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.emptyMap()); IngestDocument ingestDocument = new IngestDocument(originalIngestDocument); Exception exception = expectThrows(Exception.class, () -> processor.execute(ingestDocument)); @@ -186,6 +186,7 @@ public void testUnknown() throws Exception { assertNull(target.get("build")); assertNull(target.get("os")); + Map device = new HashMap<>(); device.put("name", "Other"); assertThat(target.get("device"), is(device)); diff --git a/modules/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/20_useragent_processor.yml b/modules/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/20_useragent_processor.yml index a7fe57c557008..f629761bd90a5 100644 --- a/modules/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/20_useragent_processor.yml +++ b/modules/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/20_useragent_processor.yml @@ -46,6 +46,7 @@ { "user_agent" : { "field" : "field1", + "ecs": true, "target_field": "field2", "properties": ["os"] } @@ -69,5 +70,62 @@ - match: { _source.field2.os.full: "Mac OS X 10.9.2" } - is_false: _source.user_agent - is_false: _source.field2.name + - is_false: _source.field2.os_name + - is_false: _source.field2.os_major + - is_false: _source.field2.os_minor + - is_false: _source.field2.major + - is_false: _source.field2.minor + - is_false: _source.field2.patch + - is_false: _source.field2.device + +--- +"Test user agent processor with non-ECS schema": + - skip: + features: warnings + + - do: + warnings: + - "setting [ecs] to false for non-common schema format is deprecated and will be removed in 8.0, set to true or remove to use the non-deprecated format" + - "the [os_major] property is deprecated for the user-agent processor" + ingest.put_pipeline: + id: "my_pipeline" + body: > + { + "description": "_description", + "processors": [ + { + "user_agent" : { + "field" : "field1", + "ecs": false, + "target_field": "field2", + "properties": ["os", "os_major"] + } + } + ] + } + - match: { acknowledged: true } + + - do: + index: + index: test + type: test + id: 1 + pipeline: "my_pipeline" + body: {field1: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.149 Safari/537.36"} + + - do: + get: + index: test + type: test + id: 1 + - match: { _source.field1: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.149 Safari/537.36" } + - match: { _source.field2.os: "Mac OS X 10.9.2" } + - match: { _source.field2.os_major: "10" } + - is_false: _source.user_agent + - is_false: _source.field2.name + - is_false: _source.field2.os_name + - is_false: _source.field2.os_minor + - is_false: _source.field2.major + - is_false: _source.field2.minor + - is_false: _source.field2.patch - is_false: _source.field2.device - - is_false: _source.field2.original diff --git a/modules/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/30_custom_regex.yml b/modules/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/30_custom_regex.yml index 763bea0ee4da0..a2cd419dd69b5 100644 --- a/modules/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/30_custom_regex.yml +++ b/modules/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/30_custom_regex.yml @@ -10,6 +10,7 @@ { "user_agent" : { "field": "field1", + "ecs": true, "regex_file": "test-regexes.yml" } } diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestService.java b/server/src/main/java/org/elasticsearch/ingest/IngestService.java index b2143d72ae65f..31023fc85d5b8 100644 --- a/server/src/main/java/org/elasticsearch/ingest/IngestService.java +++ b/server/src/main/java/org/elasticsearch/ingest/IngestService.java @@ -343,7 +343,7 @@ public String getType() { return new Pipeline(id, description, null, new CompoundProcessor(failureProcessor)); } - static ClusterState innerPut(PutPipelineRequest request, ClusterState currentState) { + public static ClusterState innerPut(PutPipelineRequest request, ClusterState currentState) { IngestMetadata currentIngestMetadata = currentState.metaData().custom(IngestMetadata.TYPE); Map pipelines; if (currentIngestMetadata != null) { diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/ClusterDeprecationChecks.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/ClusterDeprecationChecks.java new file mode 100644 index 0000000000000..38f78b7156e22 --- /dev/null +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/ClusterDeprecationChecks.java @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.deprecation; + +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.ingest.IngestService; +import org.elasticsearch.ingest.PipelineConfiguration; +import org.elasticsearch.xpack.core.deprecation.DeprecationIssue; + +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; + +public class ClusterDeprecationChecks { + + @SuppressWarnings("unchecked") + static DeprecationIssue checkUserAgentPipelines(ClusterState state) { + List pipelines = IngestService.getPipelines(state); + + List pipelinesWithDeprecatedEcsConfig = pipelines.stream() + .filter(Objects::nonNull) + .filter(pipeline -> { + Map pipelineConfig = pipeline.getConfigAsMap(); + + List>> processors = + (List>>) pipelineConfig.get("processors"); + return processors.stream() + .filter(Objects::nonNull) + .filter(processor -> processor.containsKey("user_agent")) + .map(processor -> processor.get("user_agent")) + .anyMatch(processorConfig -> processorConfig.containsKey("ecs")); + }) + .map(PipelineConfiguration::getId) + .sorted() // Make the warning consistent for testing purposes + .collect(Collectors.toList()); + if (pipelinesWithDeprecatedEcsConfig.isEmpty() == false) { + return new DeprecationIssue(DeprecationIssue.Level.WARNING, + "User-Agent ingest plugin will always use ECS-formatted output", + "https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-8.0.html" + + "#ingest-user-agent-ecs-always", + "Ingest pipelines " + pipelinesWithDeprecatedEcsConfig + " uses the [ecs] option which needs to be removed to work in 8.0"); + } + return null; + + } +} diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java index 97e273b213b79..5b980d851fcae 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java @@ -31,7 +31,10 @@ private DeprecationChecks() { } static List> CLUSTER_SETTINGS_CHECKS = - Collections.emptyList(); + Collections.unmodifiableList(Arrays.asList( + ClusterDeprecationChecks::checkUserAgentPipelines + )); + static List> NODE_SETTINGS_CHECKS = Collections.unmodifiableList(Arrays.asList( diff --git a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/ClusterDeprecationChecksTests.java b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/ClusterDeprecationChecksTests.java new file mode 100644 index 0000000000000..7f31067b6bc9e --- /dev/null +++ b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/ClusterDeprecationChecksTests.java @@ -0,0 +1,76 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.deprecation; + +import org.elasticsearch.action.ingest.PutPipelineRequest; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.ingest.IngestService; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.core.deprecation.DeprecationIssue; + +import java.util.List; + +import static java.util.Collections.singletonList; +import static org.elasticsearch.xpack.deprecation.DeprecationChecks.CLUSTER_SETTINGS_CHECKS; + +public class ClusterDeprecationChecksTests extends ESTestCase { + + public void testUserAgentEcsCheck() { + PutPipelineRequest ecsFalseRequest = new PutPipelineRequest("ecs_false", + new BytesArray("{\n" + + " \"description\" : \"This has ecs set to false\",\n" + + " \"processors\" : [\n" + + " {\n" + + " \"user_agent\" : {\n" + + " \"field\" : \"agent\",\n" + + " \"ecs\" : false\n" + + " }\n" + + " }\n" + + " ]\n" + + "}"), XContentType.JSON); + PutPipelineRequest ecsNullRequest = new PutPipelineRequest("ecs_null", + new BytesArray("{\n" + + " \"description\" : \"This has ecs set to false\",\n" + + " \"processors\" : [\n" + + " {\n" + + " \"user_agent\" : {\n" + + " \"field\" : \"agent\"\n" + + " }\n" + + " }\n" + + " ]\n" + + "}"), XContentType.JSON); + PutPipelineRequest ecsTrueRequest = new PutPipelineRequest("ecs_true", + new BytesArray("{\n" + + " \"description\" : \"This has ecs set to false\",\n" + + " \"processors\" : [\n" + + " {\n" + + " \"user_agent\" : {\n" + + " \"field\" : \"agent\",\n" + + " \"ecs\" : true\n" + + " }\n" + + " }\n" + + " ]\n" + + "}"), XContentType.JSON); + + ClusterState state = ClusterState.builder(new ClusterName("test")).build(); + state = IngestService.innerPut(ecsTrueRequest, state); + state = IngestService.innerPut(ecsFalseRequest, state); + state = IngestService.innerPut(ecsNullRequest, state); + + final ClusterState finalState = state; + List issues = DeprecationChecks.filterChecks(CLUSTER_SETTINGS_CHECKS, c -> c.apply(finalState)); + + DeprecationIssue expected = new DeprecationIssue(DeprecationIssue.Level.WARNING, + "User-Agent ingest plugin will always use ECS-formatted output", + "https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-8.0.html" + + "#ingest-user-agent-ecs-always", + "Ingest pipelines [ecs_false, ecs_true] uses the [ecs] option which needs to be removed to work in 8.0"); + assertEquals(singletonList(expected), issues); + } +}