From 08499fa2a7fc60a9c2e5a7b6d7c046b465926d34 Mon Sep 17 00:00:00 2001 From: Jake Landis Date: Tue, 12 Feb 2019 18:53:52 -0600 Subject: [PATCH] Make 7.0 like 6.7 user agent ecs, but default to true (#38757) This change reverts the initial 7.0 commits and replaces them with the 6.7 variant that still allows for the ecs flag. This commit differs from the 6.7 variants in that ecs flag will now default to true. 6.7: `ecs` : default `false` 7.x: `ecs` : default `true` 8.0: no option, but behaves as `true` * Revert "Ingest node - user agent, move device to an object (#38115)" This reverts commit 5b008a34aa3c07e37b12b415d3c22a44da491329. * Revert "Add ECS schema for user-agent ingest processor (#37727) (#37984)" This reverts commit cac6b8e06f051d68919faf6081f1c87fa5b6757d. * cherry-pick 5dfe1935345da3799931fd4a3ebe0b6aa9c17f57 Add ECS schema for user-agent ingest processor (#37727) * cherry-pick ec8ddc890a34853ee8db6af66f608b0ad0cd1099 Ingest node - user agent, move device to an object (#38115) (#38121) * cherry-pick f63cbdb9b426ba24ee4d987ca767ca05a22f2fbb (with manual merge fixes) Dep. check for ECS changes to User Agent processor (#38362) * make true the default for the ecs option, and update 7.0 references and tests --- .../ingest/processors/user-agent.asciidoc | 1 + .../migration/migrate_7_0/settings.asciidoc | 6 +- .../ingest/useragent/UserAgentProcessor.java | 227 +++++++++++++----- .../UserAgentProcessorFactoryTests.java | 27 ++- .../useragent/UserAgentProcessorTests.java | 11 +- .../20_useragent_processor.yml | 60 ++++- .../test/ingest-useragent/30_custom_regex.yml | 1 + .../elasticsearch/ingest/IngestService.java | 2 +- .../deprecation/ClusterDeprecationChecks.java | 51 ++++ .../xpack/deprecation/DeprecationChecks.java | 5 +- .../ClusterDeprecationChecksTests.java | 76 ++++++ 11 files changed, 391 insertions(+), 76 deletions(-) create mode 100644 x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/ClusterDeprecationChecks.java create mode 100644 x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/ClusterDeprecationChecksTests.java diff --git a/docs/reference/ingest/processors/user-agent.asciidoc b/docs/reference/ingest/processors/user-agent.asciidoc index 942ba9f148799..152989e956de9 100644 --- a/docs/reference/ingest/processors/user-agent.asciidoc +++ b/docs/reference/ingest/processors/user-agent.asciidoc @@ -19,6 +19,7 @@ The ingest-user-agent module ships by default with the regexes.yaml made availab | `regex_file` | no | - | The name of the file in the `config/ingest-user-agent` directory containing the regular expressions for parsing the user agent string. Both the directory and the file have to be created before starting Elasticsearch. If not specified, ingest-user-agent will use the regexes.yaml from uap-core it ships with (see below). | `properties` | no | [`name`, `major`, `minor`, `patch`, `build`, `os`, `os_name`, `os_major`, `os_minor`, `device`] | Controls what properties are added to `target_field`. | `ignore_missing` | no | `false` | If `true` and `field` does not exist, the processor quietly exits without modifying the document +| `ecs` | no | `true` | Whether to return the output in Elastic Common Schema format. NOTE: This setting is deprecated and will be removed in a future version. |====== Here is an example that adds the user agent details to the `user_agent` field based on the `agent` field: diff --git a/docs/reference/migration/migrate_7_0/settings.asciidoc b/docs/reference/migration/migrate_7_0/settings.asciidoc index 2e5631b378652..0c8d26aae9d53 100644 --- a/docs/reference/migration/migrate_7_0/settings.asciidoc +++ b/docs/reference/migration/migrate_7_0/settings.asciidoc @@ -201,6 +201,6 @@ Elastic Stack to handle the indexing part. [float] [[ingest-user-agent-ecs-always]] -==== Ingest User Agent processor always uses `ecs` output format -The deprecated `ecs` setting for the user agent ingest processor has been -removed. https://github.com/elastic/ecs[ECS] format is now the default. +==== Ingest User Agent processor defaults uses `ecs` output format +https://github.com/elastic/ecs[ECS] format is now the default. +The `ecs` setting for the user agent ingest processor now defaults to true. diff --git a/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java b/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java index d83762a5e47ab..1d5b1e0fbc503 100644 --- a/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java +++ b/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java @@ -27,9 +27,11 @@ import org.elasticsearch.ingest.useragent.UserAgentParser.Details; import org.elasticsearch.ingest.useragent.UserAgentParser.VersionedName; +import java.lang.reflect.Field; import java.util.Arrays; import java.util.EnumSet; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Locale; import java.util.Map; @@ -51,15 +53,17 @@ public class UserAgentProcessor extends AbstractProcessor { private final Set properties; private final UserAgentParser parser; private final boolean ignoreMissing; + private final boolean useECS; public UserAgentProcessor(String tag, String field, String targetField, UserAgentParser parser, Set properties, - boolean ignoreMissing) { + boolean ignoreMissing, boolean useECS) { super(tag); this.field = field; this.targetField = targetField; this.parser = parser; this.properties = properties; this.ignoreMissing = ignoreMissing; + this.useECS = useECS; } boolean isIgnoreMissing() { @@ -80,68 +84,135 @@ public IngestDocument execute(IngestDocument ingestDocument) { Map uaDetails = new HashMap<>(); - // Parse the user agent in the ECS (Elastic Common Schema) format - for (Property property : this.properties) { - switch (property) { - case ORIGINAL: - uaDetails.put("original", userAgent); - break; - case NAME: - if (uaClient.userAgent != null && uaClient.userAgent.name != null) { - uaDetails.put("name", uaClient.userAgent.name); - } else { - uaDetails.put("name", "Other"); - } - break; - case VERSION: - StringBuilder version = new StringBuilder(); - if (uaClient.userAgent != null && uaClient.userAgent.major != null) { - version.append(uaClient.userAgent.major); - if (uaClient.userAgent.minor != null) { - version.append(".").append(uaClient.userAgent.minor); - if (uaClient.userAgent.patch != null) { - version.append(".").append(uaClient.userAgent.patch); - if (uaClient.userAgent.build != null) { - version.append(".").append(uaClient.userAgent.build); + if (useECS) { + // Parse the user agent in the ECS (Elastic Common Schema) format + for (Property property : this.properties) { + switch (property) { + case ORIGINAL: + uaDetails.put("original", userAgent); + break; + case NAME: + if (uaClient.userAgent != null && uaClient.userAgent.name != null) { + uaDetails.put("name", uaClient.userAgent.name); + } else { + uaDetails.put("name", "Other"); + } + break; + case VERSION: + StringBuilder version = new StringBuilder(); + if (uaClient.userAgent != null && uaClient.userAgent.major != null) { + version.append(uaClient.userAgent.major); + if (uaClient.userAgent.minor != null) { + version.append(".").append(uaClient.userAgent.minor); + if (uaClient.userAgent.patch != null) { + version.append(".").append(uaClient.userAgent.patch); + if (uaClient.userAgent.build != null) { + version.append(".").append(uaClient.userAgent.build); + } } } + uaDetails.put("version", version.toString()); } - uaDetails.put("version", version.toString()); - } - break; - case OS: - if (uaClient.operatingSystem != null) { - Map osDetails = new HashMap<>(3); - if (uaClient.operatingSystem.name != null) { - osDetails.put("name", uaClient.operatingSystem.name); - StringBuilder sb = new StringBuilder(); - if (uaClient.operatingSystem.major != null) { - sb.append(uaClient.operatingSystem.major); - if (uaClient.operatingSystem.minor != null) { - sb.append(".").append(uaClient.operatingSystem.minor); - if (uaClient.operatingSystem.patch != null) { - sb.append(".").append(uaClient.operatingSystem.patch); - if (uaClient.operatingSystem.build != null) { - sb.append(".").append(uaClient.operatingSystem.build); + break; + case OS: + if (uaClient.operatingSystem != null) { + Map osDetails = new HashMap<>(3); + if (uaClient.operatingSystem.name != null) { + osDetails.put("name", uaClient.operatingSystem.name); + StringBuilder sb = new StringBuilder(); + if (uaClient.operatingSystem.major != null) { + sb.append(uaClient.operatingSystem.major); + if (uaClient.operatingSystem.minor != null) { + sb.append(".").append(uaClient.operatingSystem.minor); + if (uaClient.operatingSystem.patch != null) { + sb.append(".").append(uaClient.operatingSystem.patch); + if (uaClient.operatingSystem.build != null) { + sb.append(".").append(uaClient.operatingSystem.build); + } } } + osDetails.put("version", sb.toString()); + osDetails.put("full", uaClient.operatingSystem.name + " " + sb.toString()); } - osDetails.put("version", sb.toString()); - osDetails.put("full", uaClient.operatingSystem.name + " " + sb.toString()); + uaDetails.put("os", osDetails); } - uaDetails.put("os", osDetails); } - } - break; - case DEVICE: - Map deviceDetails = new HashMap<>(1); - if (uaClient.device != null && uaClient.device.name != null) { - deviceDetails.put("name", uaClient.device.name); - } else { - deviceDetails.put("name", "Other"); - } - uaDetails.put("device", deviceDetails); - break; + break; + case DEVICE: + Map deviceDetails = new HashMap<>(1); + if (uaClient.device != null && uaClient.device.name != null) { + deviceDetails.put("name", uaClient.device.name); + } else { + deviceDetails.put("name", "Other"); + } + uaDetails.put("device", deviceDetails); + break; + } + } + } else { + // Deprecated format, removed in 8.0 + for (Property property : this.properties) { + switch (property) { + case NAME: + if (uaClient.userAgent != null && uaClient.userAgent.name != null) { + uaDetails.put("name", uaClient.userAgent.name); + } else { + uaDetails.put("name", "Other"); + } + break; + case MAJOR: + if (uaClient.userAgent != null && uaClient.userAgent.major != null) { + uaDetails.put("major", uaClient.userAgent.major); + } + break; + case MINOR: + if (uaClient.userAgent != null && uaClient.userAgent.minor != null) { + uaDetails.put("minor", uaClient.userAgent.minor); + } + break; + case PATCH: + if (uaClient.userAgent != null && uaClient.userAgent.patch != null) { + uaDetails.put("patch", uaClient.userAgent.patch); + } + break; + case BUILD: + if (uaClient.userAgent != null && uaClient.userAgent.build != null) { + uaDetails.put("build", uaClient.userAgent.build); + } + break; + case OS: + if (uaClient.operatingSystem != null) { + uaDetails.put("os", buildFullOSName(uaClient.operatingSystem)); + } else { + uaDetails.put("os", "Other"); + } + + break; + case OS_NAME: + if (uaClient.operatingSystem != null && uaClient.operatingSystem.name != null) { + uaDetails.put("os_name", uaClient.operatingSystem.name); + } else { + uaDetails.put("os_name", "Other"); + } + break; + case OS_MAJOR: + if (uaClient.operatingSystem != null && uaClient.operatingSystem.major != null) { + uaDetails.put("os_major", uaClient.operatingSystem.major); + } + break; + case OS_MINOR: + if (uaClient.operatingSystem != null && uaClient.operatingSystem.minor != null) { + uaDetails.put("os_minor", uaClient.operatingSystem.minor); + } + break; + case DEVICE: + if (uaClient.device != null && uaClient.device.name != null) { + uaDetails.put("device", uaClient.device.name); + } else { + uaDetails.put("device", "Other"); + } + break; + } } } @@ -201,6 +272,10 @@ UserAgentParser getUaParser() { return parser; } + public boolean isUseECS() { + return useECS; + } + public static final class Factory implements Processor.Factory { private final Map userAgentParsers; @@ -217,10 +292,7 @@ public UserAgentProcessor create(Map factories, Strin String regexFilename = readStringProperty(TYPE, processorTag, config, "regex_file", IngestUserAgentPlugin.DEFAULT_PARSER_NAME); List propertyNames = readOptionalList(TYPE, processorTag, config, "properties"); boolean ignoreMissing = readBooleanProperty(TYPE, processorTag, config, "ignore_missing", false); - Object ecsValue = config.remove("ecs"); - if (ecsValue != null) { - deprecationLogger.deprecated("setting [ecs] is deprecated as ECS format is the default and only option"); - } + boolean useECS = readBooleanProperty(TYPE, processorTag, config, "ecs", true); UserAgentParser parser = userAgentParsers.get(regexFilename); if (parser == null) { @@ -242,22 +314,53 @@ public UserAgentProcessor create(Map factories, Strin properties = EnumSet.allOf(Property.class); } - return new UserAgentProcessor(processorTag, field, targetField, parser, properties, ignoreMissing); + if (useECS == false) { + deprecationLogger.deprecated("setting [ecs] to false for non-common schema " + + "format is deprecated and will be removed in 8.0, set to true or remove to use the non-deprecated format"); + } + + return new UserAgentProcessor(processorTag, field, targetField, parser, properties, ignoreMissing, useECS); } } enum Property { NAME, + // Deprecated in 6.7 (superceded by VERSION), to be removed in 8.0 + @Deprecated MAJOR, + @Deprecated MINOR, + @Deprecated PATCH, OS, + // Deprecated in 6.7 (superceded by just using OS), to be removed in 8.0 + @Deprecated OS_NAME, + @Deprecated OS_MAJOR, + @Deprecated OS_MINOR, DEVICE, + @Deprecated BUILD, // Same deprecated as OS_* above ORIGINAL, VERSION; + private static Set DEPRECATED_PROPERTIES; + + static { + Set deprecated = new HashSet<>(); + for (Field field : Property.class.getFields()) { + if (field.isEnumConstant() && field.isAnnotationPresent(Deprecated.class)) { + deprecated.add(valueOf(field.getName())); + } + } + DEPRECATED_PROPERTIES = deprecated; + } + public static Property parseProperty(String propertyName) { try { - return valueOf(propertyName.toUpperCase(Locale.ROOT)); - } catch (IllegalArgumentException e) { + Property value = valueOf(propertyName.toUpperCase(Locale.ROOT)); + if (DEPRECATED_PROPERTIES.contains(value)) { + deprecationLogger.deprecated("the [{}] property is deprecated for the user-agent processor", propertyName); + } + return value; + } + catch (IllegalArgumentException e) { throw new IllegalArgumentException("illegal property value [" + propertyName + "]. valid values are " + Arrays.toString(EnumSet.allOf(Property.class).toArray())); } diff --git a/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorFactoryTests.java b/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorFactoryTests.java index f723c13f23022..c25200b457985 100644 --- a/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorFactoryTests.java +++ b/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorFactoryTests.java @@ -20,6 +20,7 @@ package org.elasticsearch.ingest.useragent; import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.common.Strings; import org.elasticsearch.test.ESTestCase; import org.junit.BeforeClass; @@ -27,17 +28,21 @@ import java.io.BufferedWriter; import java.io.IOException; import java.io.InputStreamReader; +import java.lang.reflect.Field; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.EnumSet; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; @@ -84,12 +89,12 @@ public void testBuildDefaults() throws Exception { UserAgentProcessor processor = factory.create(null, processorTag, config); assertThat(processor.getTag(), equalTo(processorTag)); assertThat(processor.getField(), equalTo("_field")); - assertThat(processor.getTargetField(), equalTo("user_agent")); assertThat(processor.getUaParser().getUaPatterns().size(), greaterThan(0)); assertThat(processor.getUaParser().getOsPatterns().size(), greaterThan(0)); assertThat(processor.getUaParser().getDevicePatterns().size(), greaterThan(0)); assertThat(processor.getProperties(), equalTo(EnumSet.allOf(UserAgentProcessor.Property.class))); assertFalse(processor.isIgnoreMissing()); + assertTrue(processor.isUseECS()); } public void testBuildWithIgnoreMissing() throws Exception { @@ -98,6 +103,7 @@ public void testBuildWithIgnoreMissing() throws Exception { Map config = new HashMap<>(); config.put("field", "_field"); config.put("ignore_missing", true); + config.put("ecs", true); String processorTag = randomAlphaOfLength(10); @@ -118,6 +124,7 @@ public void testBuildTargetField() throws Exception { Map config = new HashMap<>(); config.put("field", "_field"); config.put("target_field", "_target_field"); + config.put("ecs", true); UserAgentProcessor processor = factory.create(null, null, config); assertThat(processor.getField(), equalTo("_field")); @@ -130,6 +137,7 @@ public void testBuildRegexFile() throws Exception { Map config = new HashMap<>(); config.put("field", "_field"); config.put("regex_file", regexWithoutDevicesFilename); + config.put("ecs", true); UserAgentProcessor processor = factory.create(null, null, config); assertThat(processor.getField(), equalTo("_field")); @@ -155,8 +163,17 @@ public void testBuildFields() throws Exception { Set properties = EnumSet.noneOf(UserAgentProcessor.Property.class); List fieldNames = new ArrayList<>(); int numFields = scaledRandomIntBetween(1, UserAgentProcessor.Property.values().length); + Set warnings = new HashSet<>(); + Set deprecated = Arrays.stream(UserAgentProcessor.Property.class.getFields()) + .filter(Field::isEnumConstant) + .filter(field -> field.isAnnotationPresent(Deprecated.class)) + .map(field -> UserAgentProcessor.Property.valueOf(field.getName())) + .collect(Collectors.toSet()); for (int i = 0; i < numFields; i++) { UserAgentProcessor.Property property = UserAgentProcessor.Property.values()[i]; + if (deprecated.contains(property)) { + warnings.add("the [" + property.name().toLowerCase(Locale.ROOT) + "] property is deprecated for the user-agent processor"); + } properties.add(property); fieldNames.add(property.name().toLowerCase(Locale.ROOT)); } @@ -164,10 +181,14 @@ public void testBuildFields() throws Exception { Map config = new HashMap<>(); config.put("field", "_field"); config.put("properties", fieldNames); + config.put("ecs", true); UserAgentProcessor processor = factory.create(null, null, config); assertThat(processor.getField(), equalTo("_field")); assertThat(processor.getProperties(), equalTo(properties)); + if (warnings.size() > 0) { + assertWarnings(warnings.toArray(Strings.EMPTY_ARRAY)); + } } public void testInvalidProperty() throws Exception { @@ -178,8 +199,8 @@ public void testInvalidProperty() throws Exception { config.put("properties", Collections.singletonList("invalid")); ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, config)); - assertThat(e.getMessage(), equalTo("[properties] illegal property value [invalid]. valid values are [NAME, OS, DEVICE, " + - "ORIGINAL, VERSION]")); + assertThat(e.getMessage(), equalTo("[properties] illegal property value [invalid]. valid values are [NAME, MAJOR, MINOR, " + + "PATCH, OS, OS_NAME, OS_MAJOR, OS_MINOR, DEVICE, BUILD, ORIGINAL, VERSION]")); } public void testInvalidPropertiesType() throws Exception { diff --git a/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorTests.java b/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorTests.java index f043cc5369a26..4cb270e75a94b 100644 --- a/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorTests.java +++ b/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorTests.java @@ -48,12 +48,12 @@ public static void setupProcessor() throws IOException { UserAgentParser parser = new UserAgentParser(randomAlphaOfLength(10), regexStream, new UserAgentCache(1000)); processor = new UserAgentProcessor(randomAlphaOfLength(10), "source_field", "target_field", parser, - EnumSet.allOf(UserAgentProcessor.Property.class), false); + EnumSet.allOf(UserAgentProcessor.Property.class), false, true); } public void testNullValueWithIgnoreMissing() throws Exception { UserAgentProcessor processor = new UserAgentProcessor(randomAlphaOfLength(10), "source_field", "target_field", null, - EnumSet.allOf(UserAgentProcessor.Property.class), true); + EnumSet.allOf(UserAgentProcessor.Property.class), true, true); IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("source_field", null)); IngestDocument ingestDocument = new IngestDocument(originalIngestDocument); @@ -63,7 +63,7 @@ public void testNullValueWithIgnoreMissing() throws Exception { public void testNonExistentWithIgnoreMissing() throws Exception { UserAgentProcessor processor = new UserAgentProcessor(randomAlphaOfLength(10), "source_field", "target_field", null, - EnumSet.allOf(UserAgentProcessor.Property.class), true); + EnumSet.allOf(UserAgentProcessor.Property.class), true, true); IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.emptyMap()); IngestDocument ingestDocument = new IngestDocument(originalIngestDocument); processor.execute(ingestDocument); @@ -72,7 +72,7 @@ public void testNonExistentWithIgnoreMissing() throws Exception { public void testNullWithoutIgnoreMissing() throws Exception { UserAgentProcessor processor = new UserAgentProcessor(randomAlphaOfLength(10), "source_field", "target_field", null, - EnumSet.allOf(UserAgentProcessor.Property.class), false); + EnumSet.allOf(UserAgentProcessor.Property.class), false, true); IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("source_field", null)); IngestDocument ingestDocument = new IngestDocument(originalIngestDocument); @@ -82,7 +82,7 @@ public void testNullWithoutIgnoreMissing() throws Exception { public void testNonExistentWithoutIgnoreMissing() throws Exception { UserAgentProcessor processor = new UserAgentProcessor(randomAlphaOfLength(10), "source_field", "target_field", null, - EnumSet.allOf(UserAgentProcessor.Property.class), false); + EnumSet.allOf(UserAgentProcessor.Property.class), false, true); IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.emptyMap()); IngestDocument ingestDocument = new IngestDocument(originalIngestDocument); Exception exception = expectThrows(Exception.class, () -> processor.execute(ingestDocument)); @@ -186,6 +186,7 @@ public void testUnknown() throws Exception { assertNull(target.get("build")); assertNull(target.get("os")); + Map device = new HashMap<>(); device.put("name", "Other"); assertThat(target.get("device"), is(device)); diff --git a/modules/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/20_useragent_processor.yml b/modules/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/20_useragent_processor.yml index a7fe57c557008..f629761bd90a5 100644 --- a/modules/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/20_useragent_processor.yml +++ b/modules/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/20_useragent_processor.yml @@ -46,6 +46,7 @@ { "user_agent" : { "field" : "field1", + "ecs": true, "target_field": "field2", "properties": ["os"] } @@ -69,5 +70,62 @@ - match: { _source.field2.os.full: "Mac OS X 10.9.2" } - is_false: _source.user_agent - is_false: _source.field2.name + - is_false: _source.field2.os_name + - is_false: _source.field2.os_major + - is_false: _source.field2.os_minor + - is_false: _source.field2.major + - is_false: _source.field2.minor + - is_false: _source.field2.patch + - is_false: _source.field2.device + +--- +"Test user agent processor with non-ECS schema": + - skip: + features: warnings + + - do: + warnings: + - "setting [ecs] to false for non-common schema format is deprecated and will be removed in 8.0, set to true or remove to use the non-deprecated format" + - "the [os_major] property is deprecated for the user-agent processor" + ingest.put_pipeline: + id: "my_pipeline" + body: > + { + "description": "_description", + "processors": [ + { + "user_agent" : { + "field" : "field1", + "ecs": false, + "target_field": "field2", + "properties": ["os", "os_major"] + } + } + ] + } + - match: { acknowledged: true } + + - do: + index: + index: test + type: test + id: 1 + pipeline: "my_pipeline" + body: {field1: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.149 Safari/537.36"} + + - do: + get: + index: test + type: test + id: 1 + - match: { _source.field1: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.149 Safari/537.36" } + - match: { _source.field2.os: "Mac OS X 10.9.2" } + - match: { _source.field2.os_major: "10" } + - is_false: _source.user_agent + - is_false: _source.field2.name + - is_false: _source.field2.os_name + - is_false: _source.field2.os_minor + - is_false: _source.field2.major + - is_false: _source.field2.minor + - is_false: _source.field2.patch - is_false: _source.field2.device - - is_false: _source.field2.original diff --git a/modules/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/30_custom_regex.yml b/modules/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/30_custom_regex.yml index 763bea0ee4da0..a2cd419dd69b5 100644 --- a/modules/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/30_custom_regex.yml +++ b/modules/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/30_custom_regex.yml @@ -10,6 +10,7 @@ { "user_agent" : { "field": "field1", + "ecs": true, "regex_file": "test-regexes.yml" } } diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestService.java b/server/src/main/java/org/elasticsearch/ingest/IngestService.java index b2143d72ae65f..31023fc85d5b8 100644 --- a/server/src/main/java/org/elasticsearch/ingest/IngestService.java +++ b/server/src/main/java/org/elasticsearch/ingest/IngestService.java @@ -343,7 +343,7 @@ public String getType() { return new Pipeline(id, description, null, new CompoundProcessor(failureProcessor)); } - static ClusterState innerPut(PutPipelineRequest request, ClusterState currentState) { + public static ClusterState innerPut(PutPipelineRequest request, ClusterState currentState) { IngestMetadata currentIngestMetadata = currentState.metaData().custom(IngestMetadata.TYPE); Map pipelines; if (currentIngestMetadata != null) { diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/ClusterDeprecationChecks.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/ClusterDeprecationChecks.java new file mode 100644 index 0000000000000..38f78b7156e22 --- /dev/null +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/ClusterDeprecationChecks.java @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.deprecation; + +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.ingest.IngestService; +import org.elasticsearch.ingest.PipelineConfiguration; +import org.elasticsearch.xpack.core.deprecation.DeprecationIssue; + +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; + +public class ClusterDeprecationChecks { + + @SuppressWarnings("unchecked") + static DeprecationIssue checkUserAgentPipelines(ClusterState state) { + List pipelines = IngestService.getPipelines(state); + + List pipelinesWithDeprecatedEcsConfig = pipelines.stream() + .filter(Objects::nonNull) + .filter(pipeline -> { + Map pipelineConfig = pipeline.getConfigAsMap(); + + List>> processors = + (List>>) pipelineConfig.get("processors"); + return processors.stream() + .filter(Objects::nonNull) + .filter(processor -> processor.containsKey("user_agent")) + .map(processor -> processor.get("user_agent")) + .anyMatch(processorConfig -> processorConfig.containsKey("ecs")); + }) + .map(PipelineConfiguration::getId) + .sorted() // Make the warning consistent for testing purposes + .collect(Collectors.toList()); + if (pipelinesWithDeprecatedEcsConfig.isEmpty() == false) { + return new DeprecationIssue(DeprecationIssue.Level.WARNING, + "User-Agent ingest plugin will always use ECS-formatted output", + "https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-8.0.html" + + "#ingest-user-agent-ecs-always", + "Ingest pipelines " + pipelinesWithDeprecatedEcsConfig + " uses the [ecs] option which needs to be removed to work in 8.0"); + } + return null; + + } +} diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java index 97e273b213b79..5b980d851fcae 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java @@ -31,7 +31,10 @@ private DeprecationChecks() { } static List> CLUSTER_SETTINGS_CHECKS = - Collections.emptyList(); + Collections.unmodifiableList(Arrays.asList( + ClusterDeprecationChecks::checkUserAgentPipelines + )); + static List> NODE_SETTINGS_CHECKS = Collections.unmodifiableList(Arrays.asList( diff --git a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/ClusterDeprecationChecksTests.java b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/ClusterDeprecationChecksTests.java new file mode 100644 index 0000000000000..7f31067b6bc9e --- /dev/null +++ b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/ClusterDeprecationChecksTests.java @@ -0,0 +1,76 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.deprecation; + +import org.elasticsearch.action.ingest.PutPipelineRequest; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.ingest.IngestService; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.core.deprecation.DeprecationIssue; + +import java.util.List; + +import static java.util.Collections.singletonList; +import static org.elasticsearch.xpack.deprecation.DeprecationChecks.CLUSTER_SETTINGS_CHECKS; + +public class ClusterDeprecationChecksTests extends ESTestCase { + + public void testUserAgentEcsCheck() { + PutPipelineRequest ecsFalseRequest = new PutPipelineRequest("ecs_false", + new BytesArray("{\n" + + " \"description\" : \"This has ecs set to false\",\n" + + " \"processors\" : [\n" + + " {\n" + + " \"user_agent\" : {\n" + + " \"field\" : \"agent\",\n" + + " \"ecs\" : false\n" + + " }\n" + + " }\n" + + " ]\n" + + "}"), XContentType.JSON); + PutPipelineRequest ecsNullRequest = new PutPipelineRequest("ecs_null", + new BytesArray("{\n" + + " \"description\" : \"This has ecs set to false\",\n" + + " \"processors\" : [\n" + + " {\n" + + " \"user_agent\" : {\n" + + " \"field\" : \"agent\"\n" + + " }\n" + + " }\n" + + " ]\n" + + "}"), XContentType.JSON); + PutPipelineRequest ecsTrueRequest = new PutPipelineRequest("ecs_true", + new BytesArray("{\n" + + " \"description\" : \"This has ecs set to false\",\n" + + " \"processors\" : [\n" + + " {\n" + + " \"user_agent\" : {\n" + + " \"field\" : \"agent\",\n" + + " \"ecs\" : true\n" + + " }\n" + + " }\n" + + " ]\n" + + "}"), XContentType.JSON); + + ClusterState state = ClusterState.builder(new ClusterName("test")).build(); + state = IngestService.innerPut(ecsTrueRequest, state); + state = IngestService.innerPut(ecsFalseRequest, state); + state = IngestService.innerPut(ecsNullRequest, state); + + final ClusterState finalState = state; + List issues = DeprecationChecks.filterChecks(CLUSTER_SETTINGS_CHECKS, c -> c.apply(finalState)); + + DeprecationIssue expected = new DeprecationIssue(DeprecationIssue.Level.WARNING, + "User-Agent ingest plugin will always use ECS-formatted output", + "https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-8.0.html" + + "#ingest-user-agent-ecs-always", + "Ingest pipelines [ecs_false, ecs_true] uses the [ecs] option which needs to be removed to work in 8.0"); + assertEquals(singletonList(expected), issues); + } +}