Skip to content

Commit 46bb663

Browse files
jakelandisjakelandis
authored
Make 7.x like 6.7 user agent ecs, but default to true (#38828)
Forward port of #38757 This change reverts the initial 7.0 commits and replaces them with the 6.7 variant that still allows for the ecs flag. This commit differs from the 6.7 variants in that ecs flag will now default to true. 6.7: `ecs` : default `false` 7.x: `ecs` : default `true` 8.0: no option, but behaves as `true` * Revert "Ingest node - user agent, move device to an object (#38115)" This reverts commit 5b008a3. * Revert "Add ECS schema for user-agent ingest processor (#37727) (#37984)" This reverts commit cac6b8e. * cherry-pick 5dfe193 Add ECS schema for user-agent ingest processor (#37727) * cherry-pick ec8ddc8 Ingest node - user agent, move device to an object (#38115) (#38121) * cherry-pick f63cbdb (with manual merge fixes) Dep. check for ECS changes to User Agent processor (#38362) * make true the default for the ecs option, and update 7.0 references and tests
1 parent 7404882 commit 46bb663

File tree

11 files changed

+391
-76
lines changed

11 files changed

+391
-76
lines changed

‎docs/reference/ingest/processors/user-agent.asciidoc‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ The ingest-user-agent module ships by default with the regexes.yaml made availab
1919
| `regex_file` | no | - | The name of the file in the `config/ingest-user-agent` directory containing the regular expressions for parsing the user agent string. Both the directory and the file have to be created before starting Elasticsearch. If not specified, ingest-user-agent will use the regexes.yaml from uap-core it ships with (see below).
2020
| `properties` | no | [`name`, `major`, `minor`, `patch`, `build`, `os`, `os_name`, `os_major`, `os_minor`, `device`] | Controls what properties are added to `target_field`.
2121
| `ignore_missing` | no | `false` | If `true` and `field` does not exist, the processor quietly exits without modifying the document
22+
| `ecs` | no | `true` | Whether to return the output in Elastic Common Schema format. NOTE: This setting is deprecated and will be removed in a future version.
2223
|======
2324

2425
Here is an example that adds the user agent details to the `user_agent` field based on the `agent` field:

‎docs/reference/migration/migrate_7_0/settings.asciidoc‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -201,6 +201,6 @@ Elastic Stack to handle the indexing part.
201201

202202
[float]
203203
[[ingest-user-agent-ecs-always]]
204-
==== Ingest User Agent processor always uses `ecs` output format
205-
The deprecated `ecs` setting for the user agent ingest processor has been
206-
removed. https://github.com/elastic/ecs[ECS] format is now the default.
204+
==== Ingest User Agent processor defaults uses `ecs` output format
205+
https://github.com/elastic/ecs[ECS] format is now the default.
206+
The `ecs` setting for the user agent ingest processor now defaults to true.

‎modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java‎

Lines changed: 165 additions & 62 deletions
Original file line numberDiff line numberDiff line change
@@ -27,9 +27,11 @@
2727
import org.elasticsearch.ingest.useragent.UserAgentParser.Details;
2828
import org.elasticsearch.ingest.useragent.UserAgentParser.VersionedName;
2929

30+
import java.lang.reflect.Field;
3031
import java.util.Arrays;
3132
import java.util.EnumSet;
3233
import java.util.HashMap;
34+
import java.util.HashSet;
3335
import java.util.List;
3436
import java.util.Locale;
3537
import java.util.Map;
@@ -51,15 +53,17 @@ public class UserAgentProcessor extends AbstractProcessor {
5153
private final Set<Property> properties;
5254
private final UserAgentParser parser;
5355
private final boolean ignoreMissing;
56+
private final boolean useECS;
5457

5558
public UserAgentProcessor(String tag, String field, String targetField, UserAgentParser parser, Set<Property> properties,
56-
boolean ignoreMissing) {
59+
boolean ignoreMissing, boolean useECS) {
5760
super(tag);
5861
this.field = field;
5962
this.targetField = targetField;
6063
this.parser = parser;
6164
this.properties = properties;
6265
this.ignoreMissing = ignoreMissing;
66+
this.useECS = useECS;
6367
}
6468

6569
boolean isIgnoreMissing() {
@@ -80,68 +84,135 @@ public IngestDocument execute(IngestDocument ingestDocument) {
8084

8185
Map<String, Object> uaDetails = new HashMap<>();
8286

83-
// Parse the user agent in the ECS (Elastic Common Schema) format
84-
for (Property property : this.properties) {
85-
switch (property) {
86-
case ORIGINAL:
87-
uaDetails.put("original", userAgent);
88-
break;
89-
case NAME:
90-
if (uaClient.userAgent != null && uaClient.userAgent.name != null) {
91-
uaDetails.put("name", uaClient.userAgent.name);
92-
} else {
93-
uaDetails.put("name", "Other");
94-
}
95-
break;
96-
case VERSION:
97-
StringBuilder version = new StringBuilder();
98-
if (uaClient.userAgent != null && uaClient.userAgent.major != null) {
99-
version.append(uaClient.userAgent.major);
100-
if (uaClient.userAgent.minor != null) {
101-
version.append(".").append(uaClient.userAgent.minor);
102-
if (uaClient.userAgent.patch != null) {
103-
version.append(".").append(uaClient.userAgent.patch);
104-
if (uaClient.userAgent.build != null) {
105-
version.append(".").append(uaClient.userAgent.build);
87+
if (useECS) {
88+
// Parse the user agent in the ECS (Elastic Common Schema) format
89+
for (Property property : this.properties) {
90+
switch (property) {
91+
case ORIGINAL:
92+
uaDetails.put("original", userAgent);
93+
break;
94+
case NAME:
95+
if (uaClient.userAgent != null && uaClient.userAgent.name != null) {
96+
uaDetails.put("name", uaClient.userAgent.name);
97+
} else {
98+
uaDetails.put("name", "Other");
99+
}
100+
break;
101+
case VERSION:
102+
StringBuilder version = new StringBuilder();
103+
if (uaClient.userAgent != null && uaClient.userAgent.major != null) {
104+
version.append(uaClient.userAgent.major);
105+
if (uaClient.userAgent.minor != null) {
106+
version.append(".").append(uaClient.userAgent.minor);
107+
if (uaClient.userAgent.patch != null) {
108+
version.append(".").append(uaClient.userAgent.patch);
109+
if (uaClient.userAgent.build != null) {
110+
version.append(".").append(uaClient.userAgent.build);
111+
}
106112
}
107113
}
114+
uaDetails.put("version", version.toString());
108115
}
109-
uaDetails.put("version", version.toString());
110-
}
111-
break;
112-
case OS:
113-
if (uaClient.operatingSystem != null) {
114-
Map<String, String> osDetails = new HashMap<>(3);
115-
if (uaClient.operatingSystem.name != null) {
116-
osDetails.put("name", uaClient.operatingSystem.name);
117-
StringBuilder sb = new StringBuilder();
118-
if (uaClient.operatingSystem.major != null) {
119-
sb.append(uaClient.operatingSystem.major);
120-
if (uaClient.operatingSystem.minor != null) {
121-
sb.append(".").append(uaClient.operatingSystem.minor);
122-
if (uaClient.operatingSystem.patch != null) {
123-
sb.append(".").append(uaClient.operatingSystem.patch);
124-
if (uaClient.operatingSystem.build != null) {
125-
sb.append(".").append(uaClient.operatingSystem.build);
116+
break;
117+
case OS:
118+
if (uaClient.operatingSystem != null) {
119+
Map<String, String> osDetails = new HashMap<>(3);
120+
if (uaClient.operatingSystem.name != null) {
121+
osDetails.put("name", uaClient.operatingSystem.name);
122+
StringBuilder sb = new StringBuilder();
123+
if (uaClient.operatingSystem.major != null) {
124+
sb.append(uaClient.operatingSystem.major);
125+
if (uaClient.operatingSystem.minor != null) {
126+
sb.append(".").append(uaClient.operatingSystem.minor);
127+
if (uaClient.operatingSystem.patch != null) {
128+
sb.append(".").append(uaClient.operatingSystem.patch);
129+
if (uaClient.operatingSystem.build != null) {
130+
sb.append(".").append(uaClient.operatingSystem.build);
131+
}
126132
}
127133
}
134+
osDetails.put("version", sb.toString());
135+
osDetails.put("full", uaClient.operatingSystem.name + " " + sb.toString());
128136
}
129-
osDetails.put("version", sb.toString());
130-
osDetails.put("full", uaClient.operatingSystem.name + " " + sb.toString());
137+
uaDetails.put("os", osDetails);
131138
}
132-
uaDetails.put("os", osDetails);
133139
}
134-
}
135-
break;
136-
case DEVICE:
137-
Map<String, String> deviceDetails = new HashMap<>(1);
138-
if (uaClient.device != null && uaClient.device.name != null) {
139-
deviceDetails.put("name", uaClient.device.name);
140-
} else {
141-
deviceDetails.put("name", "Other");
142-
}
143-
uaDetails.put("device", deviceDetails);
144-
break;
140+
break;
141+
case DEVICE:
142+
Map<String, String> deviceDetails = new HashMap<>(1);
143+
if (uaClient.device != null && uaClient.device.name != null) {
144+
deviceDetails.put("name", uaClient.device.name);
145+
} else {
146+
deviceDetails.put("name", "Other");
147+
}
148+
uaDetails.put("device", deviceDetails);
149+
break;
150+
}
151+
}
152+
} else {
153+
// Deprecated format, removed in 8.0
154+
for (Property property : this.properties) {
155+
switch (property) {
156+
case NAME:
157+
if (uaClient.userAgent != null && uaClient.userAgent.name != null) {
158+
uaDetails.put("name", uaClient.userAgent.name);
159+
} else {
160+
uaDetails.put("name", "Other");
161+
}
162+
break;
163+
case MAJOR:
164+
if (uaClient.userAgent != null && uaClient.userAgent.major != null) {
165+
uaDetails.put("major", uaClient.userAgent.major);
166+
}
167+
break;
168+
case MINOR:
169+
if (uaClient.userAgent != null && uaClient.userAgent.minor != null) {
170+
uaDetails.put("minor", uaClient.userAgent.minor);
171+
}
172+
break;
173+
case PATCH:
174+
if (uaClient.userAgent != null && uaClient.userAgent.patch != null) {
175+
uaDetails.put("patch", uaClient.userAgent.patch);
176+
}
177+
break;
178+
case BUILD:
179+
if (uaClient.userAgent != null && uaClient.userAgent.build != null) {
180+
uaDetails.put("build", uaClient.userAgent.build);
181+
}
182+
break;
183+
case OS:
184+
if (uaClient.operatingSystem != null) {
185+
uaDetails.put("os", buildFullOSName(uaClient.operatingSystem));
186+
} else {
187+
uaDetails.put("os", "Other");
188+
}
189+
190+
break;
191+
case OS_NAME:
192+
if (uaClient.operatingSystem != null && uaClient.operatingSystem.name != null) {
193+
uaDetails.put("os_name", uaClient.operatingSystem.name);
194+
} else {
195+
uaDetails.put("os_name", "Other");
196+
}
197+
break;
198+
case OS_MAJOR:
199+
if (uaClient.operatingSystem != null && uaClient.operatingSystem.major != null) {
200+
uaDetails.put("os_major", uaClient.operatingSystem.major);
201+
}
202+
break;
203+
case OS_MINOR:
204+
if (uaClient.operatingSystem != null && uaClient.operatingSystem.minor != null) {
205+
uaDetails.put("os_minor", uaClient.operatingSystem.minor);
206+
}
207+
break;
208+
case DEVICE:
209+
if (uaClient.device != null && uaClient.device.name != null) {
210+
uaDetails.put("device", uaClient.device.name);
211+
} else {
212+
uaDetails.put("device", "Other");
213+
}
214+
break;
215+
}
145216
}
146217
}
147218

@@ -201,6 +272,10 @@ UserAgentParser getUaParser() {
201272
return parser;
202273
}
203274

275+
public boolean isUseECS() {
276+
return useECS;
277+
}
278+
204279
public static final class Factory implements Processor.Factory {
205280

206281
private final Map<String, UserAgentParser> userAgentParsers;
@@ -217,10 +292,7 @@ public UserAgentProcessor create(Map<String, Processor.Factory> factories, Strin
217292
String regexFilename = readStringProperty(TYPE, processorTag, config, "regex_file", IngestUserAgentPlugin.DEFAULT_PARSER_NAME);
218293
List<String> propertyNames = readOptionalList(TYPE, processorTag, config, "properties");
219294
boolean ignoreMissing = readBooleanProperty(TYPE, processorTag, config, "ignore_missing", false);
220-
Object ecsValue = config.remove("ecs");
221-
if (ecsValue != null) {
222-
deprecationLogger.deprecated("setting [ecs] is deprecated as ECS format is the default and only option");
223-
}
295+
boolean useECS = readBooleanProperty(TYPE, processorTag, config, "ecs", true);
224296

225297
UserAgentParser parser = userAgentParsers.get(regexFilename);
226298
if (parser == null) {
@@ -242,22 +314,53 @@ public UserAgentProcessor create(Map<String, Processor.Factory> factories, Strin
242314
properties = EnumSet.allOf(Property.class);
243315
}
244316

245-
return new UserAgentProcessor(processorTag, field, targetField, parser, properties, ignoreMissing);
317+
if (useECS == false) {
318+
deprecationLogger.deprecated("setting [ecs] to false for non-common schema " +
319+
"format is deprecated and will be removed in 8.0, set to true or remove to use the non-deprecated format");
320+
}
321+
322+
return new UserAgentProcessor(processorTag, field, targetField, parser, properties, ignoreMissing, useECS);
246323
}
247324
}
248325

249326
enum Property {
250327

251328
NAME,
329+
// Deprecated in 6.7 (superceded by VERSION), to be removed in 8.0
330+
@Deprecated MAJOR,
331+
@Deprecated MINOR,
332+
@Deprecated PATCH,
252333
OS,
334+
// Deprecated in 6.7 (superceded by just using OS), to be removed in 8.0
335+
@Deprecated OS_NAME,
336+
@Deprecated OS_MAJOR,
337+
@Deprecated OS_MINOR,
253338
DEVICE,
339+
@Deprecated BUILD, // Same deprecated as OS_* above
254340
ORIGINAL,
255341
VERSION;
256342

343+
private static Set<Property> DEPRECATED_PROPERTIES;
344+
345+
static {
346+
Set<Property> deprecated = new HashSet<>();
347+
for (Field field : Property.class.getFields()) {
348+
if (field.isEnumConstant() && field.isAnnotationPresent(Deprecated.class)) {
349+
deprecated.add(valueOf(field.getName()));
350+
}
351+
}
352+
DEPRECATED_PROPERTIES = deprecated;
353+
}
354+
257355
public static Property parseProperty(String propertyName) {
258356
try {
259-
return valueOf(propertyName.toUpperCase(Locale.ROOT));
260-
} catch (IllegalArgumentException e) {
357+
Property value = valueOf(propertyName.toUpperCase(Locale.ROOT));
358+
if (DEPRECATED_PROPERTIES.contains(value)) {
359+
deprecationLogger.deprecated("the [{}] property is deprecated for the user-agent processor", propertyName);
360+
}
361+
return value;
362+
}
363+
catch (IllegalArgumentException e) {
261364
throw new IllegalArgumentException("illegal property value [" + propertyName + "]. valid values are " +
262365
Arrays.toString(EnumSet.allOf(Property.class).toArray()));
263366
}

0 commit comments

Comments
 (0)