Skip to content

Commit d945fce

Browse files
stu-elasticstu-elastic
authored
Ingest: Enforce _version metadata not null in sourceAndMetadata map (#88102)
* Ingest: Enforce _version metadata not null in sourceAndMetadata map The `_version` metadata field should always exist in the sourceAndMetadata map, this change enforces that invariant but allows tests to avoid it if necessary. Refs: #87309
1 parent 8684c08 commit d945fce

File tree

19 files changed

+142
-65
lines changed

19 files changed

+142
-65
lines changed

‎modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/CommunityIdProcessorTests.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -351,7 +351,7 @@ private void testCommunityIdProcessor(Map<String, Object> source, int seed, Stri
351351
ignoreMissing
352352
);
353353

354-
IngestDocument input = TestIngestDocument.ofSourceAndMetadata(source);
354+
IngestDocument input = TestIngestDocument.withDefaultVersion(source);
355355
IngestDocument output = processor.execute(input);
356356

357357
String hash = output.getFieldValue(DEFAULT_TARGET, String.class, ignoreMissing);

‎modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DotExpanderProcessorTests.java‎

Lines changed: 21 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -26,15 +26,15 @@ public class DotExpanderProcessorTests extends ESTestCase {
2626
public void testEscapeFields() throws Exception {
2727
Map<String, Object> source = new HashMap<>();
2828
source.put("foo.bar", "baz1");
29-
IngestDocument document = TestIngestDocument.ofSourceAndMetadata(source);
29+
IngestDocument document = TestIngestDocument.withDefaultVersion(source);
3030
DotExpanderProcessor processor = new DotExpanderProcessor("_tag", null, null, "foo.bar");
3131
processor.execute(document);
3232
assertThat(document.getFieldValue("foo", Map.class).size(), equalTo(1));
3333
assertThat(document.getFieldValue("foo.bar", String.class), equalTo("baz1"));
3434

3535
source = new HashMap<>();
3636
source.put("foo.bar.baz", "value");
37-
document = TestIngestDocument.ofSourceAndMetadata(source);
37+
document = TestIngestDocument.withDefaultVersion(source);
3838
processor = new DotExpanderProcessor("_tag", null, null, "foo.bar.baz");
3939
processor.execute(document);
4040
assertThat(document.getFieldValue("foo", Map.class).size(), equalTo(1));
@@ -44,21 +44,23 @@ public void testEscapeFields() throws Exception {
4444
source = new HashMap<>();
4545
source.put("foo.bar", "baz1");
4646
source.put("foo", new HashMap<>(Collections.singletonMap("bar", "baz2")));
47-
document = TestIngestDocument.ofSourceAndMetadata(source);
47+
document = TestIngestDocument.withDefaultVersion(source);
4848
processor = new DotExpanderProcessor("_tag", null, null, "foo.bar");
4949
processor.execute(document);
50-
assertThat(document.getSourceAndMetadata().size(), equalTo(1));
50+
assertThat(document.getSource().size(), equalTo(1));
51+
assertThat(document.getMetadata().size(), equalTo(1)); // the default version
5152
assertThat(document.getFieldValue("foo.bar", List.class).size(), equalTo(2));
5253
assertThat(document.getFieldValue("foo.bar.0", String.class), equalTo("baz2"));
5354
assertThat(document.getFieldValue("foo.bar.1", String.class), equalTo("baz1"));
5455

5556
source = new HashMap<>();
5657
source.put("foo.bar", "2");
5758
source.put("foo", new HashMap<>(Collections.singletonMap("bar", 1)));
58-
document = TestIngestDocument.ofSourceAndMetadata(source);
59+
document = TestIngestDocument.withDefaultVersion(source);
5960
processor = new DotExpanderProcessor("_tag", null, null, "foo.bar");
6061
processor.execute(document);
61-
assertThat(document.getSourceAndMetadata().size(), equalTo(1));
62+
assertThat(document.getSource().size(), equalTo(1));
63+
assertThat(document.getMetadata().size(), equalTo(1)); // the default version
6264
assertThat(document.getFieldValue("foo.bar", List.class).size(), equalTo(2));
6365
assertThat(document.getFieldValue("foo.bar.0", Integer.class), equalTo(1));
6466
assertThat(document.getFieldValue("foo.bar.1", String.class), equalTo("2"));
@@ -68,15 +70,15 @@ public void testEscapeFields_valueField() throws Exception {
6870
Map<String, Object> source = new HashMap<>();
6971
source.put("foo.bar", "baz1");
7072
source.put("foo", "baz2");
71-
IngestDocument document1 = TestIngestDocument.ofSourceAndMetadata(source);
73+
IngestDocument document1 = TestIngestDocument.withDefaultVersion(source);
7274
Processor processor1 = new DotExpanderProcessor("_tag", null, null, "foo.bar");
7375
// foo already exists and if a leaf field and therefor can't be replaced by a map field:
7476
Exception e = expectThrows(IllegalArgumentException.class, () -> processor1.execute(document1));
7577
assertThat(e.getMessage(), equalTo("cannot expand [foo.bar], because [foo] is not an object field, but a value field"));
7678

7779
// so because foo is no branch field but a value field the `foo.bar` field can't be expanded
7880
// into [foo].[bar], so foo should be renamed first into `[foo].[bar]:
79-
IngestDocument document = TestIngestDocument.ofSourceAndMetadata(source);
81+
IngestDocument document = TestIngestDocument.withDefaultVersion(source);
8082
Processor processor = new RenameProcessor(
8183
"_tag",
8284
null,
@@ -93,7 +95,7 @@ public void testEscapeFields_valueField() throws Exception {
9395

9496
source = new HashMap<>();
9597
source.put("foo.bar", "baz1");
96-
document = TestIngestDocument.ofSourceAndMetadata(source);
98+
document = TestIngestDocument.withDefaultVersion(source);
9799
processor = new DotExpanderProcessor("_tag", null, null, "foo.bar");
98100
processor.execute(document);
99101
assertThat(document.getFieldValue("foo", Map.class).size(), equalTo(1));
@@ -102,7 +104,7 @@ public void testEscapeFields_valueField() throws Exception {
102104
source = new HashMap<>();
103105
source.put("foo.bar.baz", "baz1");
104106
source.put("foo", new HashMap<>(Collections.singletonMap("bar", new HashMap<>())));
105-
document = TestIngestDocument.ofSourceAndMetadata(source);
107+
document = TestIngestDocument.withDefaultVersion(source);
106108
processor = new DotExpanderProcessor("_tag", null, null, "foo.bar.baz");
107109
processor.execute(document);
108110
assertThat(document.getFieldValue("foo", Map.class).size(), equalTo(1));
@@ -112,7 +114,7 @@ public void testEscapeFields_valueField() throws Exception {
112114
source = new HashMap<>();
113115
source.put("foo.bar.baz", "baz1");
114116
source.put("foo", new HashMap<>(Collections.singletonMap("bar", "baz2")));
115-
IngestDocument document2 = TestIngestDocument.ofSourceAndMetadata(source);
117+
IngestDocument document2 = TestIngestDocument.withDefaultVersion(source);
116118
Processor processor2 = new DotExpanderProcessor("_tag", null, null, "foo.bar.baz");
117119
e = expectThrows(IllegalArgumentException.class, () -> processor2.execute(document2));
118120
assertThat(e.getMessage(), equalTo("cannot expand [foo.bar.baz], because [foo.bar] is not an object field, but a value field"));
@@ -121,7 +123,7 @@ public void testEscapeFields_valueField() throws Exception {
121123
public void testEscapeFields_path() throws Exception {
122124
Map<String, Object> source = new HashMap<>();
123125
source.put("foo", new HashMap<>(Collections.singletonMap("bar.baz", "value")));
124-
IngestDocument document = TestIngestDocument.ofSourceAndMetadata(source);
126+
IngestDocument document = TestIngestDocument.withDefaultVersion(source);
125127
DotExpanderProcessor processor = new DotExpanderProcessor("_tag", null, "foo", "bar.baz");
126128
processor.execute(document);
127129
assertThat(document.getFieldValue("foo", Map.class).size(), equalTo(1));
@@ -130,7 +132,7 @@ public void testEscapeFields_path() throws Exception {
130132

131133
source = new HashMap<>();
132134
source.put("field", new HashMap<>(Collections.singletonMap("foo.bar.baz", "value")));
133-
document = TestIngestDocument.ofSourceAndMetadata(source);
135+
document = TestIngestDocument.withDefaultVersion(source);
134136
processor = new DotExpanderProcessor("_tag", null, "field", "foo.bar.baz");
135137
processor.execute(document);
136138
assertThat(document.getFieldValue("field.foo", Map.class).size(), equalTo(1));
@@ -142,7 +144,7 @@ public void testEscapeFields_doNothingIfFieldNotInSourceDoc() throws Exception {
142144
// asking to expand a (literal) field that is not present in the source document
143145
Map<String, Object> source = new HashMap<>();
144146
source.put("foo.bar", "baz1");
145-
IngestDocument document = TestIngestDocument.ofSourceAndMetadata(source);
147+
IngestDocument document = TestIngestDocument.withDefaultVersion(source);
146148
// abc.def does not exist in source, so don't mutate document
147149
DotExpanderProcessor processor = new DotExpanderProcessor("_tag", null, null, "abc.def");
148150
processor.execute(document);
@@ -160,7 +162,7 @@ public void testEscapeFields_doNothingIfFieldNotInSourceDoc() throws Exception {
160162
Map<String, Object> inner = new HashMap<>();
161163
inner.put("bar", "baz1");
162164
source.put("foo", inner);
163-
document = TestIngestDocument.ofSourceAndMetadata(source);
165+
document = TestIngestDocument.withDefaultVersion(source);
164166
// foo.bar, the literal value (as opposed to nested value) does not exist in source, so don't mutate document
165167
processor = new DotExpanderProcessor("_tag", null, null, "foo.bar");
166168
processor.execute(document);
@@ -178,7 +180,7 @@ public void testOverride() throws Exception {
178180
inner.put("qux", "quux");
179181
source.put("foo", inner);
180182
source.put("foo.bar", "baz2");
181-
IngestDocument document = TestIngestDocument.ofSourceAndMetadata(source);
183+
IngestDocument document = TestIngestDocument.withDefaultVersion(source);
182184
DotExpanderProcessor processor = new DotExpanderProcessor("_tag", null, null, "foo.bar", true);
183185
processor.execute(document);
184186
assertThat(document.getFieldValue("foo", Map.class).size(), equalTo(2));
@@ -190,7 +192,7 @@ public void testWildcard() throws Exception {
190192
Map<String, Object> source = new HashMap<>();
191193
source.put("foo.bar", "baz");
192194
source.put("qux.quux", "corge");
193-
IngestDocument document = TestIngestDocument.ofSourceAndMetadata(source);
195+
IngestDocument document = TestIngestDocument.withDefaultVersion(source);
194196
DotExpanderProcessor processor = new DotExpanderProcessor("_tag", null, null, "*");
195197
processor.execute(document);
196198
assertThat(document.getFieldValue("foo", Map.class).size(), equalTo(1));
@@ -202,7 +204,7 @@ public void testWildcard() throws Exception {
202204
Map<String, Object> inner = new HashMap<>();
203205
inner.put("bar.baz", "qux");
204206
source.put("foo", inner);
205-
document = TestIngestDocument.ofSourceAndMetadata(source);
207+
document = TestIngestDocument.withDefaultVersion(source);
206208
processor = new DotExpanderProcessor("_tag", null, "foo", "*");
207209
processor.execute(document);
208210
assertThat(document.getFieldValue("foo", Map.class).size(), equalTo(1));
@@ -213,7 +215,7 @@ public void testWildcard() throws Exception {
213215
inner = new HashMap<>();
214216
inner.put("bar.baz", "qux");
215217
source.put("foo", inner);
216-
document = TestIngestDocument.ofSourceAndMetadata(source);
218+
document = TestIngestDocument.withDefaultVersion(source);
217219
processor = new DotExpanderProcessor("_tag", null, null, "*");
218220
processor.execute(document);
219221
assertThat(document.getFieldValue("foo", Map.class).size(), equalTo(1));

‎modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/FingerprintProcessorTests.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -227,7 +227,7 @@ private String doTestFingerprint(
227227
MessageDigest md = MessageDigest.getInstance(FingerprintProcessor.Factory.DEFAULT_METHOD);
228228
expectedBytes = md.digest(expectedBytes);
229229

230-
var input = TestIngestDocument.ofSourceAndMetadata(inputMap);
230+
var input = TestIngestDocument.withDefaultVersion(inputMap);
231231
var output = fp.execute(input);
232232
assertTrue(output.hasField("fingerprint"));
233233
String fingerprint = output.getFieldValue("fingerprint", String.class);
@@ -257,7 +257,7 @@ public void testMethod() throws Exception {
257257
config.put("method", FingerprintProcessor.Factory.SUPPORTED_DIGESTS[k]);
258258

259259
FingerprintProcessor fp = factory.create(null, randomAlphaOfLength(10), null, config);
260-
var input = TestIngestDocument.ofSourceAndMetadata(inputMap);
260+
var input = TestIngestDocument.withDefaultVersion(inputMap);
261261
var output = fp.execute(input);
262262
assertTrue(output.hasField("fingerprint"));
263263
String fingerprint = output.getFieldValue("fingerprint", String.class);
@@ -394,7 +394,7 @@ private void doTestObjectTraversal(Map<String, Object> inputMap, List<String> fi
394394
expectedBytes = concatBytes(expectedBytes, toBytes(value));
395395
}
396396

397-
var input = TestIngestDocument.ofSourceAndMetadata(inputMap);
397+
var input = TestIngestDocument.withDefaultVersion(inputMap);
398398
var output = fp.execute(input);
399399
var hasher = (TestHasher) threadLocalHasher.get();
400400
assertThat(hasher.getBytesSeen(), equalTo(expectedBytes));

‎modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/NetworkDirectionProcessorTests.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -150,7 +150,7 @@ public void testReadFromField() throws Exception {
150150
null,
151151
config
152152
);
153-
IngestDocument input = TestIngestDocument.ofSourceAndMetadata(source);
153+
IngestDocument input = TestIngestDocument.withDefaultVersion(source);
154154
IngestDocument output = processor.execute(input);
155155
String hash = output.getFieldValue(DEFAULT_TARGET, String.class);
156156
assertThat(hash, equalTo("external"));
@@ -196,7 +196,7 @@ private void testNetworkDirectionProcessor(
196196
config
197197
);
198198

199-
IngestDocument input = TestIngestDocument.ofSourceAndMetadata(source);
199+
IngestDocument input = TestIngestDocument.withDefaultVersion(source);
200200
IngestDocument output = processor.execute(input);
201201

202202
String hash = output.getFieldValue(DEFAULT_TARGET, String.class, ignoreMissing);

‎modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RegisteredDomainProcessorTests.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,7 @@ public void testUseRoot() throws Exception {
8080

8181
var processor = new RegisteredDomainProcessor(null, null, "domain", "", false);
8282

83-
IngestDocument input = TestIngestDocument.ofSourceAndMetadata(source);
83+
IngestDocument input = TestIngestDocument.withDefaultVersion(source);
8484
IngestDocument output = processor.execute(input);
8585

8686
String domain = output.getFieldValue(domainField, String.class);
@@ -131,7 +131,7 @@ private void testRegisteredDomainProcessor(
131131

132132
var processor = new RegisteredDomainProcessor(null, null, "domain", "url", ignoreMissing);
133133

134-
IngestDocument input = TestIngestDocument.ofSourceAndMetadata(source);
134+
IngestDocument input = TestIngestDocument.withDefaultVersion(source);
135135
IngestDocument output = processor.execute(input);
136136

137137
String domain = output.getFieldValue(domainField, String.class, expectedDomain == null);

‎modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RenameProcessorTests.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -179,7 +179,7 @@ public void testRenameAtomicOperationRemoveFails() throws Exception {
179179
public void testRenameLeafIntoBranch() throws Exception {
180180
Map<String, Object> source = new HashMap<>();
181181
source.put("foo", "bar");
182-
IngestDocument ingestDocument = TestIngestDocument.ofSourceAndMetadata(source);
182+
IngestDocument ingestDocument = TestIngestDocument.withDefaultVersion(source);
183183
Processor processor1 = createRenameProcessor("foo", "foo.bar", false);
184184
processor1.execute(ingestDocument);
185185
assertThat(ingestDocument.getFieldValue("foo", Map.class), equalTo(Collections.singletonMap("bar", "bar")));

‎modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/SplitProcessorTests.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -100,7 +100,7 @@ public void testSplitAppendable() throws Exception {
100100
Processor splitProcessor = (new SplitProcessor.Factory()).create(null, null, null, splitConfig);
101101
Map<String, Object> source = new HashMap<>();
102102
source.put("flags", "new|hot|super|fun|interesting");
103-
IngestDocument ingestDocument = TestIngestDocument.ofSourceAndMetadata(source);
103+
IngestDocument ingestDocument = TestIngestDocument.withDefaultVersion(source);
104104
splitProcessor.execute(ingestDocument);
105105
@SuppressWarnings("unchecked")
106106
List<String> flags = (List<String>) ingestDocument.getFieldValue("flags", List.class);

‎modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/UriPartsProcessorTests.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -186,7 +186,7 @@ public void testRemoveIfSuccessfulDoesNotRemoveTargetField() throws Exception {
186186

187187
Map<String, Object> source = new HashMap<>();
188188
source.put(field, "http://www.google.com");
189-
IngestDocument input = TestIngestDocument.ofSourceAndMetadata(source);
189+
IngestDocument input = TestIngestDocument.withDefaultVersion(source);
190190
IngestDocument output = processor.execute(input);
191191

192192
Map<String, Object> expectedSourceAndMetadata = new HashMap<>();
@@ -202,7 +202,7 @@ public void testInvalidUri() {
202202

203203
Map<String, Object> source = new HashMap<>();
204204
source.put("field", uri);
205-
IngestDocument input = TestIngestDocument.ofSourceAndMetadata(source);
205+
IngestDocument input = TestIngestDocument.withDefaultVersion(source);
206206

207207
IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> processor.execute(input));
208208
assertThat(e.getMessage(), containsString("unable to parse URI [" + uri + "]"));
@@ -218,7 +218,7 @@ private void testUriParsing(boolean keepOriginal, boolean removeIfSuccessful, St
218218

219219
Map<String, Object> source = new HashMap<>();
220220
source.put("field", uri);
221-
IngestDocument input = TestIngestDocument.ofSourceAndMetadata(source);
221+
IngestDocument input = TestIngestDocument.withDefaultVersion(source);
222222
IngestDocument output = processor.execute(input);
223223

224224
Map<String, Object> expectedSourceAndMetadata = new HashMap<>();

‎server/src/main/java/org/elasticsearch/ingest/IngestDocument.java‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,13 @@ public IngestDocument(Map<String, Object> sourceAndMetadata, Map<String, Object>
9797
IngestSourceAndMetadata.getTimestamp(ingestMetadata),
9898
IngestSourceAndMetadata.VALIDATORS
9999
);
100-
this.ingestMetadata = ingestMetadata;
100+
this.ingestMetadata = new HashMap<>(ingestMetadata);
101+
this.ingestMetadata.computeIfPresent(TIMESTAMP, (k, v) -> {
102+
if (v instanceof String) {
103+
return this.sourceAndMetadata.getTimestamp();
104+
}
105+
return v;
106+
});
101107
}
102108

103109
/**

‎server/src/main/java/org/elasticsearch/ingest/IngestSourceAndMetadata.java‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ class IngestSourceAndMetadata extends AbstractMap<String, Object> {
5454
IngestDocument.Metadata.ROUTING.getFieldName(),
5555
IngestSourceAndMetadata::stringValidator,
5656
IngestDocument.Metadata.VERSION.getFieldName(),
57-
IngestSourceAndMetadata::longValidator,
57+
IngestSourceAndMetadata::versionValidator,
5858
IngestDocument.Metadata.VERSION_TYPE.getFieldName(),
5959
IngestSourceAndMetadata::versionTypeValidator,
6060
IngestDocument.Metadata.DYNAMIC_TEMPLATES.getFieldName(),
@@ -151,8 +151,11 @@ public static ZonedDateTime getTimestamp(Map<String, Object> ingestMetadata) {
151151
if (ingestMetadata == null) {
152152
return null;
153153
}
154-
if (ingestMetadata.get(IngestDocument.TIMESTAMP)instanceof ZonedDateTime timestamp) {
154+
Object ts = ingestMetadata.get(IngestDocument.TIMESTAMP);
155+
if (ts instanceof ZonedDateTime timestamp) {
155156
return timestamp;
157+
} else if (ts instanceof String str) {
158+
return ZonedDateTime.parse(str);
156159
}
157160
return null;
158161
}
@@ -532,6 +535,16 @@ protected static void longValidator(String key, Object value) {
532535
);
533536
}
534537

538+
/**
539+
* Version must be non-null and representable as a long without loss of precision
540+
*/
541+
protected static void versionValidator(String key, Object value) {
542+
if (value == null) {
543+
throw new IllegalArgumentException(key + " cannot be null");
544+
}
545+
longValidator(key, value);
546+
}
547+
535548
/**
536549
* Allow lower case Strings that map to VersionType values, or null
537550
*/

0 commit comments

Comments
 (0)