From 36053260abe8c21cca3c1a62dce961c63b9d9d72 Mon Sep 17 00:00:00 2001 From: David Turner Date: Wed, 10 Oct 2018 05:34:59 +0100 Subject: [PATCH] Handle pre-6.x time fields In ccb9ab5717d85c6786b081f197d67ac7dde4d317 we changed how we deal with time fields to support the `DateTime`-format fields added in 6.0, but dropped support for pre-6.x `Long`-format fields. This change reinstates this support for cases where pre-6.x data is made available to ML (e.g. in a mixed-version CCS setup or after an upgrade). --- .../extractor/scroll/ExtractedField.java | 2 +- .../extractor/scroll/ExtractedFieldTests.java | 28 +++++++++++++++++-- .../scroll/ExtractedFieldsTests.java | 20 +++++++++++-- 3 files changed, 43 insertions(+), 7 deletions(-) diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ExtractedField.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ExtractedField.java index ef0dffa269114..710aa017e741b 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ExtractedField.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ExtractedField.java @@ -107,7 +107,7 @@ public Object[] value(SearchHit hit) { value[0] = Long.parseLong((String) value[0]); } else if (value[0] instanceof BaseDateTime) { // script field value[0] = ((BaseDateTime) value[0]).getMillis(); - } else { + } else if (value[0] instanceof Long == false) { // pre-6.0 field throw new IllegalStateException("Unexpected value for a time field: " + value[0].getClass()); } return value; diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ExtractedFieldTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ExtractedFieldTests.java index 54afc2e707b64..d2e13368d0d29 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ExtractedFieldTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ExtractedFieldTests.java @@ -13,6 +13,7 @@ import java.util.Arrays; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.startsWith; public class ExtractedFieldTests extends ESTestCase { @@ -96,11 +97,32 @@ public void testNewTimeFieldGivenSource() { } public void testValueGivenTimeField() { - SearchHit hit = new SearchHitBuilder(42).addField("time", new DateTime(123456789L)).build(); + final long millis = randomLong(); + final SearchHit hit = new SearchHitBuilder(randomInt()).addField("time", new DateTime(millis)).build(); + final ExtractedField timeField = ExtractedField.newTimeField("time", ExtractedField.ExtractionMethod.DOC_VALUE); + assertThat(timeField.value(hit), equalTo(new Object[] { millis })); + } + + public void testValueGivenStringTimeField() { + final long millis = randomLong(); + final SearchHit hit = new SearchHitBuilder(randomInt()).addField("time", Long.toString(millis)).build(); + final ExtractedField timeField = ExtractedField.newTimeField("time", ExtractedField.ExtractionMethod.DOC_VALUE); + assertThat(timeField.value(hit), equalTo(new Object[] { millis })); + } - ExtractedField timeField = ExtractedField.newTimeField("time", ExtractedField.ExtractionMethod.DOC_VALUE); + public void testValueGivenPre6xTimeField() { + // Prior to 6.x, timestamps were simply `long` milliseconds-past-the-epoch values + final long millis = randomLong(); + final SearchHit hit = new SearchHitBuilder(randomInt()).addField("time", millis).build(); + final ExtractedField timeField = ExtractedField.newTimeField("time", ExtractedField.ExtractionMethod.DOC_VALUE); + assertThat(timeField.value(hit), equalTo(new Object[] { millis })); + } - assertThat(timeField.value(hit), equalTo(new Object[] { 123456789L })); + public void testValueGivenUnknownFormatTimeField() { + final SearchHit hit = new SearchHitBuilder(randomInt()).addField("time", new Object()).build(); + final ExtractedField timeField = ExtractedField.newTimeField("time", ExtractedField.ExtractionMethod.DOC_VALUE); + assertThat(expectThrows(IllegalStateException.class, () -> timeField.value(hit)).getMessage(), + startsWith("Unexpected value for a time field")); } public void testAliasVersusName() { diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ExtractedFieldsTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ExtractedFieldsTests.java index a921d1f3e738a..7e98dd417cd81 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ExtractedFieldsTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ExtractedFieldsTests.java @@ -64,11 +64,25 @@ public void testAllTypesOfFields() { } public void testTimeFieldValue() { - SearchHit hit = new SearchHitBuilder(1).addField("time", new DateTime(1000L)).build(); + final long millis = randomLong(); + final SearchHit hit = new SearchHitBuilder(randomInt()).addField("time", new DateTime(millis)).build(); + final ExtractedFields extractedFields = new ExtractedFields(timeField, Collections.singletonList(timeField)); + assertThat(extractedFields.timeFieldValue(hit), equalTo(millis)); + } - ExtractedFields extractedFields = new ExtractedFields(timeField, Arrays.asList(timeField)); + public void testStringTimeFieldValue() { + final long millis = randomLong(); + final SearchHit hit = new SearchHitBuilder(randomInt()).addField("time", Long.toString(millis)).build(); + final ExtractedFields extractedFields = new ExtractedFields(timeField, Collections.singletonList(timeField)); + assertThat(extractedFields.timeFieldValue(hit), equalTo(millis)); + } - assertThat(extractedFields.timeFieldValue(hit), equalTo(1000L)); + public void testPre6xTimeFieldValue() { + // Prior to 6.x, timestamps were simply `long` milliseconds-past-the-epoch values + final long millis = randomLong(); + final SearchHit hit = new SearchHitBuilder(randomInt()).addField("time", millis).build(); + final ExtractedFields extractedFields = new ExtractedFields(timeField, Collections.singletonList(timeField)); + assertThat(extractedFields.timeFieldValue(hit), equalTo(millis)); } public void testTimeFieldValueGivenEmptyArray() {