From 904854fabab38e8f727ab9892ead441b24ef16ad Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Mon, 30 Nov 2020 11:21:54 +0100 Subject: [PATCH 01/15] write dates in ISO format, provide backwards compatibility for old jobs and old behavior fixes #63787 --- .../transform/transforms/SettingsConfig.java | 51 ++++++++++-- .../transforms/SettingsConfigTests.java | 28 ++++++- .../transforms/hlrc/SettingsConfigTests.java | 4 +- .../common/xcontent/ObjectParser.java | 1 + .../xpack/core/transform/TransformField.java | 1 + .../transform/transforms/SettingsConfig.java | 81 ++++++++++++++++--- .../transform/transforms/TransformConfig.java | 32 +++++++- .../transforms/SettingsConfigTests.java | 32 +++++++- .../transforms/TransformConfigTests.java | 38 +++++++++ .../TransformConfigUpdateTests.java | 17 ++-- .../integration/TransformProgressIT.java | 6 +- .../continuous/DateHistogramGroupByIT.java | 24 ++++-- .../DateHistogramGroupByOtherTimeFieldIT.java | 15 ++-- .../continuous/TermsOnDateGroupByIT.java | 13 +-- .../transform/transforms/FunctionFactory.java | 2 +- .../pivot/AggregationResultUtils.java | 31 ++++++- .../transform/transforms/pivot/Pivot.java | 19 ++++- .../TransformIndexerStateTests.java | 2 +- .../transforms/TransformIndexerTests.java | 6 +- .../pivot/AggregationResultUtilsTests.java | 3 +- .../transforms/pivot/PivotTests.java | 30 +++++-- 21 files changed, 366 insertions(+), 70 deletions(-) diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/SettingsConfig.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/SettingsConfig.java index b7c16707796f8..4b0c210b2e6c9 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/SettingsConfig.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/SettingsConfig.java @@ -21,6 +21,7 @@ import org.elasticsearch.common.ParseField; import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ObjectParser.ValueType; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; @@ -34,30 +35,42 @@ public class SettingsConfig implements ToXContentObject { private static final ParseField MAX_PAGE_SEARCH_SIZE = new ParseField("max_page_search_size"); private static final ParseField DOCS_PER_SECOND = new ParseField("docs_per_second"); + private static final ParseField WRITE_DATE_AS_EPOCH_MILLIS = new ParseField("write_date_as_epoch_millis"); private static final int DEFAULT_MAX_PAGE_SEARCH_SIZE = -1; private static final float DEFAULT_DOCS_PER_SECOND = -1F; + // use an integer as we need to code 4 states: true, false, null (unchanged), default (defined server side) + private static final int DEFAULT_WRITE_DATE_AS_EPOCH_MILLIS = -1; + private final Integer maxPageSearchSize; private final Float docsPerSecond; + private final Integer writeDateAsEpochMillis; private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( "settings_config", true, - args -> new SettingsConfig((Integer) args[0], (Float) args[1]) + args -> new SettingsConfig((Integer) args[0], (Float) args[1], (Integer) args[2]) ); static { PARSER.declareIntOrNull(optionalConstructorArg(), DEFAULT_MAX_PAGE_SEARCH_SIZE, MAX_PAGE_SEARCH_SIZE); PARSER.declareFloatOrNull(optionalConstructorArg(), DEFAULT_DOCS_PER_SECOND, DOCS_PER_SECOND); + PARSER.declareField( + optionalConstructorArg(), + p -> p.currentToken() == XContentParser.Token.VALUE_NULL ? DEFAULT_WRITE_DATE_AS_EPOCH_MILLIS : p.booleanValue() ? 1 : 0, + WRITE_DATE_AS_EPOCH_MILLIS, + ValueType.BOOLEAN_OR_NULL + ); } public static SettingsConfig fromXContent(final XContentParser parser) { return PARSER.apply(parser, null); } - SettingsConfig(Integer maxPageSearchSize, Float docsPerSecond) { + SettingsConfig(Integer maxPageSearchSize, Float docsPerSecond, Integer writeDateAsEpochMillis) { this.maxPageSearchSize = maxPageSearchSize; this.docsPerSecond = docsPerSecond; + this.writeDateAsEpochMillis = writeDateAsEpochMillis; } @Override @@ -77,6 +90,13 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(DOCS_PER_SECOND.getPreferredName(), docsPerSecond); } } + if (writeDateAsEpochMillis != null) { + if (writeDateAsEpochMillis.equals(DEFAULT_WRITE_DATE_AS_EPOCH_MILLIS)) { + builder.field(WRITE_DATE_AS_EPOCH_MILLIS.getPreferredName(), (Boolean) null); + } else { + builder.field(WRITE_DATE_AS_EPOCH_MILLIS.getPreferredName(), writeDateAsEpochMillis > 0 ? true : false); + } + } builder.endObject(); return builder; } @@ -89,6 +109,10 @@ public Float getDocsPerSecond() { return docsPerSecond; } + public Boolean getWriteDateAsEpochMillis() { + return writeDateAsEpochMillis != null ? writeDateAsEpochMillis > 0 : null; + } + @Override public boolean equals(Object other) { if (other == this) { @@ -99,12 +123,14 @@ public boolean equals(Object other) { } SettingsConfig that = (SettingsConfig) other; - return Objects.equals(maxPageSearchSize, that.maxPageSearchSize) && Objects.equals(docsPerSecond, that.docsPerSecond); + return Objects.equals(maxPageSearchSize, that.maxPageSearchSize) + && Objects.equals(docsPerSecond, that.docsPerSecond) + && Objects.equals(writeDateAsEpochMillis, that.writeDateAsEpochMillis); } @Override public int hashCode() { - return Objects.hash(maxPageSearchSize, docsPerSecond); + return Objects.hash(maxPageSearchSize, docsPerSecond, writeDateAsEpochMillis); } public static Builder builder() { @@ -114,6 +140,7 @@ public static Builder builder() { public static class Builder { private Integer maxPageSearchSize; private Float docsPerSecond; + private Integer writeDateAsEpochMilli; /** * Sets the paging maximum paging maxPageSearchSize that transform can use when @@ -143,8 +170,22 @@ public Builder setRequestsPerSecond(Float docsPerSecond) { return this; } + /** + * Sets whether to write the output of a date aggregation as millis since epoch or as formatted string (ISO format). + * + * This setting ensures backwards compatibility for transforms created before 7.11, which wrote dates as epoch_millis. + * The new default is ISO string. + * + * @param writeDateAsEpochMilli true if dates should be written as epoch_millis. + * @return the {@link Builder} with writeDateAsEpochMilli set. + */ + public Builder setWriteDateAsEpochMilli(Boolean writeDateAsEpochMilli) { + this.writeDateAsEpochMilli = writeDateAsEpochMilli == null ? DEFAULT_WRITE_DATE_AS_EPOCH_MILLIS : writeDateAsEpochMilli ? 1 : 0; + return this; + } + public SettingsConfig build() { - return new SettingsConfig(maxPageSearchSize, docsPerSecond); + return new SettingsConfig(maxPageSearchSize, docsPerSecond, writeDateAsEpochMilli); } } } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/SettingsConfigTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/SettingsConfigTests.java index 3126e70251af6..f1e35c03d2a2e 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/SettingsConfigTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/SettingsConfigTests.java @@ -38,7 +38,11 @@ public class SettingsConfigTests extends AbstractXContentTestCase { public static SettingsConfig randomSettingsConfig() { - return new SettingsConfig(randomBoolean() ? null : randomIntBetween(10, 10_000), randomBoolean() ? null : randomFloat()); + return new SettingsConfig( + randomBoolean() ? null : randomIntBetween(10, 10_000), + randomBoolean() ? null : randomFloat(), + randomBoolean() ? null : randomIntBetween(-1, 1) + ); } @Override @@ -67,6 +71,7 @@ public void testExplicitNullOnWriteParser() throws IOException { SettingsConfig emptyConfig = fromString("{}"); assertNull(emptyConfig.getMaxPageSearchSize()); + assertNull(emptyConfig.getWriteDateAsEpochMillis()); settingsAsMap = xContentToMap(emptyConfig); assertTrue(settingsAsMap.isEmpty()); @@ -77,6 +82,15 @@ public void testExplicitNullOnWriteParser() throws IOException { settingsAsMap = xContentToMap(config); assertThat(settingsAsMap.getOrDefault("max_page_search_size", "not_set"), equalTo("not_set")); assertNull(settingsAsMap.getOrDefault("docs_per_second", "not_set")); + assertThat(settingsAsMap.getOrDefault("write_date_as_epoch_millis", "not_set"), equalTo("not_set")); + + config = fromString("{\"write_date_as_epoch_millis\" : null}"); + assertNull(emptyConfig.getWriteDateAsEpochMillis()); + + settingsAsMap = xContentToMap(config); + assertThat(settingsAsMap.getOrDefault("max_page_search_size", "not_set"), equalTo("not_set")); + assertThat(settingsAsMap.getOrDefault("docs_per_second", "not_set"), equalTo("not_set")); + assertNull(settingsAsMap.getOrDefault("write_date_as_epoch_millis", "not_set")); } public void testExplicitNullOnWriteBuilder() throws IOException { @@ -87,9 +101,11 @@ public void testExplicitNullOnWriteBuilder() throws IOException { Map settingsAsMap = xContentToMap(config); assertNull(settingsAsMap.getOrDefault("max_page_search_size", "not_set")); assertThat(settingsAsMap.getOrDefault("docs_per_second", "not_set"), equalTo("not_set")); + assertThat(settingsAsMap.getOrDefault("write_date_as_epoch_millis", "not_set"), equalTo("not_set")); SettingsConfig emptyConfig = new SettingsConfig.Builder().build(); assertNull(emptyConfig.getMaxPageSearchSize()); + assertNull(emptyConfig.getWriteDateAsEpochMillis()); settingsAsMap = xContentToMap(emptyConfig); assertTrue(settingsAsMap.isEmpty()); @@ -100,6 +116,16 @@ public void testExplicitNullOnWriteBuilder() throws IOException { settingsAsMap = xContentToMap(config); assertThat(settingsAsMap.getOrDefault("max_page_search_size", "not_set"), equalTo("not_set")); assertNull(settingsAsMap.getOrDefault("docs_per_second", "not_set")); + assertThat(settingsAsMap.getOrDefault("write_date_as_epoch_millis", "not_set"), equalTo("not_set")); + + config = new SettingsConfig.Builder().setWriteDateAsEpochMilli(null).build(); + // returns null, however it's a different `null` only visible after writing xContent + assertNull(emptyConfig.getWriteDateAsEpochMillis()); + + settingsAsMap = xContentToMap(config); + assertThat(settingsAsMap.getOrDefault("max_page_search_size", "not_set"), equalTo("not_set")); + assertThat(settingsAsMap.getOrDefault("docs_per_second", "not_set"), equalTo("not_set")); + assertNull(settingsAsMap.getOrDefault("write_date_as_epoch_millis", "not_set")); } private Map xContentToMap(ToXContent xcontent) throws IOException { diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/hlrc/SettingsConfigTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/hlrc/SettingsConfigTests.java index 9597c45ce07d3..c182fc3b0692f 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/hlrc/SettingsConfigTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/hlrc/SettingsConfigTests.java @@ -33,7 +33,8 @@ public class SettingsConfigTests extends AbstractResponseTestCase< public static org.elasticsearch.xpack.core.transform.transforms.SettingsConfig randomSettingsConfig() { return new org.elasticsearch.xpack.core.transform.transforms.SettingsConfig( randomBoolean() ? null : randomIntBetween(10, 10_000), - randomBoolean() ? null : randomFloat() + randomBoolean() ? null : randomFloat(), + randomBoolean() ? null : randomIntBetween(0, 1) ); } @@ -43,6 +44,7 @@ public static void assertHlrcEquals( ) { assertEquals(serverTestInstance.getMaxPageSearchSize(), clientInstance.getMaxPageSearchSize()); assertEquals(serverTestInstance.getDocsPerSecond(), clientInstance.getDocsPerSecond()); + assertEquals(serverTestInstance.getWriteDateAsEpochMillis(), clientInstance.getWriteDateAsEpochMillis()); } @Override diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/ObjectParser.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/ObjectParser.java index 26b77551909c0..768c85fd5e363 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/ObjectParser.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/ObjectParser.java @@ -632,6 +632,7 @@ public enum ValueType { INT(VALUE_NUMBER, VALUE_STRING), INT_OR_NULL(VALUE_NUMBER, VALUE_STRING, VALUE_NULL), BOOLEAN(VALUE_BOOLEAN, VALUE_STRING), + BOOLEAN_OR_NULL(VALUE_BOOLEAN, VALUE_STRING, VALUE_NULL), STRING_ARRAY(START_ARRAY, VALUE_STRING), FLOAT_ARRAY(START_ARRAY, VALUE_NUMBER, VALUE_STRING), DOUBLE_ARRAY(START_ARRAY, VALUE_NUMBER, VALUE_STRING), diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformField.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformField.java index cb52699c5b26d..0b305f65baf7a 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformField.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformField.java @@ -35,6 +35,7 @@ public final class TransformField { public static final ParseField FORCE = new ParseField("force"); public static final ParseField MAX_PAGE_SEARCH_SIZE = new ParseField("max_page_search_size"); public static final ParseField DOCS_PER_SECOND = new ParseField("docs_per_second"); + public static final ParseField WRITE_DATE_AS_EPOCH_MILLIS = new ParseField("write_date_as_epoch_millis"); public static final ParseField FIELD = new ParseField("field"); public static final ParseField SYNC = new ParseField("sync"); public static final ParseField TIME_BASED_SYNC = new ParseField("time"); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/SettingsConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/SettingsConfig.java index 56807f7ea6a1b..fedb0a9bb2374 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/SettingsConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/SettingsConfig.java @@ -6,11 +6,14 @@ package org.elasticsearch.xpack.core.transform.transforms; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ObjectParser.ValueType; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; @@ -28,33 +31,53 @@ public class SettingsConfig implements Writeable, ToXContentObject { private static final int DEFAULT_MAX_PAGE_SEARCH_SIZE = -1; private static final float DEFAULT_DOCS_PER_SECOND = -1F; + private static final int DEFAULT_WRITE_DATE_AS_EPOCH_MILLIS = -1; private static ConstructingObjectParser createParser(boolean lenient) { ConstructingObjectParser parser = new ConstructingObjectParser<>( "transform_config_settings", lenient, - args -> new SettingsConfig((Integer) args[0], (Float) args[1]) + args -> new SettingsConfig((Integer) args[0], (Float) args[1], (Integer) args[2]) ); parser.declareIntOrNull(optionalConstructorArg(), DEFAULT_MAX_PAGE_SEARCH_SIZE, TransformField.MAX_PAGE_SEARCH_SIZE); parser.declareFloatOrNull(optionalConstructorArg(), DEFAULT_DOCS_PER_SECOND, TransformField.DOCS_PER_SECOND); + parser.declareField( + optionalConstructorArg(), + p -> p.currentToken() == XContentParser.Token.VALUE_NULL ? DEFAULT_WRITE_DATE_AS_EPOCH_MILLIS : p.booleanValue() ? 1 : 0, + TransformField.WRITE_DATE_AS_EPOCH_MILLIS, + ValueType.BOOLEAN_OR_NULL + ); return parser; } private final Integer maxPageSearchSize; private final Float docsPerSecond; + private final Integer writeDateAsEpochMillis; public SettingsConfig() { - this(null, null); + this(null, null, (Integer) null); } - public SettingsConfig(Integer maxPageSearchSize, Float docsPerSecond) { + public SettingsConfig(Integer maxPageSearchSize, Float docsPerSecond, Boolean writeDateAsEpochMillis) { this.maxPageSearchSize = maxPageSearchSize; this.docsPerSecond = docsPerSecond; + this.writeDateAsEpochMillis = writeDateAsEpochMillis == null ? null : writeDateAsEpochMillis ? 1 : 0; + } + + public SettingsConfig(Integer maxPageSearchSize, Float docsPerSecond, Integer writeDateAsEpochMillis) { + this.maxPageSearchSize = maxPageSearchSize; + this.docsPerSecond = docsPerSecond; + this.writeDateAsEpochMillis = writeDateAsEpochMillis; } public SettingsConfig(final StreamInput in) throws IOException { this.maxPageSearchSize = in.readOptionalInt(); this.docsPerSecond = in.readOptionalFloat(); + if (in.getVersion().onOrAfter(Version.V_8_0_0)) { // todo: change to V_7_11 + this.writeDateAsEpochMillis = in.readOptionalInt(); + } else { + this.writeDateAsEpochMillis = DEFAULT_WRITE_DATE_AS_EPOCH_MILLIS; + } } public Integer getMaxPageSearchSize() { @@ -65,6 +88,14 @@ public Float getDocsPerSecond() { return docsPerSecond; } + public Boolean getWriteDateAsEpochMillis() { + return writeDateAsEpochMillis != null ? writeDateAsEpochMillis > 0 : null; + } + + public Integer getWriteDateAsEpochMillisForUpdate() { + return writeDateAsEpochMillis; + } + public ActionRequestValidationException validate(ActionRequestValidationException validationException) { // TODO: make this dependent on search.max_buckets if (maxPageSearchSize != null && (maxPageSearchSize < 10 || maxPageSearchSize > 10_000)) { @@ -85,6 +116,9 @@ public boolean isValid() { public void writeTo(StreamOutput out) throws IOException { out.writeOptionalInt(maxPageSearchSize); out.writeOptionalFloat(docsPerSecond); + if (out.getVersion().onOrAfter(Version.V_8_0_0)) { // todo: change to V_7_11_0 + out.writeOptionalInt(writeDateAsEpochMillis); + } } @Override @@ -97,6 +131,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (docsPerSecond != null && (docsPerSecond.equals(DEFAULT_DOCS_PER_SECOND) == false)) { builder.field(TransformField.DOCS_PER_SECOND.getPreferredName(), docsPerSecond); } + if (writeDateAsEpochMillis != null && (writeDateAsEpochMillis.equals(DEFAULT_WRITE_DATE_AS_EPOCH_MILLIS) == false)) { + builder.field(TransformField.WRITE_DATE_AS_EPOCH_MILLIS.getPreferredName(), writeDateAsEpochMillis > 0 ? true : false); + } builder.endObject(); return builder; } @@ -111,12 +148,19 @@ public boolean equals(Object other) { } SettingsConfig that = (SettingsConfig) other; - return Objects.equals(maxPageSearchSize, that.maxPageSearchSize) && Objects.equals(docsPerSecond, that.docsPerSecond); + return Objects.equals(maxPageSearchSize, that.maxPageSearchSize) + && Objects.equals(docsPerSecond, that.docsPerSecond) + && Objects.equals(writeDateAsEpochMillis, that.writeDateAsEpochMillis); } @Override public int hashCode() { - return Objects.hash(maxPageSearchSize, docsPerSecond); + return Objects.hash(maxPageSearchSize, docsPerSecond, writeDateAsEpochMillis); + } + + @Override + public String toString() { + return Strings.toString(this, true, true) + " wdaem: " + this.writeDateAsEpochMillis; } public static SettingsConfig fromXContent(final XContentParser parser, boolean lenient) throws IOException { @@ -126,13 +170,12 @@ public static SettingsConfig fromXContent(final XContentParser parser, boolean l public static class Builder { private Integer maxPageSearchSize; private Float docsPerSecond; + private Integer writeDateAsEpochMilli; /** * Default builder */ - public Builder() { - - } + public Builder() {} /** * Builder starting from existing settings as base, for the purpose of partially updating settings. @@ -142,6 +185,7 @@ public Builder() { public Builder(SettingsConfig base) { this.maxPageSearchSize = base.maxPageSearchSize; this.docsPerSecond = base.docsPerSecond; + this.writeDateAsEpochMilli = base.writeDateAsEpochMillis; } /** @@ -172,6 +216,20 @@ public Builder setRequestsPerSecond(Float docsPerSecond) { return this; } + /** + * Sets whether to write the output of a date aggregation as millis since epoch or as formatted string (ISO format). + * + * This setting ensures backwards compatibility for transforms created before 7.11, which wrote dates as epoch_millis. + * The new default is ISO string. + * + * @param writeDateAsEpochMilli true if dates should be written as epoch_millis. + * @return the {@link Builder} with writeDateAsEpochMilli set. + */ + public Builder setWriteDateAsEpochMilli(Boolean writeDateAsEpochMilli) { + this.writeDateAsEpochMilli = writeDateAsEpochMilli == null ? DEFAULT_WRITE_DATE_AS_EPOCH_MILLIS : writeDateAsEpochMilli ? 1 : 0; + return this; + } + /** * Update settings according to given settings config. * @@ -189,12 +247,17 @@ public Builder update(SettingsConfig update) { ? null : update.getMaxPageSearchSize(); } + if (update.getWriteDateAsEpochMillisForUpdate() != null) { + this.writeDateAsEpochMilli = update.getWriteDateAsEpochMillisForUpdate().equals(DEFAULT_WRITE_DATE_AS_EPOCH_MILLIS) + ? null + : update.getWriteDateAsEpochMillisForUpdate(); + } return this; } public SettingsConfig build() { - return new SettingsConfig(maxPageSearchSize, docsPerSecond); + return new SettingsConfig(maxPageSearchSize, docsPerSecond, writeDateAsEpochMilli); } } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java index d5b3e7ddb74a5..355705b9b08b7 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java @@ -344,8 +344,8 @@ public void writeTo(final StreamOutput out) throws IOException { public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException { final boolean excludeGenerated = params.paramAsBoolean(TransformField.EXCLUDE_GENERATED, false); final boolean forInternalStorage = params.paramAsBoolean(TransformField.FOR_INTERNAL_STORAGE, false); - assert (forInternalStorage && excludeGenerated) == false: - "unsupported behavior, exclude_generated is true and for_internal_storage is true"; + assert (forInternalStorage + && excludeGenerated) == false : "unsupported behavior, exclude_generated is true and for_internal_storage is true"; builder.startObject(); builder.field(TransformField.ID.getPreferredName(), id); if (excludeGenerated == false) { @@ -449,12 +449,17 @@ public static TransformConfig fromXContent(final XContentParser parser, @Nullabl public static TransformConfig rewriteForUpdate(final TransformConfig transformConfig) { // quick checks for deprecated features, if none found just return the original - if (transformConfig.getPivotConfig() == null || transformConfig.getPivotConfig().getMaxPageSearchSize() == null) { + if (transformConfig.getVersion() != null + && transformConfig.getVersion().onOrAfter(Version.V_8_0_0) // todo: V_7_11_0 + && (transformConfig.getPivotConfig() == null || transformConfig.getPivotConfig().getMaxPageSearchSize() == null)) { return transformConfig; } Builder builder = new Builder(transformConfig); + /* + * Move pivot.max_page_size_search to settings.max_page_size_search + */ if (transformConfig.getPivotConfig() != null && transformConfig.getPivotConfig().getMaxPageSearchSize() != null) { // create a new pivot config but set maxPageSearchSize to null PivotConfig newPivotConfig = new PivotConfig( @@ -468,9 +473,28 @@ public static TransformConfig rewriteForUpdate(final TransformConfig transformCo Integer maxPageSearchSize = transformConfig.getSettings().getMaxPageSearchSize() != null ? transformConfig.getSettings().getMaxPageSearchSize() : maxPageSearchSizeDeprecated; + Boolean writeDateAsEpochMillis = transformConfig.getSettings().getWriteDateAsEpochMillis(); - builder.setSettings(new SettingsConfig(maxPageSearchSize, transformConfig.getSettings().getDocsPerSecond())); + builder.setSettings( + new SettingsConfig(maxPageSearchSize, transformConfig.getSettings().getDocsPerSecond(), writeDateAsEpochMillis) + ); + } + + /* + * Keep date normalization backwards compatible for transforms created before 7.11 + */ + if (transformConfig.getVersion() != null && transformConfig.getVersion().before(Version.V_8_0_0)) { // todo: V_7_11_0 + Integer maxPageSearchSizeDeprecated = transformConfig.getPivotConfig().getMaxPageSearchSize(); + Integer maxPageSearchSize = transformConfig.getSettings().getMaxPageSearchSize() != null + ? transformConfig.getSettings().getMaxPageSearchSize() + : maxPageSearchSizeDeprecated; + Boolean writeDateAsEpochMillis = true; + + builder.setSettings( + new SettingsConfig(maxPageSearchSize, transformConfig.getSettings().getDocsPerSecond(), writeDateAsEpochMillis) + ); } + return builder.setVersion(Version.CURRENT).build(); } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/SettingsConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/SettingsConfigTests.java index a1b87d74538a8..3439bba3a61cf 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/SettingsConfigTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/SettingsConfigTests.java @@ -28,11 +28,15 @@ public class SettingsConfigTests extends AbstractSerializingTransformTestCase xContentToMap(ToXContent xcontent) throws IOException { diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigTests.java index 697c723f6b8f4..66ccfa591bf1c 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigTests.java @@ -389,6 +389,44 @@ public void testRewriteForUpdateConflicting() throws IOException { assertWarnings("[max_page_search_size] is deprecated inside pivot please use settings instead"); } + public void testRewriteForBWCofDateNormalization() throws IOException { + String pivotTransform = "{" + + " \"id\" : \"body_id\"," + + " \"source\" : {\"index\":\"src\"}," + + " \"dest\" : {\"index\": \"dest\"}," + + " \"pivot\" : {" + + " \"group_by\": {" + + " \"id\": {" + + " \"terms\": {" + + " \"field\": \"id\"" + + "} } }," + + " \"aggs\": {" + + " \"avg\": {" + + " \"avg\": {" + + " \"field\": \"points\"" + + "} } }" + + "}," + + " \"version\" : \"" + + Version.V_7_6_0.toString() + + "\"" + + "}"; + + TransformConfig transformConfig = createTransformConfigFromString(pivotTransform, "body_id", true); + TransformConfig transformConfigRewritten = TransformConfig.rewriteForUpdate(transformConfig); + + assertTrue(transformConfigRewritten.getSettings().getWriteDateAsEpochMillis()); + assertEquals(Version.CURRENT, transformConfigRewritten.getVersion()); + + TransformConfig explicitTrueAfter711 = new TransformConfig.Builder(transformConfig).setSettings( + new SettingsConfig.Builder(transformConfigRewritten.getSettings()).setWriteDateAsEpochMilli(true).build() + ).setVersion(Version.V_8_0_0).build(); // todo: V_7_11_0 + + transformConfigRewritten = TransformConfig.rewriteForUpdate(explicitTrueAfter711); + + assertTrue(transformConfigRewritten.getSettings().getWriteDateAsEpochMillis()); + assertEquals(Version.V_8_0_0, transformConfigRewritten.getVersion()); // todo: V_7_11_0 + } + private TransformConfig createTransformConfigFromString(String json, String id) throws IOException { return createTransformConfigFromString(json, id, false); } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigUpdateTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigUpdateTests.java index e7bbc07262b54..43f0d56e4e9fc 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigUpdateTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigUpdateTests.java @@ -104,7 +104,7 @@ public void testApply() { TimeValue frequency = TimeValue.timeValueSeconds(10); SyncConfig syncConfig = new TimeSyncConfig("time_field", TimeValue.timeValueSeconds(30)); String newDescription = "new description"; - SettingsConfig settings = new SettingsConfig(4_000, 4_000.400F); + SettingsConfig settings = new SettingsConfig(4_000, 4_000.400F, true); update = new TransformConfigUpdate(sourceConfig, destConfig, frequency, syncConfig, newDescription, settings); Map headers = Collections.singletonMap("foo", "bar"); @@ -136,7 +136,14 @@ public void testApplySettings() { randomBoolean() ? null : Version.V_7_2_0.toString() ); - TransformConfigUpdate update = new TransformConfigUpdate(null, null, null, null, null, new SettingsConfig(4_000, null)); + TransformConfigUpdate update = new TransformConfigUpdate( + null, + null, + null, + null, + null, + new SettingsConfig(4_000, null, (Boolean) null) + ); TransformConfig updatedConfig = update.apply(config); // for settings we allow partial updates, so changing 1 setting should not overwrite the other @@ -144,18 +151,18 @@ public void testApplySettings() { assertThat(updatedConfig.getSettings().getMaxPageSearchSize(), equalTo(4_000)); assertThat(updatedConfig.getSettings().getDocsPerSecond(), equalTo(config.getSettings().getDocsPerSecond())); - update = new TransformConfigUpdate(null, null, null, null, null, new SettingsConfig(null, 43.244F)); + update = new TransformConfigUpdate(null, null, null, null, null, new SettingsConfig(null, 43.244F, (Boolean) null)); updatedConfig = update.apply(updatedConfig); assertThat(updatedConfig.getSettings().getMaxPageSearchSize(), equalTo(4_000)); assertThat(updatedConfig.getSettings().getDocsPerSecond(), equalTo(43.244F)); // now reset to default using the magic -1 - update = new TransformConfigUpdate(null, null, null, null, null, new SettingsConfig(-1, null)); + update = new TransformConfigUpdate(null, null, null, null, null, new SettingsConfig(-1, null, (Boolean) null)); updatedConfig = update.apply(updatedConfig); assertNull(updatedConfig.getSettings().getMaxPageSearchSize()); assertThat(updatedConfig.getSettings().getDocsPerSecond(), equalTo(43.244F)); - update = new TransformConfigUpdate(null, null, null, null, null, new SettingsConfig(-1, -1F)); + update = new TransformConfigUpdate(null, null, null, null, null, new SettingsConfig(-1, -1F, (Boolean) null)); updatedConfig = update.apply(updatedConfig); assertNull(updatedConfig.getSettings().getMaxPageSearchSize()); assertNull(updatedConfig.getSettings().getDocsPerSecond()); diff --git a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformProgressIT.java b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformProgressIT.java index 3b9c5265d05ad..63a73a3b97331 100644 --- a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformProgressIT.java +++ b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformProgressIT.java @@ -6,6 +6,7 @@ package org.elasticsearch.xpack.transform.integration; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.LatchedActionListener; import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; @@ -29,6 +30,7 @@ import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.xpack.core.transform.transforms.DestConfig; +import org.elasticsearch.xpack.core.transform.transforms.SettingsConfig; import org.elasticsearch.xpack.core.transform.transforms.SourceConfig; import org.elasticsearch.xpack.core.transform.transforms.TransformConfig; import org.elasticsearch.xpack.core.transform.transforms.TransformProgress; @@ -151,7 +153,7 @@ public void assertGetProgress(int userWithMissingBuckets) throws Exception { PivotConfig pivotConfig = new PivotConfig(histgramGroupConfig, aggregationConfig, null); TransformConfig config = new TransformConfig(transformId, sourceConfig, destConfig, null, null, null, pivotConfig, null, null); - Pivot pivot = new Pivot(pivotConfig, transformId); + Pivot pivot = new Pivot(pivotConfig, transformId, new SettingsConfig(), Version.CURRENT); TransformProgress progress = getProgress(pivot, getProgressQuery(pivot, config.getSource().getIndex(), null)); @@ -179,7 +181,7 @@ public void assertGetProgress(int userWithMissingBuckets) throws Exception { Collections.singletonMap("every_50", new HistogramGroupSource("missing_field", null, missingBucket, 50.0)) ); pivotConfig = new PivotConfig(histgramGroupConfig, aggregationConfig, null); - pivot = new Pivot(pivotConfig, transformId); + pivot = new Pivot(pivotConfig, transformId, new SettingsConfig(), Version.CURRENT); progress = getProgress( pivot, diff --git a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/DateHistogramGroupByIT.java b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/DateHistogramGroupByIT.java index 4364355d2afa8..b3ffe772e7798 100644 --- a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/DateHistogramGroupByIT.java +++ b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/DateHistogramGroupByIT.java @@ -10,6 +10,7 @@ import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.transform.transforms.DestConfig; +import org.elasticsearch.client.transform.transforms.SettingsConfig; import org.elasticsearch.client.transform.transforms.SourceConfig; import org.elasticsearch.client.transform.transforms.TransformConfig; import org.elasticsearch.client.transform.transforms.pivot.DateHistogramGroupSource; @@ -28,7 +29,6 @@ import java.io.IOException; import java.time.Instant; import java.time.ZoneId; -import java.time.ZonedDateTime; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -43,15 +43,21 @@ public class DateHistogramGroupByIT extends ContinuousTestCase { .format(Instant.ofEpochMilli(42)); private final boolean missing; + private final boolean dateAsEpochMillis; public DateHistogramGroupByIT() { missing = randomBoolean(); + dateAsEpochMillis = randomBoolean(); } @Override public TransformConfig createConfig() { TransformConfig.Builder transformConfigBuilder = new TransformConfig.Builder(); addCommonBuilderParameters(transformConfigBuilder); + if (dateAsEpochMillis) { + transformConfigBuilder.setSettings(addCommonSetings(new SettingsConfig.Builder()).setWriteDateAsEpochMilli(true).build()); + } + transformConfigBuilder.setSource(new SourceConfig(CONTINUOUS_EVENTS_SOURCE_INDEX)); transformConfigBuilder.setDest(new DestConfig(NAME, INGEST_PIPELINE)); transformConfigBuilder.setId(NAME); @@ -110,9 +116,16 @@ public void testIteration(int iteration) throws IOException { SearchHit searchHit = destIterator.next(); Map source = searchHit.getSourceAsMap(); - Long transformBucketKey = (Long) XContentMapValues.extractValue("second", source); + String transformBucketKey; + if (dateAsEpochMillis) { + transformBucketKey = ContinuousTestCase.STRICT_DATE_OPTIONAL_TIME_PRINTER_NANOS.withZone(ZoneId.of("UTC")) + .format(Instant.ofEpochMilli((Long) XContentMapValues.extractValue("second", source))); + } else { + transformBucketKey = (String) XContentMapValues.extractValue("second", source); + } + if (transformBucketKey == null) { - transformBucketKey = 42L; + transformBucketKey = MISSING_BUCKET_KEY; } // aggs return buckets with 0 doc_count while composite aggs skip over them @@ -120,13 +133,12 @@ public void testIteration(int iteration) throws IOException { assertTrue(sourceIterator.hasNext()); bucket = sourceIterator.next(); } - long bucketKey = ((ZonedDateTime) bucket.getKey()).toEpochSecond() * 1000; // test correctness, the results from the aggregation and the results from the transform should be the same assertThat( - "Buckets did not match, source: " + source + ", expected: " + bucketKey + ", iteration: " + iteration, + "Buckets did not match, source: " + source + ", expected: " + bucket.getKeyAsString() + ", iteration: " + iteration, transformBucketKey, - equalTo(bucketKey) + equalTo(bucket.getKeyAsString()) ); assertThat( "Doc count did not match, source: " + source + ", expected: " + bucket.getDocCount() + ", iteration: " + iteration, diff --git a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/DateHistogramGroupByOtherTimeFieldIT.java b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/DateHistogramGroupByOtherTimeFieldIT.java index d79c68d986d90..cdad2fe9232e0 100644 --- a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/DateHistogramGroupByOtherTimeFieldIT.java +++ b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/DateHistogramGroupByOtherTimeFieldIT.java @@ -23,7 +23,6 @@ import org.elasticsearch.search.builder.SearchSourceBuilder; import java.io.IOException; -import java.time.ZonedDateTime; import java.util.ArrayList; import java.util.HashMap; import java.util.Iterator; @@ -121,20 +120,19 @@ private void assertResultsGroupByDateHistogram(int iteration, SearchResponse res SearchHit searchHit = destIterator.next(); Map source = searchHit.getSourceAsMap(); - Long transformBucketKey = (Long) XContentMapValues.extractValue("second", source); + String transformBucketKey = (String) XContentMapValues.extractValue("second", source); // aggs return buckets with 0 doc_count while composite aggs skip over them while (bucket.getDocCount() == 0L) { assertTrue(sourceIterator.hasNext()); bucket = sourceIterator.next(); } - long bucketKey = ((ZonedDateTime) bucket.getKey()).toEpochSecond() * 1000; // test correctness, the results from the aggregation and the results from the transform should be the same assertThat( - "Buckets did not match, source: " + source + ", expected: " + bucketKey + ", iteration: " + iteration, + "Buckets did not match, source: " + source + ", expected: " + bucket.getKeyAsString() + ", iteration: " + iteration, transformBucketKey, - equalTo(bucketKey) + equalTo(bucket.getKeyAsString()) ); assertThat( "Doc count did not match, source: " + source + ", expected: " + bucket.getDocCount() + ", iteration: " + iteration, @@ -172,10 +170,9 @@ private void assertResultsGroupByDateHistogramAndTerms(int iteration, SearchResp if (b.getDocCount() == 0) { continue; } - long second = ((ZonedDateTime) b.getKey()).toEpochSecond() * 1000; List terms = ((Terms) b.getAggregations().get("event")).getBuckets(); for (Terms.Bucket t : terms) { - flattenedBuckets.add(flattenedResult(second, t.getKeyAsString(), t.getDocCount())); + flattenedBuckets.add(flattenedResult(b.getKeyAsString(), t.getKeyAsString(), t.getDocCount())); } } @@ -188,7 +185,7 @@ private void assertResultsGroupByDateHistogramAndTerms(int iteration, SearchResp SearchHit searchHit = destIterator.next(); Map source = searchHit.getSourceAsMap(); - Long transformBucketKey = (Long) XContentMapValues.extractValue("second", source); + String transformBucketKey = (String) XContentMapValues.extractValue("second", source); // test correctness, the results from the aggregation and the results from the transform should be the same assertThat( @@ -228,7 +225,7 @@ private void assertResultsGroupByDateHistogramAndTerms(int iteration, SearchResp assertFalse(destIterator.hasNext()); } - private static Map flattenedResult(long second, String event, long count) { + private static Map flattenedResult(String second, String event, long count) { Map doc = new HashMap<>(); doc.put("second", second); doc.put("event", event); diff --git a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/TermsOnDateGroupByIT.java b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/TermsOnDateGroupByIT.java index 2d62643bf0bc9..eac3c5067d288 100644 --- a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/TermsOnDateGroupByIT.java +++ b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/TermsOnDateGroupByIT.java @@ -27,6 +27,8 @@ import org.elasticsearch.search.builder.SearchSourceBuilder; import java.io.IOException; +import java.time.Instant; +import java.time.ZoneId; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -41,8 +43,8 @@ public class TermsOnDateGroupByIT extends ContinuousTestCase { private static final String NAME = "continuous-terms-on-date-pivot-test"; - private static final long MISSING_BUCKET_KEY = 1262304000000L; // 01/01/2010 should end up last when sorting - + private static final String MISSING_BUCKET_KEY = ContinuousTestCase.STRICT_DATE_OPTIONAL_TIME_PRINTER_NANOS.withZone(ZoneId.of("UTC")) + .format(Instant.ofEpochMilli(1262304000000L)); // 01/01/2010 should end up last when sorting private final boolean missing; public TermsOnDateGroupByIT() { @@ -126,15 +128,16 @@ public void testIteration(int iteration) throws IOException { SearchHit searchHit = destIterator.next(); Map source = searchHit.getSourceAsMap(); - Long transformBucketKey = (Long) XContentMapValues.extractValue("some-timestamp", source); + String transformBucketKey = (String) XContentMapValues.extractValue("some-timestamp", source); + if (transformBucketKey == null) { transformBucketKey = MISSING_BUCKET_KEY; } // test correctness, the results from the aggregation and the results from the transform should be the same assertThat( - "Buckets did not match, source: " + source + ", expected: " + bucket.getKey() + ", iteration: " + iteration, + "Buckets did not match, source: " + source + ", expected: " + bucket.getKeyAsString() + ", iteration: " + iteration, transformBucketKey, - equalTo(bucket.getKey()) + equalTo(bucket.getKeyAsString()) ); assertThat( "Doc count did not match, source: " + source + ", expected: " + bucket.getDocCount() + ", iteration: " + iteration, diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/FunctionFactory.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/FunctionFactory.java index 549f7b2793f0e..61b3ec02f7458 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/FunctionFactory.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/FunctionFactory.java @@ -24,7 +24,7 @@ private FunctionFactory() {} */ public static Function create(TransformConfig config) { if (config.getPivotConfig() != null) { - return new Pivot(config.getPivotConfig(), config.getId()); + return new Pivot(config.getPivotConfig(), config.getId(), config.getSettings(), config.getVersion()); } else { throw new IllegalArgumentException("unknown transform function"); } diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/AggregationResultUtils.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/AggregationResultUtils.java index 4d81fd44ff7be..20103d9a1d3db 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/AggregationResultUtils.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/AggregationResultUtils.java @@ -14,6 +14,7 @@ import org.elasticsearch.common.geo.builders.PolygonBuilder; import org.elasticsearch.common.geo.parsers.ShapeParser; import org.elasticsearch.geometry.Rectangle; +import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.search.aggregations.Aggregation; import org.elasticsearch.search.aggregations.AggregationBuilder; import org.elasticsearch.search.aggregations.PipelineAggregationBuilder; @@ -64,6 +65,8 @@ public final class AggregationResultUtils { private static final Map BUCKET_KEY_EXTRACTOR_MAP; private static final BucketKeyExtractor DEFAULT_BUCKET_KEY_EXTRACTOR = new DefaultBucketKeyExtractor(); + private static final BucketKeyExtractor DATE_AS_EPOCH_BUCKET_KEY_EXTRACTOR = new DateAsEpochBucketKeyExtractor(); + static { Map tempMap = new HashMap<>(); tempMap.put(GeoTileGroupSource.class.getName(), new GeoTileBucketKeyExtractor()); @@ -87,7 +90,8 @@ public static Stream> extractCompositeAggregationResults( Collection aggregationBuilders, Collection pipelineAggs, Map fieldTypeMap, - TransformIndexerStats stats + TransformIndexerStats stats, + boolean datesAsEpoch ) { return agg.getBuckets().stream().map(bucket -> { stats.incrementNumDocuments(bucket.getDocCount()); @@ -104,7 +108,7 @@ public static Stream> extractCompositeAggregationResults( updateDocument( document, destinationFieldName, - getBucketKeyExtractor(singleGroupSource).value(value, fieldTypeMap.get(destinationFieldName)) + getBucketKeyExtractor(singleGroupSource, datesAsEpoch).value(value, fieldTypeMap.get(destinationFieldName)) ); }); @@ -128,8 +132,11 @@ public static Stream> extractCompositeAggregationResults( }); } - static BucketKeyExtractor getBucketKeyExtractor(SingleGroupSource groupSource) { - return BUCKET_KEY_EXTRACTOR_MAP.getOrDefault(groupSource.getClass().getName(), DEFAULT_BUCKET_KEY_EXTRACTOR); + static BucketKeyExtractor getBucketKeyExtractor(SingleGroupSource groupSource, boolean datesAsEpoch) { + return BUCKET_KEY_EXTRACTOR_MAP.getOrDefault( + groupSource.getClass().getName(), + datesAsEpoch ? DATE_AS_EPOCH_BUCKET_KEY_EXTRACTOR : DEFAULT_BUCKET_KEY_EXTRACTOR + ); } static AggValueExtractor getExtractor(Aggregation aggregation) { @@ -409,6 +416,22 @@ public Object value(Object key, String type) { static class DefaultBucketKeyExtractor implements BucketKeyExtractor { + @Override + public Object value(Object key, String type) { + if (isNumericType(type) && key instanceof Double) { + return dropFloatingPointComponentIfTypeRequiresIt(type, (Double) key); + } else if ((DateFieldMapper.CONTENT_TYPE.equals(type) || DateFieldMapper.DATE_NANOS_CONTENT_TYPE.equals(type)) + && key instanceof Long) { + return DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.formatMillis((Long) key); + } + + return key; + } + + } + + static class DateAsEpochBucketKeyExtractor implements BucketKeyExtractor { + @Override public Object value(Object key, String type) { if (isNumericType(type) && key instanceof Double) { diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/Pivot.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/Pivot.java index 9ad108ec3e06e..18cc016baf4f4 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/Pivot.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/Pivot.java @@ -10,6 +10,7 @@ import org.apache.logging.log4j.Logger; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ElasticsearchStatusException; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.search.SearchAction; @@ -37,6 +38,7 @@ import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; import org.elasticsearch.xpack.core.transform.TransformField; import org.elasticsearch.xpack.core.transform.TransformMessages; +import org.elasticsearch.xpack.core.transform.transforms.SettingsConfig; import org.elasticsearch.xpack.core.transform.transforms.SourceConfig; import org.elasticsearch.xpack.core.transform.transforms.TransformIndexerStats; import org.elasticsearch.xpack.core.transform.transforms.TransformProgress; @@ -63,13 +65,17 @@ public class Pivot implements Function { private final PivotConfig config; private final String transformId; + private final SettingsConfig settings; + private final Version version; // objects for re-using private final CompositeAggregationBuilder cachedCompositeAggregation; - public Pivot(PivotConfig config, String transformId) { + public Pivot(PivotConfig config, String transformId, SettingsConfig settings, Version version) { this.config = config; this.transformId = transformId; + this.settings = settings; + this.version = version == null ? Version.CURRENT : version; this.cachedCompositeAggregation = createCompositeAggregation(config); } @@ -217,13 +223,22 @@ public Stream> extractResults( Collection aggregationBuilders = config.getAggregationConfig().getAggregatorFactories(); Collection pipelineAggregationBuilders = config.getAggregationConfig().getPipelineAggregatorFactories(); + // defines how dates are written, if not specified in settings + // < 7.11 as epoch millis + // >= 7.11 as string + // note: it depends on the version when the transform has been created, not the version of the code + boolean datesAsEpoch = settings.getWriteDateAsEpochMillis() != null ? settings.getWriteDateAsEpochMillis() + : version.onOrAfter(Version.V_8_0_0) ? false // todo V_7_11_0 + : true; + return AggregationResultUtils.extractCompositeAggregationResults( agg, groups, aggregationBuilders, pipelineAggregationBuilders, fieldTypeMap, - transformIndexerStats + transformIndexerStats, + datesAsEpoch ); } diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerStateTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerStateTests.java index 5b978ba1485cd..cea6cb6462d5e 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerStateTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerStateTests.java @@ -461,7 +461,7 @@ public void testStopAtCheckpointForThrottledTransform() throws Exception { null, randomPivotConfig(), randomBoolean() ? null : randomAlphaOfLengthBetween(1, 1000), - new SettingsConfig(null, Float.valueOf(1.0f)) + new SettingsConfig(null, Float.valueOf(1.0f), (Boolean) null) ); AtomicReference state = new AtomicReference<>(IndexerState.STARTED); diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerTests.java index 8bf19339d09d8..0d13abf2f8176 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerTests.java @@ -263,7 +263,7 @@ public void testPageSizeAdapt() throws Exception { null, randomPivotConfig(), randomBoolean() ? null : randomAlphaOfLengthBetween(1, 1000), - new SettingsConfig(pageSize, null) + new SettingsConfig(pageSize, null, (Boolean) null) ); AtomicReference state = new AtomicReference<>(IndexerState.STOPPED); final long initialPageSize = pageSize == null ? Transform.DEFAULT_INITIAL_MAX_PAGE_SEARCH_SIZE : pageSize; @@ -331,7 +331,7 @@ public void testDoProcessAggNullCheck() { null, randomPivotConfig(), randomBoolean() ? null : randomAlphaOfLengthBetween(1, 1000), - new SettingsConfig(pageSize, null) + new SettingsConfig(pageSize, null, (Boolean) null) ); SearchResponse searchResponse = new SearchResponse( new InternalSearchResponse( @@ -389,7 +389,7 @@ public void testScriptError() throws Exception { null, randomPivotConfig(), randomBoolean() ? null : randomAlphaOfLengthBetween(1, 1000), - new SettingsConfig(pageSize, null) + new SettingsConfig(pageSize, null, (Boolean) null) ); AtomicReference state = new AtomicReference<>(IndexerState.STOPPED); Function searchFunction = searchRequest -> { diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/AggregationResultUtilsTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/AggregationResultUtilsTests.java index 49ff7ad0a35fd..27296f1765b6b 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/AggregationResultUtilsTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/AggregationResultUtilsTests.java @@ -923,7 +923,8 @@ private List> runExtraction( aggregationBuilders, pipelineAggregationBuilders, fieldTypeMap, - stats + stats, + true ).collect(Collectors.toList()); } } diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/PivotTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/PivotTests.java index bdccea45eed14..bfcee639ac988 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/PivotTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/PivotTests.java @@ -8,6 +8,7 @@ import org.apache.lucene.search.TotalHits; import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionResponse; @@ -29,6 +30,7 @@ import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.client.NoOpClient; import org.elasticsearch.xpack.core.transform.transforms.QueryConfig; +import org.elasticsearch.xpack.core.transform.transforms.SettingsConfig; import org.elasticsearch.xpack.core.transform.transforms.SourceConfig; import org.elasticsearch.xpack.core.transform.transforms.pivot.AggregationConfig; import org.elasticsearch.xpack.core.transform.transforms.pivot.GroupConfigTests; @@ -93,14 +95,14 @@ protected NamedXContentRegistry xContentRegistry() { public void testValidateExistingIndex() throws Exception { SourceConfig source = new SourceConfig(new String[] { "existing_source_index" }, QueryConfig.matchAll()); - Function pivot = new Pivot(getValidPivotConfig(), randomAlphaOfLength(10)); + Function pivot = new Pivot(getValidPivotConfig(), randomAlphaOfLength(10), new SettingsConfig(), Version.CURRENT); assertValidTransform(client, source, pivot); } public void testValidateNonExistingIndex() throws Exception { SourceConfig source = new SourceConfig(new String[] { "non_existing_source_index" }, QueryConfig.matchAll()); - Function pivot = new Pivot(getValidPivotConfig(), randomAlphaOfLength(10)); + Function pivot = new Pivot(getValidPivotConfig(), randomAlphaOfLength(10), new SettingsConfig(), Version.CURRENT); assertInvalidTransform(client, source, pivot); } @@ -110,13 +112,17 @@ public void testInitialPageSize() throws Exception { Function pivot = new Pivot( new PivotConfig(GroupConfigTests.randomGroupConfig(), getValidAggregationConfig(), expectedPageSize), - randomAlphaOfLength(10) + randomAlphaOfLength(10), + new SettingsConfig(), + Version.CURRENT ); assertThat(pivot.getInitialPageSize(), equalTo(expectedPageSize)); pivot = new Pivot( new PivotConfig(GroupConfigTests.randomGroupConfig(), getValidAggregationConfig(), null), - randomAlphaOfLength(10) + randomAlphaOfLength(10), + new SettingsConfig(), + Version.CURRENT ); assertThat(pivot.getInitialPageSize(), equalTo(Transform.DEFAULT_INITIAL_MAX_PAGE_SEARCH_SIZE)); @@ -128,7 +134,7 @@ public void testSearchFailure() throws Exception { // search has failures although they might just be temporary SourceConfig source = new SourceConfig(new String[] { "existing_source_index_with_failing_shards" }, QueryConfig.matchAll()); - Function pivot = new Pivot(getValidPivotConfig(), randomAlphaOfLength(10)); + Function pivot = new Pivot(getValidPivotConfig(), randomAlphaOfLength(10), new SettingsConfig(), Version.CURRENT); assertInvalidTransform(client, source, pivot); } @@ -138,7 +144,12 @@ public void testValidateAllSupportedAggregations() throws Exception { AggregationConfig aggregationConfig = getAggregationConfig(agg); SourceConfig source = new SourceConfig(new String[] { "existing_source" }, QueryConfig.matchAll()); - Function pivot = new Pivot(getValidPivotConfig(aggregationConfig), randomAlphaOfLength(10)); + Function pivot = new Pivot( + getValidPivotConfig(aggregationConfig), + randomAlphaOfLength(10), + new SettingsConfig(), + Version.CURRENT + ); assertValidTransform(client, source, pivot); } } @@ -147,7 +158,12 @@ public void testValidateAllUnsupportedAggregations() throws Exception { for (String agg : unsupportedAggregations) { AggregationConfig aggregationConfig = getAggregationConfig(agg); - Function pivot = new Pivot(getValidPivotConfig(aggregationConfig), randomAlphaOfLength(10)); + Function pivot = new Pivot( + getValidPivotConfig(aggregationConfig), + randomAlphaOfLength(10), + new SettingsConfig(), + Version.CURRENT + ); pivot.validateConfig(ActionListener.wrap(r -> { fail("expected an exception but got a response"); }, e -> { assertThat(e, anyOf(instanceOf(ElasticsearchException.class))); From 56a0ba54d5717dc745eb354290c8a5806e3ffd91 Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Mon, 30 Nov 2020 12:20:47 +0100 Subject: [PATCH 02/15] adapt rest tests --- .../test/transform/preview_transforms.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/transform/preview_transforms.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/transform/preview_transforms.yml index dc5d4dce4317f..6615aa0fc6676 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/transform/preview_transforms.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/transform/preview_transforms.yml @@ -84,17 +84,17 @@ setup: } } - match: { preview.0.airline: foo } - - match: { preview.0.by-hour: 1487376000000 } + - match: { preview.0.by-hour: 2017-02-18T00:00:00.000Z } - match: { preview.0.avg_response: 1.0 } - match: { preview.0.time.max: "2017-02-18T00:30:00.000Z" } - match: { preview.0.time.min: "2017-02-18T00:00:00.000Z" } - match: { preview.1.airline: bar } - - match: { preview.1.by-hour: 1487379600000 } + - match: { preview.1.by-hour: 2017-02-18T01:00:00.000Z } - match: { preview.1.avg_response: 42.0 } - match: { preview.1.time.max: "2017-02-18T01:00:00.000Z" } - match: { preview.1.time.min: "2017-02-18T01:00:00.000Z" } - match: { preview.2.airline: foo } - - match: { preview.2.by-hour: 1487379600000 } + - match: { preview.2.by-hour: 2017-02-18T01:00:00.000Z } - match: { preview.2.avg_response: 42.0 } - match: { preview.2.time.max: "2017-02-18T01:01:00.000Z" } - match: { preview.2.time.min: "2017-02-18T01:01:00.000Z" } @@ -135,15 +135,15 @@ setup: } } - match: { preview.0.airline: foo } - - match: { preview.0.by-hour: 1487376000000 } + - match: { preview.0.by-hour: 2017-02-18T00:00:00.000Z } - match: { preview.0.avg_response: 1.0 } - match: { preview.0.my_field: 42 } - match: { preview.1.airline: bar } - - match: { preview.1.by-hour: 1487379600000 } + - match: { preview.1.by-hour: 2017-02-18T01:00:00.000Z } - match: { preview.1.avg_response: 42.0 } - match: { preview.1.my_field: 42 } - match: { preview.2.airline: foo } - - match: { preview.2.by-hour: 1487379600000 } + - match: { preview.2.by-hour: 2017-02-18T01:00:00.000Z } - match: { preview.2.avg_response: 42.0 } - match: { preview.2.my_field: 42 } - match: { generated_dest_index.mappings.properties.airline.type: "keyword" } From 49d683c4213d052785e975274fa3e8259755de3e Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Mon, 30 Nov 2020 15:29:05 +0100 Subject: [PATCH 03/15] improve doc strings --- .../client/transform/transforms/SettingsConfig.java | 9 ++++++--- .../client/transform/transforms/SettingsConfigTests.java | 2 +- .../xpack/core/transform/transforms/SettingsConfig.java | 9 ++++++--- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/SettingsConfig.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/SettingsConfig.java index 4b0c210b2e6c9..e12e3377061fb 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/SettingsConfig.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/SettingsConfig.java @@ -55,6 +55,7 @@ public class SettingsConfig implements ToXContentObject { static { PARSER.declareIntOrNull(optionalConstructorArg(), DEFAULT_MAX_PAGE_SEARCH_SIZE, MAX_PAGE_SEARCH_SIZE); PARSER.declareFloatOrNull(optionalConstructorArg(), DEFAULT_DOCS_PER_SECOND, DOCS_PER_SECOND); + // this boolean requires 4 possible values: true, false, not_specified, default, therefore using a custom parser PARSER.declareField( optionalConstructorArg(), p -> p.currentToken() == XContentParser.Token.VALUE_NULL ? DEFAULT_WRITE_DATE_AS_EPOCH_MILLIS : p.booleanValue() ? 1 : 0, @@ -171,10 +172,12 @@ public Builder setRequestsPerSecond(Float docsPerSecond) { } /** - * Sets whether to write the output of a date aggregation as millis since epoch or as formatted string (ISO format). + * Whether to write the output of a date aggregation as millis since epoch or as formatted string (ISO format). * - * This setting ensures backwards compatibility for transforms created before 7.11, which wrote dates as epoch_millis. - * The new default is ISO string. + * Transforms created before 7.11 write dates as epoch_millis. The new default is ISO string. + * You can use this setter to configure the old style writing as epoch millis. + * + * An explicit `null` resets to default. * * @param writeDateAsEpochMilli true if dates should be written as epoch_millis. * @return the {@link Builder} with writeDateAsEpochMilli set. diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/SettingsConfigTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/SettingsConfigTests.java index f1e35c03d2a2e..568b5fa7b99a3 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/SettingsConfigTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/SettingsConfigTests.java @@ -119,7 +119,7 @@ public void testExplicitNullOnWriteBuilder() throws IOException { assertThat(settingsAsMap.getOrDefault("write_date_as_epoch_millis", "not_set"), equalTo("not_set")); config = new SettingsConfig.Builder().setWriteDateAsEpochMilli(null).build(); - // returns null, however it's a different `null` only visible after writing xContent + // returns null, however it's `null` as in "use default" assertNull(emptyConfig.getWriteDateAsEpochMillis()); settingsAsMap = xContentToMap(config); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/SettingsConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/SettingsConfig.java index fedb0a9bb2374..8a4e876e01516 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/SettingsConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/SettingsConfig.java @@ -41,6 +41,7 @@ private static ConstructingObjectParser createParser(boole ); parser.declareIntOrNull(optionalConstructorArg(), DEFAULT_MAX_PAGE_SEARCH_SIZE, TransformField.MAX_PAGE_SEARCH_SIZE); parser.declareFloatOrNull(optionalConstructorArg(), DEFAULT_DOCS_PER_SECOND, TransformField.DOCS_PER_SECOND); + // this boolean requires 4 possible values: true, false, not_specified, default, therefore using a custom parser parser.declareField( optionalConstructorArg(), p -> p.currentToken() == XContentParser.Token.VALUE_NULL ? DEFAULT_WRITE_DATE_AS_EPOCH_MILLIS : p.booleanValue() ? 1 : 0, @@ -217,10 +218,12 @@ public Builder setRequestsPerSecond(Float docsPerSecond) { } /** - * Sets whether to write the output of a date aggregation as millis since epoch or as formatted string (ISO format). + * Whether to write the output of a date aggregation as millis since epoch or as formatted string (ISO format). * - * This setting ensures backwards compatibility for transforms created before 7.11, which wrote dates as epoch_millis. - * The new default is ISO string. + * Transforms created before 7.11 write dates as epoch_millis. The new default is ISO string. + * You can use this setter to configure the old style writing as epoch millis. + * + * An explicit `null` resets to default. * * @param writeDateAsEpochMilli true if dates should be written as epoch_millis. * @return the {@link Builder} with writeDateAsEpochMilli set. From a28309546717427d92822ae35d7ad76666daef28 Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Mon, 30 Nov 2020 16:19:20 +0100 Subject: [PATCH 04/15] document changes --- docs/reference/rest-api/common-parms.asciidoc | 7 +++++++ docs/reference/transform/apis/put-transform.asciidoc | 3 +++ docs/reference/transform/apis/update-transform.asciidoc | 3 +++ 3 files changed, 13 insertions(+) diff --git a/docs/reference/rest-api/common-parms.asciidoc b/docs/reference/rest-api/common-parms.asciidoc index b9c2029fb1cba..b804c1ef793c6 100644 --- a/docs/reference/rest-api/common-parms.asciidoc +++ b/docs/reference/rest-api/common-parms.asciidoc @@ -965,6 +965,13 @@ adjusted to a lower value. The minimum value is `10` and the maximum is `10,000` The default value is `500`. end::transform-settings-max-page-search-size[] +tag::transform-settings-write-date-as-epoch-milli[] +Defines if dates in the ouput should be written as ISO formated string (default) +or as millis since epoch. `epoch_millis` has been the default for transforms created +before version `7.11`. For compatible output set this to true. +The default value is `false`. +end::transform-settings-write-date-as-epoch-milli[] + tag::target-index[] ``:: + diff --git a/docs/reference/transform/apis/put-transform.asciidoc b/docs/reference/transform/apis/put-transform.asciidoc index 54a7c4d635afc..92852a1c340a5 100644 --- a/docs/reference/transform/apis/put-transform.asciidoc +++ b/docs/reference/transform/apis/put-transform.asciidoc @@ -140,6 +140,9 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-doc `max_page_search_size`::: (Optional, integer) include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-max-page-search-size] +`write_date_as_epoch_millis`::: +(Optional, boolean) +include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-write-date-as-epoch-milli] ==== //End settings diff --git a/docs/reference/transform/apis/update-transform.asciidoc b/docs/reference/transform/apis/update-transform.asciidoc index be1ebbc38c128..782e920a2975a 100644 --- a/docs/reference/transform/apis/update-transform.asciidoc +++ b/docs/reference/transform/apis/update-transform.asciidoc @@ -116,6 +116,9 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-doc `max_page_search_size`::: (Optional, integer) include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-max-page-search-size] +`write_date_as_epoch_millis`::: +(Optional, boolean) +include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-write-date-as-epoch-milli] ==== //End settings From 4e1ad7f4450bb29ffa502ca102348f09f0127195 Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Tue, 1 Dec 2020 11:26:43 +0100 Subject: [PATCH 05/15] rename parameter to dates_as_epoch_millis --- .../transform/transforms/SettingsConfig.java | 36 ++++++------- .../transforms/SettingsConfigTests.java | 10 ++-- .../transforms/hlrc/SettingsConfigTests.java | 2 +- docs/reference/rest-api/common-parms.asciidoc | 12 ++--- .../transform/apis/put-transform.asciidoc | 6 +-- .../transform/apis/update-transform.asciidoc | 6 +-- .../xpack/core/transform/TransformField.java | 2 +- .../transform/transforms/SettingsConfig.java | 50 +++++++++---------- .../transform/transforms/TransformConfig.java | 8 +-- .../transforms/SettingsConfigTests.java | 24 ++++----- .../transforms/TransformConfigTests.java | 6 +-- .../continuous/DateHistogramGroupByIT.java | 2 +- .../transform/transforms/pivot/Pivot.java | 2 +- 13 files changed, 82 insertions(+), 84 deletions(-) diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/SettingsConfig.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/SettingsConfig.java index e12e3377061fb..f32aaf1a1838d 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/SettingsConfig.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/SettingsConfig.java @@ -35,16 +35,16 @@ public class SettingsConfig implements ToXContentObject { private static final ParseField MAX_PAGE_SEARCH_SIZE = new ParseField("max_page_search_size"); private static final ParseField DOCS_PER_SECOND = new ParseField("docs_per_second"); - private static final ParseField WRITE_DATE_AS_EPOCH_MILLIS = new ParseField("write_date_as_epoch_millis"); + private static final ParseField DATES_AS_EPOCH_MILLIS = new ParseField("dates_as_epoch_millis"); private static final int DEFAULT_MAX_PAGE_SEARCH_SIZE = -1; private static final float DEFAULT_DOCS_PER_SECOND = -1F; // use an integer as we need to code 4 states: true, false, null (unchanged), default (defined server side) - private static final int DEFAULT_WRITE_DATE_AS_EPOCH_MILLIS = -1; + private static final int DEFAULT_DATES_AS_EPOCH_MILLIS = -1; private final Integer maxPageSearchSize; private final Float docsPerSecond; - private final Integer writeDateAsEpochMillis; + private final Integer datesAsEpochMillis; private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( "settings_config", @@ -58,8 +58,8 @@ public class SettingsConfig implements ToXContentObject { // this boolean requires 4 possible values: true, false, not_specified, default, therefore using a custom parser PARSER.declareField( optionalConstructorArg(), - p -> p.currentToken() == XContentParser.Token.VALUE_NULL ? DEFAULT_WRITE_DATE_AS_EPOCH_MILLIS : p.booleanValue() ? 1 : 0, - WRITE_DATE_AS_EPOCH_MILLIS, + p -> p.currentToken() == XContentParser.Token.VALUE_NULL ? DEFAULT_DATES_AS_EPOCH_MILLIS : p.booleanValue() ? 1 : 0, + DATES_AS_EPOCH_MILLIS, ValueType.BOOLEAN_OR_NULL ); } @@ -68,10 +68,10 @@ public static SettingsConfig fromXContent(final XContentParser parser) { return PARSER.apply(parser, null); } - SettingsConfig(Integer maxPageSearchSize, Float docsPerSecond, Integer writeDateAsEpochMillis) { + SettingsConfig(Integer maxPageSearchSize, Float docsPerSecond, Integer datesAsEpochMillis) { this.maxPageSearchSize = maxPageSearchSize; this.docsPerSecond = docsPerSecond; - this.writeDateAsEpochMillis = writeDateAsEpochMillis; + this.datesAsEpochMillis = datesAsEpochMillis; } @Override @@ -91,11 +91,11 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(DOCS_PER_SECOND.getPreferredName(), docsPerSecond); } } - if (writeDateAsEpochMillis != null) { - if (writeDateAsEpochMillis.equals(DEFAULT_WRITE_DATE_AS_EPOCH_MILLIS)) { - builder.field(WRITE_DATE_AS_EPOCH_MILLIS.getPreferredName(), (Boolean) null); + if (datesAsEpochMillis != null) { + if (datesAsEpochMillis.equals(DEFAULT_DATES_AS_EPOCH_MILLIS)) { + builder.field(DATES_AS_EPOCH_MILLIS.getPreferredName(), (Boolean) null); } else { - builder.field(WRITE_DATE_AS_EPOCH_MILLIS.getPreferredName(), writeDateAsEpochMillis > 0 ? true : false); + builder.field(DATES_AS_EPOCH_MILLIS.getPreferredName(), datesAsEpochMillis > 0 ? true : false); } } builder.endObject(); @@ -110,8 +110,8 @@ public Float getDocsPerSecond() { return docsPerSecond; } - public Boolean getWriteDateAsEpochMillis() { - return writeDateAsEpochMillis != null ? writeDateAsEpochMillis > 0 : null; + public Boolean getDatesAsEpochMillis() { + return datesAsEpochMillis != null ? datesAsEpochMillis > 0 : null; } @Override @@ -126,12 +126,12 @@ public boolean equals(Object other) { SettingsConfig that = (SettingsConfig) other; return Objects.equals(maxPageSearchSize, that.maxPageSearchSize) && Objects.equals(docsPerSecond, that.docsPerSecond) - && Objects.equals(writeDateAsEpochMillis, that.writeDateAsEpochMillis); + && Objects.equals(datesAsEpochMillis, that.datesAsEpochMillis); } @Override public int hashCode() { - return Objects.hash(maxPageSearchSize, docsPerSecond, writeDateAsEpochMillis); + return Objects.hash(maxPageSearchSize, docsPerSecond, datesAsEpochMillis); } public static Builder builder() { @@ -179,11 +179,11 @@ public Builder setRequestsPerSecond(Float docsPerSecond) { * * An explicit `null` resets to default. * - * @param writeDateAsEpochMilli true if dates should be written as epoch_millis. + * @param datesAsEpochMilli true if dates should be written as epoch_millis. * @return the {@link Builder} with writeDateAsEpochMilli set. */ - public Builder setWriteDateAsEpochMilli(Boolean writeDateAsEpochMilli) { - this.writeDateAsEpochMilli = writeDateAsEpochMilli == null ? DEFAULT_WRITE_DATE_AS_EPOCH_MILLIS : writeDateAsEpochMilli ? 1 : 0; + public Builder setDatesAsEpochMilli(Boolean datesAsEpochMilli) { + this.writeDateAsEpochMilli = datesAsEpochMilli == null ? DEFAULT_DATES_AS_EPOCH_MILLIS : datesAsEpochMilli ? 1 : 0; return this; } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/SettingsConfigTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/SettingsConfigTests.java index 568b5fa7b99a3..9b4be5ca19449 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/SettingsConfigTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/SettingsConfigTests.java @@ -71,7 +71,7 @@ public void testExplicitNullOnWriteParser() throws IOException { SettingsConfig emptyConfig = fromString("{}"); assertNull(emptyConfig.getMaxPageSearchSize()); - assertNull(emptyConfig.getWriteDateAsEpochMillis()); + assertNull(emptyConfig.getDatesAsEpochMillis()); settingsAsMap = xContentToMap(emptyConfig); assertTrue(settingsAsMap.isEmpty()); @@ -85,7 +85,7 @@ public void testExplicitNullOnWriteParser() throws IOException { assertThat(settingsAsMap.getOrDefault("write_date_as_epoch_millis", "not_set"), equalTo("not_set")); config = fromString("{\"write_date_as_epoch_millis\" : null}"); - assertNull(emptyConfig.getWriteDateAsEpochMillis()); + assertNull(emptyConfig.getDatesAsEpochMillis()); settingsAsMap = xContentToMap(config); assertThat(settingsAsMap.getOrDefault("max_page_search_size", "not_set"), equalTo("not_set")); @@ -105,7 +105,7 @@ public void testExplicitNullOnWriteBuilder() throws IOException { SettingsConfig emptyConfig = new SettingsConfig.Builder().build(); assertNull(emptyConfig.getMaxPageSearchSize()); - assertNull(emptyConfig.getWriteDateAsEpochMillis()); + assertNull(emptyConfig.getDatesAsEpochMillis()); settingsAsMap = xContentToMap(emptyConfig); assertTrue(settingsAsMap.isEmpty()); @@ -118,9 +118,9 @@ public void testExplicitNullOnWriteBuilder() throws IOException { assertNull(settingsAsMap.getOrDefault("docs_per_second", "not_set")); assertThat(settingsAsMap.getOrDefault("write_date_as_epoch_millis", "not_set"), equalTo("not_set")); - config = new SettingsConfig.Builder().setWriteDateAsEpochMilli(null).build(); + config = new SettingsConfig.Builder().setDatesAsEpochMilli(null).build(); // returns null, however it's `null` as in "use default" - assertNull(emptyConfig.getWriteDateAsEpochMillis()); + assertNull(emptyConfig.getDatesAsEpochMillis()); settingsAsMap = xContentToMap(config); assertThat(settingsAsMap.getOrDefault("max_page_search_size", "not_set"), equalTo("not_set")); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/hlrc/SettingsConfigTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/hlrc/SettingsConfigTests.java index c182fc3b0692f..42fa428664251 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/hlrc/SettingsConfigTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/hlrc/SettingsConfigTests.java @@ -44,7 +44,7 @@ public static void assertHlrcEquals( ) { assertEquals(serverTestInstance.getMaxPageSearchSize(), clientInstance.getMaxPageSearchSize()); assertEquals(serverTestInstance.getDocsPerSecond(), clientInstance.getDocsPerSecond()); - assertEquals(serverTestInstance.getWriteDateAsEpochMillis(), clientInstance.getWriteDateAsEpochMillis()); + assertEquals(serverTestInstance.getDatesAsEpochMillis(), clientInstance.getDatesAsEpochMillis()); } @Override diff --git a/docs/reference/rest-api/common-parms.asciidoc b/docs/reference/rest-api/common-parms.asciidoc index b804c1ef793c6..be427290311d3 100644 --- a/docs/reference/rest-api/common-parms.asciidoc +++ b/docs/reference/rest-api/common-parms.asciidoc @@ -148,10 +148,10 @@ The destination for the {transform}. end::dest[] tag::dest-index[] -The _destination index_ for the {transform}. The mappings of the destination -index are deduced based on the source fields when possible. If alternate -mappings are required, use the -https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-create-index.html[Create index API] +The _destination index_ for the {transform}. The mappings of the destination +index are deduced based on the source fields when possible. If alternate +mappings are required, use the +https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-create-index.html[Create index API] prior to starting the {transform}. end::dest-index[] @@ -965,12 +965,12 @@ adjusted to a lower value. The minimum value is `10` and the maximum is `10,000` The default value is `500`. end::transform-settings-max-page-search-size[] -tag::transform-settings-write-date-as-epoch-milli[] +tag::transform-settings-dates-as-epoch-milli[] Defines if dates in the ouput should be written as ISO formated string (default) or as millis since epoch. `epoch_millis` has been the default for transforms created before version `7.11`. For compatible output set this to true. The default value is `false`. -end::transform-settings-write-date-as-epoch-milli[] +end::transform-settings-dates-as-epoch-milli[] tag::target-index[] ``:: diff --git a/docs/reference/transform/apis/put-transform.asciidoc b/docs/reference/transform/apis/put-transform.asciidoc index 92852a1c340a5..e03c45d8a2889 100644 --- a/docs/reference/transform/apis/put-transform.asciidoc +++ b/docs/reference/transform/apis/put-transform.asciidoc @@ -18,7 +18,7 @@ Instantiates a {transform}. [[put-transform-prereqs]] == {api-prereq-title} -If the {es} {security-features} are enabled, you must have the following +If the {es} {security-features} are enabled, you must have the following built-in roles and privileges: * `transform_admin` @@ -140,7 +140,7 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-doc `max_page_search_size`::: (Optional, integer) include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-max-page-search-size] -`write_date_as_epoch_millis`::: +`dates_as_epoch_millis`::: (Optional, boolean) include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-write-date-as-epoch-milli] ==== @@ -186,7 +186,7 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=sync-time] `delay`:::: (Optional, <>) include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=sync-time-delay] - + `field`:::: (Required, string) include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=sync-time-field] diff --git a/docs/reference/transform/apis/update-transform.asciidoc b/docs/reference/transform/apis/update-transform.asciidoc index 782e920a2975a..f4f64465af04c 100644 --- a/docs/reference/transform/apis/update-transform.asciidoc +++ b/docs/reference/transform/apis/update-transform.asciidoc @@ -18,7 +18,7 @@ Updates certain properties of a {transform}. [[update-transform-prereqs]] == {api-prereq-title} -If the {es} {security-features} are enabled, you must have the following +If the {es} {security-features} are enabled, you must have the following built-in roles and privileges: * `transform_admin` @@ -116,7 +116,7 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-doc `max_page_search_size`::: (Optional, integer) include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-max-page-search-size] -`write_date_as_epoch_millis`::: +`dates_as_epoch_millis`::: (Optional, boolean) include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-write-date-as-epoch-milli] ==== @@ -134,7 +134,7 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=source-transforms] `index`::: (Required, string or array) include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=source-index-transforms] - + `query`::: (Optional, object) include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=source-query-transforms] diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformField.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformField.java index 0b305f65baf7a..39879afff7b94 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformField.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformField.java @@ -35,7 +35,7 @@ public final class TransformField { public static final ParseField FORCE = new ParseField("force"); public static final ParseField MAX_PAGE_SEARCH_SIZE = new ParseField("max_page_search_size"); public static final ParseField DOCS_PER_SECOND = new ParseField("docs_per_second"); - public static final ParseField WRITE_DATE_AS_EPOCH_MILLIS = new ParseField("write_date_as_epoch_millis"); + public static final ParseField DATES_AS_EPOCH_MILLIS = new ParseField("dates_as_epoch_millis"); public static final ParseField FIELD = new ParseField("field"); public static final ParseField SYNC = new ParseField("sync"); public static final ParseField TIME_BASED_SYNC = new ParseField("time"); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/SettingsConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/SettingsConfig.java index 8a4e876e01516..4b8d44f19586f 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/SettingsConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/SettingsConfig.java @@ -45,7 +45,7 @@ private static ConstructingObjectParser createParser(boole parser.declareField( optionalConstructorArg(), p -> p.currentToken() == XContentParser.Token.VALUE_NULL ? DEFAULT_WRITE_DATE_AS_EPOCH_MILLIS : p.booleanValue() ? 1 : 0, - TransformField.WRITE_DATE_AS_EPOCH_MILLIS, + TransformField.DATES_AS_EPOCH_MILLIS, ValueType.BOOLEAN_OR_NULL ); return parser; @@ -53,31 +53,31 @@ private static ConstructingObjectParser createParser(boole private final Integer maxPageSearchSize; private final Float docsPerSecond; - private final Integer writeDateAsEpochMillis; + private final Integer datesAsEpochMillis; public SettingsConfig() { this(null, null, (Integer) null); } - public SettingsConfig(Integer maxPageSearchSize, Float docsPerSecond, Boolean writeDateAsEpochMillis) { + public SettingsConfig(Integer maxPageSearchSize, Float docsPerSecond, Boolean datesAsEpochMillis) { this.maxPageSearchSize = maxPageSearchSize; this.docsPerSecond = docsPerSecond; - this.writeDateAsEpochMillis = writeDateAsEpochMillis == null ? null : writeDateAsEpochMillis ? 1 : 0; + this.datesAsEpochMillis = datesAsEpochMillis == null ? null : datesAsEpochMillis ? 1 : 0; } - public SettingsConfig(Integer maxPageSearchSize, Float docsPerSecond, Integer writeDateAsEpochMillis) { + public SettingsConfig(Integer maxPageSearchSize, Float docsPerSecond, Integer datesAsEpochMillis) { this.maxPageSearchSize = maxPageSearchSize; this.docsPerSecond = docsPerSecond; - this.writeDateAsEpochMillis = writeDateAsEpochMillis; + this.datesAsEpochMillis = datesAsEpochMillis; } public SettingsConfig(final StreamInput in) throws IOException { this.maxPageSearchSize = in.readOptionalInt(); this.docsPerSecond = in.readOptionalFloat(); if (in.getVersion().onOrAfter(Version.V_8_0_0)) { // todo: change to V_7_11 - this.writeDateAsEpochMillis = in.readOptionalInt(); + this.datesAsEpochMillis = in.readOptionalInt(); } else { - this.writeDateAsEpochMillis = DEFAULT_WRITE_DATE_AS_EPOCH_MILLIS; + this.datesAsEpochMillis = DEFAULT_WRITE_DATE_AS_EPOCH_MILLIS; } } @@ -89,12 +89,12 @@ public Float getDocsPerSecond() { return docsPerSecond; } - public Boolean getWriteDateAsEpochMillis() { - return writeDateAsEpochMillis != null ? writeDateAsEpochMillis > 0 : null; + public Boolean getDatesAsEpochMillis() { + return datesAsEpochMillis != null ? datesAsEpochMillis > 0 : null; } - public Integer getWriteDateAsEpochMillisForUpdate() { - return writeDateAsEpochMillis; + public Integer getDatesAsEpochMillisForUpdate() { + return datesAsEpochMillis; } public ActionRequestValidationException validate(ActionRequestValidationException validationException) { @@ -118,7 +118,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalInt(maxPageSearchSize); out.writeOptionalFloat(docsPerSecond); if (out.getVersion().onOrAfter(Version.V_8_0_0)) { // todo: change to V_7_11_0 - out.writeOptionalInt(writeDateAsEpochMillis); + out.writeOptionalInt(datesAsEpochMillis); } } @@ -132,8 +132,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (docsPerSecond != null && (docsPerSecond.equals(DEFAULT_DOCS_PER_SECOND) == false)) { builder.field(TransformField.DOCS_PER_SECOND.getPreferredName(), docsPerSecond); } - if (writeDateAsEpochMillis != null && (writeDateAsEpochMillis.equals(DEFAULT_WRITE_DATE_AS_EPOCH_MILLIS) == false)) { - builder.field(TransformField.WRITE_DATE_AS_EPOCH_MILLIS.getPreferredName(), writeDateAsEpochMillis > 0 ? true : false); + if (datesAsEpochMillis != null && (datesAsEpochMillis.equals(DEFAULT_WRITE_DATE_AS_EPOCH_MILLIS) == false)) { + builder.field(TransformField.DATES_AS_EPOCH_MILLIS.getPreferredName(), datesAsEpochMillis > 0 ? true : false); } builder.endObject(); return builder; @@ -151,17 +151,17 @@ public boolean equals(Object other) { SettingsConfig that = (SettingsConfig) other; return Objects.equals(maxPageSearchSize, that.maxPageSearchSize) && Objects.equals(docsPerSecond, that.docsPerSecond) - && Objects.equals(writeDateAsEpochMillis, that.writeDateAsEpochMillis); + && Objects.equals(datesAsEpochMillis, that.datesAsEpochMillis); } @Override public int hashCode() { - return Objects.hash(maxPageSearchSize, docsPerSecond, writeDateAsEpochMillis); + return Objects.hash(maxPageSearchSize, docsPerSecond, datesAsEpochMillis); } @Override public String toString() { - return Strings.toString(this, true, true) + " wdaem: " + this.writeDateAsEpochMillis; + return Strings.toString(this, true, true) + " wdaem: " + this.datesAsEpochMillis; } public static SettingsConfig fromXContent(final XContentParser parser, boolean lenient) throws IOException { @@ -186,7 +186,7 @@ public Builder() {} public Builder(SettingsConfig base) { this.maxPageSearchSize = base.maxPageSearchSize; this.docsPerSecond = base.docsPerSecond; - this.writeDateAsEpochMilli = base.writeDateAsEpochMillis; + this.writeDateAsEpochMilli = base.datesAsEpochMillis; } /** @@ -225,11 +225,11 @@ public Builder setRequestsPerSecond(Float docsPerSecond) { * * An explicit `null` resets to default. * - * @param writeDateAsEpochMilli true if dates should be written as epoch_millis. + * @param datesAsEpochMilli true if dates should be written as epoch_millis. * @return the {@link Builder} with writeDateAsEpochMilli set. */ - public Builder setWriteDateAsEpochMilli(Boolean writeDateAsEpochMilli) { - this.writeDateAsEpochMilli = writeDateAsEpochMilli == null ? DEFAULT_WRITE_DATE_AS_EPOCH_MILLIS : writeDateAsEpochMilli ? 1 : 0; + public Builder setDatesAsEpochMilli(Boolean datesAsEpochMilli) { + this.writeDateAsEpochMilli = datesAsEpochMilli == null ? DEFAULT_WRITE_DATE_AS_EPOCH_MILLIS : datesAsEpochMilli ? 1 : 0; return this; } @@ -250,10 +250,10 @@ public Builder update(SettingsConfig update) { ? null : update.getMaxPageSearchSize(); } - if (update.getWriteDateAsEpochMillisForUpdate() != null) { - this.writeDateAsEpochMilli = update.getWriteDateAsEpochMillisForUpdate().equals(DEFAULT_WRITE_DATE_AS_EPOCH_MILLIS) + if (update.getDatesAsEpochMillisForUpdate() != null) { + this.writeDateAsEpochMilli = update.getDatesAsEpochMillisForUpdate().equals(DEFAULT_WRITE_DATE_AS_EPOCH_MILLIS) ? null - : update.getWriteDateAsEpochMillisForUpdate(); + : update.getDatesAsEpochMillisForUpdate(); } return this; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java index 355705b9b08b7..189a43142a2c1 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java @@ -473,10 +473,10 @@ public static TransformConfig rewriteForUpdate(final TransformConfig transformCo Integer maxPageSearchSize = transformConfig.getSettings().getMaxPageSearchSize() != null ? transformConfig.getSettings().getMaxPageSearchSize() : maxPageSearchSizeDeprecated; - Boolean writeDateAsEpochMillis = transformConfig.getSettings().getWriteDateAsEpochMillis(); + Boolean datesAsEpochMillis = transformConfig.getSettings().getDatesAsEpochMillis(); builder.setSettings( - new SettingsConfig(maxPageSearchSize, transformConfig.getSettings().getDocsPerSecond(), writeDateAsEpochMillis) + new SettingsConfig(maxPageSearchSize, transformConfig.getSettings().getDocsPerSecond(), datesAsEpochMillis) ); } @@ -488,10 +488,10 @@ public static TransformConfig rewriteForUpdate(final TransformConfig transformCo Integer maxPageSearchSize = transformConfig.getSettings().getMaxPageSearchSize() != null ? transformConfig.getSettings().getMaxPageSearchSize() : maxPageSearchSizeDeprecated; - Boolean writeDateAsEpochMillis = true; + Boolean datesAsEpochMillis = true; builder.setSettings( - new SettingsConfig(maxPageSearchSize, transformConfig.getSettings().getDocsPerSecond(), writeDateAsEpochMillis) + new SettingsConfig(maxPageSearchSize, transformConfig.getSettings().getDocsPerSecond(), datesAsEpochMillis) ); } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/SettingsConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/SettingsConfigTests.java index 3439bba3a61cf..b315b3098ca92 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/SettingsConfigTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/SettingsConfigTests.java @@ -74,31 +74,29 @@ public void testExplicitNullParsing() throws IOException { assertThat(fromString("{\"docs_per_second\" : null}").getDocsPerSecond(), equalTo(-1F)); assertNull(fromString("{}").getDocsPerSecond()); - assertThat(fromString("{\"write_date_as_epoch_millis\" : null}").getWriteDateAsEpochMillisForUpdate(), equalTo(-1)); - assertNull(fromString("{}").getWriteDateAsEpochMillisForUpdate()); + assertThat(fromString("{\"dates_as_epoch_millis\" : null}").getDatesAsEpochMillisForUpdate(), equalTo(-1)); + assertNull(fromString("{}").getDatesAsEpochMillisForUpdate()); } public void testUpdateUsingBuilder() throws IOException { - SettingsConfig config = fromString( - "{\"max_page_search_size\" : 10000, \"docs_per_second\" :42, \"write_date_as_epoch_millis\": true}" - ); + SettingsConfig config = fromString("{\"max_page_search_size\" : 10000, \"docs_per_second\" :42, \"dates_as_epoch_millis\": true}"); SettingsConfig.Builder builder = new SettingsConfig.Builder(config); builder.update(fromString("{\"max_page_search_size\" : 100}")); assertThat(builder.build().getMaxPageSearchSize(), equalTo(100)); assertThat(builder.build().getDocsPerSecond(), equalTo(42F)); - assertThat(builder.build().getWriteDateAsEpochMillisForUpdate(), equalTo(1)); + assertThat(builder.build().getDatesAsEpochMillisForUpdate(), equalTo(1)); builder.update(fromString("{\"max_page_search_size\" : null}")); assertNull(builder.build().getMaxPageSearchSize()); assertThat(builder.build().getDocsPerSecond(), equalTo(42F)); - assertThat(builder.build().getWriteDateAsEpochMillisForUpdate(), equalTo(1)); + assertThat(builder.build().getDatesAsEpochMillisForUpdate(), equalTo(1)); - builder.update(fromString("{\"max_page_search_size\" : 77, \"docs_per_second\" :null, \"write_date_as_epoch_millis\": null}")); + builder.update(fromString("{\"max_page_search_size\" : 77, \"docs_per_second\" :null, \"dates_as_epoch_millis\": null}")); assertThat(builder.build().getMaxPageSearchSize(), equalTo(77)); assertNull(builder.build().getDocsPerSecond()); - assertNull(builder.build().getWriteDateAsEpochMillisForUpdate()); + assertNull(builder.build().getDatesAsEpochMillisForUpdate()); } public void testOmmitDefaultsOnWriteParser() throws IOException { @@ -121,8 +119,8 @@ public void testOmmitDefaultsOnWriteParser() throws IOException { settingsAsMap = xContentToMap(config); assertTrue(settingsAsMap.isEmpty()); - config = fromString("{\"write_date_as_epoch_millis\" : null}"); - assertThat(config.getWriteDateAsEpochMillisForUpdate(), equalTo(-1)); + config = fromString("{\"dates_as_epoch_millis\" : null}"); + assertThat(config.getDatesAsEpochMillisForUpdate(), equalTo(-1)); settingsAsMap = xContentToMap(config); assertTrue(settingsAsMap.isEmpty()); @@ -148,8 +146,8 @@ public void testOmmitDefaultsOnWriteBuilder() throws IOException { settingsAsMap = xContentToMap(config); assertTrue(settingsAsMap.isEmpty()); - config = new SettingsConfig.Builder().setWriteDateAsEpochMilli(null).build(); - assertThat(config.getWriteDateAsEpochMillisForUpdate(), equalTo(-1)); + config = new SettingsConfig.Builder().setDatesAsEpochMilli(null).build(); + assertThat(config.getDatesAsEpochMillisForUpdate(), equalTo(-1)); settingsAsMap = xContentToMap(config); assertTrue(settingsAsMap.isEmpty()); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigTests.java index 66ccfa591bf1c..c2b60a3d7fa87 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigTests.java @@ -414,16 +414,16 @@ public void testRewriteForBWCofDateNormalization() throws IOException { TransformConfig transformConfig = createTransformConfigFromString(pivotTransform, "body_id", true); TransformConfig transformConfigRewritten = TransformConfig.rewriteForUpdate(transformConfig); - assertTrue(transformConfigRewritten.getSettings().getWriteDateAsEpochMillis()); + assertTrue(transformConfigRewritten.getSettings().getDatesAsEpochMillis()); assertEquals(Version.CURRENT, transformConfigRewritten.getVersion()); TransformConfig explicitTrueAfter711 = new TransformConfig.Builder(transformConfig).setSettings( - new SettingsConfig.Builder(transformConfigRewritten.getSettings()).setWriteDateAsEpochMilli(true).build() + new SettingsConfig.Builder(transformConfigRewritten.getSettings()).setDatesAsEpochMilli(true).build() ).setVersion(Version.V_8_0_0).build(); // todo: V_7_11_0 transformConfigRewritten = TransformConfig.rewriteForUpdate(explicitTrueAfter711); - assertTrue(transformConfigRewritten.getSettings().getWriteDateAsEpochMillis()); + assertTrue(transformConfigRewritten.getSettings().getDatesAsEpochMillis()); assertEquals(Version.V_8_0_0, transformConfigRewritten.getVersion()); // todo: V_7_11_0 } diff --git a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/DateHistogramGroupByIT.java b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/DateHistogramGroupByIT.java index b3ffe772e7798..274d8c1b29444 100644 --- a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/DateHistogramGroupByIT.java +++ b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/DateHistogramGroupByIT.java @@ -55,7 +55,7 @@ public TransformConfig createConfig() { TransformConfig.Builder transformConfigBuilder = new TransformConfig.Builder(); addCommonBuilderParameters(transformConfigBuilder); if (dateAsEpochMillis) { - transformConfigBuilder.setSettings(addCommonSetings(new SettingsConfig.Builder()).setWriteDateAsEpochMilli(true).build()); + transformConfigBuilder.setSettings(addCommonSetings(new SettingsConfig.Builder()).setDatesAsEpochMilli(true).build()); } transformConfigBuilder.setSource(new SourceConfig(CONTINUOUS_EVENTS_SOURCE_INDEX)); diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/Pivot.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/Pivot.java index 18cc016baf4f4..744825f2e2b6f 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/Pivot.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/Pivot.java @@ -227,7 +227,7 @@ public Stream> extractResults( // < 7.11 as epoch millis // >= 7.11 as string // note: it depends on the version when the transform has been created, not the version of the code - boolean datesAsEpoch = settings.getWriteDateAsEpochMillis() != null ? settings.getWriteDateAsEpochMillis() + boolean datesAsEpoch = settings.getDatesAsEpochMillis() != null ? settings.getDatesAsEpochMillis() : version.onOrAfter(Version.V_8_0_0) ? false // todo V_7_11_0 : true; From 2fd4c51b4cd8f689f32726f40fce19072c01b833 Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Tue, 1 Dec 2020 12:00:42 +0100 Subject: [PATCH 06/15] fix leftover renamings --- .../transform/transforms/SettingsConfig.java | 8 ++++---- .../transform/apis/put-transform.asciidoc | 2 +- .../transform/apis/update-transform.asciidoc | 2 +- .../transform/transforms/SettingsConfig.java | 20 +++++++++---------- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/SettingsConfig.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/SettingsConfig.java index f32aaf1a1838d..246a0e1ab1538 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/SettingsConfig.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/SettingsConfig.java @@ -141,7 +141,7 @@ public static Builder builder() { public static class Builder { private Integer maxPageSearchSize; private Float docsPerSecond; - private Integer writeDateAsEpochMilli; + private Integer datesAsEpochMilli; /** * Sets the paging maximum paging maxPageSearchSize that transform can use when @@ -180,15 +180,15 @@ public Builder setRequestsPerSecond(Float docsPerSecond) { * An explicit `null` resets to default. * * @param datesAsEpochMilli true if dates should be written as epoch_millis. - * @return the {@link Builder} with writeDateAsEpochMilli set. + * @return the {@link Builder} with datesAsEpochMilli set. */ public Builder setDatesAsEpochMilli(Boolean datesAsEpochMilli) { - this.writeDateAsEpochMilli = datesAsEpochMilli == null ? DEFAULT_DATES_AS_EPOCH_MILLIS : datesAsEpochMilli ? 1 : 0; + this.datesAsEpochMilli = datesAsEpochMilli == null ? DEFAULT_DATES_AS_EPOCH_MILLIS : datesAsEpochMilli ? 1 : 0; return this; } public SettingsConfig build() { - return new SettingsConfig(maxPageSearchSize, docsPerSecond, writeDateAsEpochMilli); + return new SettingsConfig(maxPageSearchSize, docsPerSecond, datesAsEpochMilli); } } } diff --git a/docs/reference/transform/apis/put-transform.asciidoc b/docs/reference/transform/apis/put-transform.asciidoc index e03c45d8a2889..c807bd3b378f5 100644 --- a/docs/reference/transform/apis/put-transform.asciidoc +++ b/docs/reference/transform/apis/put-transform.asciidoc @@ -142,7 +142,7 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-doc include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-max-page-search-size] `dates_as_epoch_millis`::: (Optional, boolean) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-write-date-as-epoch-milli] +include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-write-dates-as-epoch-milli] ==== //End settings diff --git a/docs/reference/transform/apis/update-transform.asciidoc b/docs/reference/transform/apis/update-transform.asciidoc index f4f64465af04c..e31ec381d937c 100644 --- a/docs/reference/transform/apis/update-transform.asciidoc +++ b/docs/reference/transform/apis/update-transform.asciidoc @@ -118,7 +118,7 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-doc include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-max-page-search-size] `dates_as_epoch_millis`::: (Optional, boolean) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-write-date-as-epoch-milli] +include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-write-dates-as-epoch-milli] ==== //End settings diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/SettingsConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/SettingsConfig.java index 4b8d44f19586f..78d3f3cc4fa00 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/SettingsConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/SettingsConfig.java @@ -31,7 +31,7 @@ public class SettingsConfig implements Writeable, ToXContentObject { private static final int DEFAULT_MAX_PAGE_SEARCH_SIZE = -1; private static final float DEFAULT_DOCS_PER_SECOND = -1F; - private static final int DEFAULT_WRITE_DATE_AS_EPOCH_MILLIS = -1; + private static final int DEFAULT_DATES_AS_EPOCH_MILLIS = -1; private static ConstructingObjectParser createParser(boolean lenient) { ConstructingObjectParser parser = new ConstructingObjectParser<>( @@ -44,7 +44,7 @@ private static ConstructingObjectParser createParser(boole // this boolean requires 4 possible values: true, false, not_specified, default, therefore using a custom parser parser.declareField( optionalConstructorArg(), - p -> p.currentToken() == XContentParser.Token.VALUE_NULL ? DEFAULT_WRITE_DATE_AS_EPOCH_MILLIS : p.booleanValue() ? 1 : 0, + p -> p.currentToken() == XContentParser.Token.VALUE_NULL ? DEFAULT_DATES_AS_EPOCH_MILLIS : p.booleanValue() ? 1 : 0, TransformField.DATES_AS_EPOCH_MILLIS, ValueType.BOOLEAN_OR_NULL ); @@ -77,7 +77,7 @@ public SettingsConfig(final StreamInput in) throws IOException { if (in.getVersion().onOrAfter(Version.V_8_0_0)) { // todo: change to V_7_11 this.datesAsEpochMillis = in.readOptionalInt(); } else { - this.datesAsEpochMillis = DEFAULT_WRITE_DATE_AS_EPOCH_MILLIS; + this.datesAsEpochMillis = DEFAULT_DATES_AS_EPOCH_MILLIS; } } @@ -132,7 +132,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (docsPerSecond != null && (docsPerSecond.equals(DEFAULT_DOCS_PER_SECOND) == false)) { builder.field(TransformField.DOCS_PER_SECOND.getPreferredName(), docsPerSecond); } - if (datesAsEpochMillis != null && (datesAsEpochMillis.equals(DEFAULT_WRITE_DATE_AS_EPOCH_MILLIS) == false)) { + if (datesAsEpochMillis != null && (datesAsEpochMillis.equals(DEFAULT_DATES_AS_EPOCH_MILLIS) == false)) { builder.field(TransformField.DATES_AS_EPOCH_MILLIS.getPreferredName(), datesAsEpochMillis > 0 ? true : false); } builder.endObject(); @@ -171,7 +171,7 @@ public static SettingsConfig fromXContent(final XContentParser parser, boolean l public static class Builder { private Integer maxPageSearchSize; private Float docsPerSecond; - private Integer writeDateAsEpochMilli; + private Integer dateAsEpochMilli; /** * Default builder @@ -186,7 +186,7 @@ public Builder() {} public Builder(SettingsConfig base) { this.maxPageSearchSize = base.maxPageSearchSize; this.docsPerSecond = base.docsPerSecond; - this.writeDateAsEpochMilli = base.datesAsEpochMillis; + this.dateAsEpochMilli = base.datesAsEpochMillis; } /** @@ -226,10 +226,10 @@ public Builder setRequestsPerSecond(Float docsPerSecond) { * An explicit `null` resets to default. * * @param datesAsEpochMilli true if dates should be written as epoch_millis. - * @return the {@link Builder} with writeDateAsEpochMilli set. + * @return the {@link Builder} with datesAsEpochMilli set. */ public Builder setDatesAsEpochMilli(Boolean datesAsEpochMilli) { - this.writeDateAsEpochMilli = datesAsEpochMilli == null ? DEFAULT_WRITE_DATE_AS_EPOCH_MILLIS : datesAsEpochMilli ? 1 : 0; + this.dateAsEpochMilli = datesAsEpochMilli == null ? DEFAULT_DATES_AS_EPOCH_MILLIS : datesAsEpochMilli ? 1 : 0; return this; } @@ -251,7 +251,7 @@ public Builder update(SettingsConfig update) { : update.getMaxPageSearchSize(); } if (update.getDatesAsEpochMillisForUpdate() != null) { - this.writeDateAsEpochMilli = update.getDatesAsEpochMillisForUpdate().equals(DEFAULT_WRITE_DATE_AS_EPOCH_MILLIS) + this.dateAsEpochMilli = update.getDatesAsEpochMillisForUpdate().equals(DEFAULT_DATES_AS_EPOCH_MILLIS) ? null : update.getDatesAsEpochMillisForUpdate(); } @@ -260,7 +260,7 @@ public Builder update(SettingsConfig update) { } public SettingsConfig build() { - return new SettingsConfig(maxPageSearchSize, docsPerSecond, writeDateAsEpochMilli); + return new SettingsConfig(maxPageSearchSize, docsPerSecond, dateAsEpochMilli); } } } From a3a7646b652a5514729cf6852d1b4000860e6d65 Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Tue, 1 Dec 2020 12:02:48 +0100 Subject: [PATCH 07/15] fix test --- .../transform/transforms/SettingsConfigTests.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/SettingsConfigTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/SettingsConfigTests.java index 9b4be5ca19449..0db44eda059fa 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/SettingsConfigTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/SettingsConfigTests.java @@ -82,15 +82,15 @@ public void testExplicitNullOnWriteParser() throws IOException { settingsAsMap = xContentToMap(config); assertThat(settingsAsMap.getOrDefault("max_page_search_size", "not_set"), equalTo("not_set")); assertNull(settingsAsMap.getOrDefault("docs_per_second", "not_set")); - assertThat(settingsAsMap.getOrDefault("write_date_as_epoch_millis", "not_set"), equalTo("not_set")); + assertThat(settingsAsMap.getOrDefault("dates_as_epoch_millis", "not_set"), equalTo("not_set")); - config = fromString("{\"write_date_as_epoch_millis\" : null}"); + config = fromString("{\"dates_as_epoch_millis\" : null}"); assertNull(emptyConfig.getDatesAsEpochMillis()); settingsAsMap = xContentToMap(config); assertThat(settingsAsMap.getOrDefault("max_page_search_size", "not_set"), equalTo("not_set")); assertThat(settingsAsMap.getOrDefault("docs_per_second", "not_set"), equalTo("not_set")); - assertNull(settingsAsMap.getOrDefault("write_date_as_epoch_millis", "not_set")); + assertNull(settingsAsMap.getOrDefault("dates_as_epoch_millis", "not_set")); } public void testExplicitNullOnWriteBuilder() throws IOException { @@ -101,7 +101,7 @@ public void testExplicitNullOnWriteBuilder() throws IOException { Map settingsAsMap = xContentToMap(config); assertNull(settingsAsMap.getOrDefault("max_page_search_size", "not_set")); assertThat(settingsAsMap.getOrDefault("docs_per_second", "not_set"), equalTo("not_set")); - assertThat(settingsAsMap.getOrDefault("write_date_as_epoch_millis", "not_set"), equalTo("not_set")); + assertThat(settingsAsMap.getOrDefault("dates_as_epoch_millis", "not_set"), equalTo("not_set")); SettingsConfig emptyConfig = new SettingsConfig.Builder().build(); assertNull(emptyConfig.getMaxPageSearchSize()); @@ -116,7 +116,7 @@ public void testExplicitNullOnWriteBuilder() throws IOException { settingsAsMap = xContentToMap(config); assertThat(settingsAsMap.getOrDefault("max_page_search_size", "not_set"), equalTo("not_set")); assertNull(settingsAsMap.getOrDefault("docs_per_second", "not_set")); - assertThat(settingsAsMap.getOrDefault("write_date_as_epoch_millis", "not_set"), equalTo("not_set")); + assertThat(settingsAsMap.getOrDefault("dates_as_epoch_millis", "not_set"), equalTo("not_set")); config = new SettingsConfig.Builder().setDatesAsEpochMilli(null).build(); // returns null, however it's `null` as in "use default" @@ -125,7 +125,7 @@ public void testExplicitNullOnWriteBuilder() throws IOException { settingsAsMap = xContentToMap(config); assertThat(settingsAsMap.getOrDefault("max_page_search_size", "not_set"), equalTo("not_set")); assertThat(settingsAsMap.getOrDefault("docs_per_second", "not_set"), equalTo("not_set")); - assertNull(settingsAsMap.getOrDefault("write_date_as_epoch_millis", "not_set")); + assertNull(settingsAsMap.getOrDefault("dates_as_epoch_millis", "not_set")); } private Map xContentToMap(ToXContent xcontent) throws IOException { From c3befde0a9a6b981e714d020503cf389b4fa9977 Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Tue, 1 Dec 2020 12:10:40 +0100 Subject: [PATCH 08/15] fix docs --- docs/reference/transform/apis/put-transform.asciidoc | 2 +- docs/reference/transform/apis/update-transform.asciidoc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/reference/transform/apis/put-transform.asciidoc b/docs/reference/transform/apis/put-transform.asciidoc index c807bd3b378f5..3d5b648f0a0bd 100644 --- a/docs/reference/transform/apis/put-transform.asciidoc +++ b/docs/reference/transform/apis/put-transform.asciidoc @@ -142,7 +142,7 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-doc include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-max-page-search-size] `dates_as_epoch_millis`::: (Optional, boolean) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-write-dates-as-epoch-milli] +include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-dates-as-epoch-milli] ==== //End settings diff --git a/docs/reference/transform/apis/update-transform.asciidoc b/docs/reference/transform/apis/update-transform.asciidoc index e31ec381d937c..3d4759e7a2ef9 100644 --- a/docs/reference/transform/apis/update-transform.asciidoc +++ b/docs/reference/transform/apis/update-transform.asciidoc @@ -118,7 +118,7 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-doc include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-max-page-search-size] `dates_as_epoch_millis`::: (Optional, boolean) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-write-dates-as-epoch-milli] +include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-dates-as-epoch-milli] ==== //End settings From f06088fafd308ff4af1ccf5446ed4b3d55c9f754 Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Fri, 4 Dec 2020 11:08:32 +0100 Subject: [PATCH 09/15] address review comments --- .../transforms/SettingsConfigTests.java | 6 +- docs/reference/rest-api/common-parms.asciidoc | 4 +- .../transform/transforms/TransformConfig.java | 117 ++++++++++++------ .../transforms/TransformConfigTests.java | 6 +- .../continuous/DateHistogramGroupByIT.java | 8 +- .../pivot/AggregationResultUtils.java | 6 +- 6 files changed, 95 insertions(+), 52 deletions(-) diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/SettingsConfigTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/SettingsConfigTests.java index 0db44eda059fa..9945d2575f4d0 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/SettingsConfigTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/SettingsConfigTests.java @@ -85,7 +85,7 @@ public void testExplicitNullOnWriteParser() throws IOException { assertThat(settingsAsMap.getOrDefault("dates_as_epoch_millis", "not_set"), equalTo("not_set")); config = fromString("{\"dates_as_epoch_millis\" : null}"); - assertNull(emptyConfig.getDatesAsEpochMillis()); + assertFalse(config.getDatesAsEpochMillis()); settingsAsMap = xContentToMap(config); assertThat(settingsAsMap.getOrDefault("max_page_search_size", "not_set"), equalTo("not_set")); @@ -119,8 +119,8 @@ public void testExplicitNullOnWriteBuilder() throws IOException { assertThat(settingsAsMap.getOrDefault("dates_as_epoch_millis", "not_set"), equalTo("not_set")); config = new SettingsConfig.Builder().setDatesAsEpochMilli(null).build(); - // returns null, however it's `null` as in "use default" - assertNull(emptyConfig.getDatesAsEpochMillis()); + // returns false, however it's `null` as in "use default", checked next + assertFalse(config.getDatesAsEpochMillis()); settingsAsMap = xContentToMap(config); assertThat(settingsAsMap.getOrDefault("max_page_search_size", "not_set"), equalTo("not_set")); diff --git a/docs/reference/rest-api/common-parms.asciidoc b/docs/reference/rest-api/common-parms.asciidoc index be427290311d3..79d3a248ee7f2 100644 --- a/docs/reference/rest-api/common-parms.asciidoc +++ b/docs/reference/rest-api/common-parms.asciidoc @@ -966,9 +966,9 @@ The default value is `500`. end::transform-settings-max-page-search-size[] tag::transform-settings-dates-as-epoch-milli[] -Defines if dates in the ouput should be written as ISO formated string (default) +Defines if dates in the ouput should be written as ISO formatted string (default) or as millis since epoch. `epoch_millis` has been the default for transforms created -before version `7.11`. For compatible output set this to true. +before version `7.11`. For compatible output set this to `true`. The default value is `false`. end::transform-settings-dates-as-epoch-milli[] diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java index 189a43142a2c1..7c28b8eb305ca 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java @@ -440,15 +440,20 @@ public static TransformConfig fromXContent(final XContentParser parser, @Nullabl } /** - * Rewrites the transform config according to the latest format, for example moving deprecated - * settings to its new place. + * Rewrites the transform config according to the latest format. + * + * Operations cover: + * + * - move deprecated settings to its new place + * - change configuration options so it stays compatible (given a newer version) * * @param transformConfig original config * @return a rewritten transform config if a rewrite was necessary, otherwise the given transformConfig */ public static TransformConfig rewriteForUpdate(final TransformConfig transformConfig) { - // quick checks for deprecated features, if none found just return the original + // quick checks(optimization) if a rewrite is required, if none found just return the original + // a failing quick check, does not mean a rewrite is necessary if (transformConfig.getVersion() != null && transformConfig.getVersion().onOrAfter(Version.V_8_0_0) // todo: V_7_11_0 && (transformConfig.getPivotConfig() == null || transformConfig.getPivotConfig().getMaxPageSearchSize() == null)) { @@ -457,42 +462,37 @@ public static TransformConfig rewriteForUpdate(final TransformConfig transformCo Builder builder = new Builder(transformConfig); - /* - * Move pivot.max_page_size_search to settings.max_page_size_search - */ - if (transformConfig.getPivotConfig() != null && transformConfig.getPivotConfig().getMaxPageSearchSize() != null) { - // create a new pivot config but set maxPageSearchSize to null - PivotConfig newPivotConfig = new PivotConfig( - transformConfig.getPivotConfig().getGroupConfig(), - transformConfig.getPivotConfig().getAggregationConfig(), - null - ); - builder.setPivotConfig(newPivotConfig); - - Integer maxPageSearchSizeDeprecated = transformConfig.getPivotConfig().getMaxPageSearchSize(); - Integer maxPageSearchSize = transformConfig.getSettings().getMaxPageSearchSize() != null - ? transformConfig.getSettings().getMaxPageSearchSize() - : maxPageSearchSizeDeprecated; - Boolean datesAsEpochMillis = transformConfig.getSettings().getDatesAsEpochMillis(); - - builder.setSettings( - new SettingsConfig(maxPageSearchSize, transformConfig.getSettings().getDocsPerSecond(), datesAsEpochMillis) - ); - } - - /* - * Keep date normalization backwards compatible for transforms created before 7.11 - */ - if (transformConfig.getVersion() != null && transformConfig.getVersion().before(Version.V_8_0_0)) { // todo: V_7_11_0 - Integer maxPageSearchSizeDeprecated = transformConfig.getPivotConfig().getMaxPageSearchSize(); - Integer maxPageSearchSize = transformConfig.getSettings().getMaxPageSearchSize() != null - ? transformConfig.getSettings().getMaxPageSearchSize() - : maxPageSearchSizeDeprecated; - Boolean datesAsEpochMillis = true; + // call apply rewrite without config, to only allow reading from the builder + return applyRewriteForUpdate(builder); + } - builder.setSettings( - new SettingsConfig(maxPageSearchSize, transformConfig.getSettings().getDocsPerSecond(), datesAsEpochMillis) - ); + private static TransformConfig applyRewriteForUpdate(Builder builder) { + // 1. Move pivot.max_page_size_search to settings.max_page_size_search + if (builder.getPivotConfig() != null && builder.getPivotConfig().getMaxPageSearchSize() != null) { + + // find maxPageSearchSize value + Integer maxPageSearchSizeDeprecated = builder.getPivotConfig().getMaxPageSearchSize(); + Integer maxPageSearchSize = builder.getSettings().getMaxPageSearchSize() != null + ? builder.getSettings().getMaxPageSearchSize() + : maxPageSearchSizeDeprecated; + + // create a new pivot config but set maxPageSearchSize to null + builder.setPivotConfig( + new PivotConfig(builder.getPivotConfig().getGroupConfig(), builder.getPivotConfig().getAggregationConfig(), null) + ); + // create new settings with maxPageSearchSize + builder.setSettings( + new SettingsConfig( + maxPageSearchSize, + builder.getSettings().getDocsPerSecond(), + builder.getSettings().getDatesAsEpochMillis() + ) + ); + } + + // 2. set dates_as_epoch_millis to true for transforms < 7.11 to keep BWC + if (builder.getVersion() != null && builder.getVersion().before(Version.V_8_0_0)) { // todo: V_7_11_0 + builder.setSettings(new SettingsConfig(builder.getSettings().getMaxPageSearchSize(), builder.getSettings().getDocsPerSecond(), true)); } return builder.setVersion(Version.CURRENT).build(); @@ -531,51 +531,92 @@ public Builder setId(String id) { return this; } + String getId() { + return id; + } + public Builder setSource(SourceConfig source) { this.source = source; return this; } + SourceConfig getSource() { + return source; + } + public Builder setDest(DestConfig dest) { this.dest = dest; return this; } + DestConfig getDest() { + return dest; + } + public Builder setFrequency(TimeValue frequency) { this.frequency = frequency; return this; } + TimeValue getFrequency() { + return frequency; + } + public Builder setSyncConfig(SyncConfig syncConfig) { this.syncConfig = syncConfig; return this; } + SyncConfig getSyncConfig() { + return syncConfig; + } + public Builder setDescription(String description) { this.description = description; return this; } + String getDescription() { + return description; + } + public Builder setSettings(SettingsConfig settings) { this.settings = settings; return this; } + SettingsConfig getSettings() { + return settings; + } + public Builder setHeaders(Map headers) { this.headers = headers; return this; } + public Map getHeaders() + { + return headers; + } + public Builder setPivotConfig(PivotConfig pivotConfig) { this.pivotConfig = pivotConfig; return this; } + PivotConfig getPivotConfig() { + return pivotConfig; + } + Builder setVersion(Version version) { this.transformVersion = version; return this; } + Version getVersion() { + return transformVersion; + } + public TransformConfig build() { return new TransformConfig( id, diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigTests.java index c2b60a3d7fa87..3320b5f7c9098 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigTests.java @@ -350,11 +350,13 @@ public void testRewriteForUpdate() throws IOException { assertNull(transformConfigRewritten.getPivotConfig().getMaxPageSearchSize()); assertNotNull(transformConfigRewritten.getSettings().getMaxPageSearchSize()); assertEquals(111L, transformConfigRewritten.getSettings().getMaxPageSearchSize().longValue()); + assertTrue(transformConfigRewritten.getSettings().getDatesAsEpochMillis()); + assertWarnings("[max_page_search_size] is deprecated inside pivot please use settings instead"); assertEquals(Version.CURRENT, transformConfigRewritten.getVersion()); } - public void testRewriteForUpdateConflicting() throws IOException { + public void testRewriteForUpdateMaxPageSizeSearchConflicting() throws IOException { String pivotTransform = "{" + " \"id\" : \"body_id\"," + " \"source\" : {\"index\":\"src\"}," @@ -389,7 +391,7 @@ public void testRewriteForUpdateConflicting() throws IOException { assertWarnings("[max_page_search_size] is deprecated inside pivot please use settings instead"); } - public void testRewriteForBWCofDateNormalization() throws IOException { + public void testRewriteForBWCOfDateNormalization() throws IOException { String pivotTransform = "{" + " \"id\" : \"body_id\"," + " \"source\" : {\"index\":\"src\"}," diff --git a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/DateHistogramGroupByIT.java b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/DateHistogramGroupByIT.java index 274d8c1b29444..6dec6ce8260c8 100644 --- a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/DateHistogramGroupByIT.java +++ b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/DateHistogramGroupByIT.java @@ -43,18 +43,18 @@ public class DateHistogramGroupByIT extends ContinuousTestCase { .format(Instant.ofEpochMilli(42)); private final boolean missing; - private final boolean dateAsEpochMillis; + private final boolean datesAsEpochMillis; public DateHistogramGroupByIT() { missing = randomBoolean(); - dateAsEpochMillis = randomBoolean(); + datesAsEpochMillis = randomBoolean(); } @Override public TransformConfig createConfig() { TransformConfig.Builder transformConfigBuilder = new TransformConfig.Builder(); addCommonBuilderParameters(transformConfigBuilder); - if (dateAsEpochMillis) { + if (datesAsEpochMillis) { transformConfigBuilder.setSettings(addCommonSetings(new SettingsConfig.Builder()).setDatesAsEpochMilli(true).build()); } @@ -117,7 +117,7 @@ public void testIteration(int iteration) throws IOException { Map source = searchHit.getSourceAsMap(); String transformBucketKey; - if (dateAsEpochMillis) { + if (datesAsEpochMillis) { transformBucketKey = ContinuousTestCase.STRICT_DATE_OPTIONAL_TIME_PRINTER_NANOS.withZone(ZoneId.of("UTC")) .format(Instant.ofEpochMilli((Long) XContentMapValues.extractValue("second", source))); } else { diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/AggregationResultUtils.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/AggregationResultUtils.java index 20103d9a1d3db..b9ca06a5b9a25 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/AggregationResultUtils.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/AggregationResultUtils.java @@ -65,7 +65,7 @@ public final class AggregationResultUtils { private static final Map BUCKET_KEY_EXTRACTOR_MAP; private static final BucketKeyExtractor DEFAULT_BUCKET_KEY_EXTRACTOR = new DefaultBucketKeyExtractor(); - private static final BucketKeyExtractor DATE_AS_EPOCH_BUCKET_KEY_EXTRACTOR = new DateAsEpochBucketKeyExtractor(); + private static final BucketKeyExtractor DATES_AS_EPOCH_BUCKET_KEY_EXTRACTOR = new DatesAsEpochBucketKeyExtractor(); static { Map tempMap = new HashMap<>(); @@ -135,7 +135,7 @@ public static Stream> extractCompositeAggregationResults( static BucketKeyExtractor getBucketKeyExtractor(SingleGroupSource groupSource, boolean datesAsEpoch) { return BUCKET_KEY_EXTRACTOR_MAP.getOrDefault( groupSource.getClass().getName(), - datesAsEpoch ? DATE_AS_EPOCH_BUCKET_KEY_EXTRACTOR : DEFAULT_BUCKET_KEY_EXTRACTOR + datesAsEpoch ? DATES_AS_EPOCH_BUCKET_KEY_EXTRACTOR : DEFAULT_BUCKET_KEY_EXTRACTOR ); } @@ -430,7 +430,7 @@ public Object value(Object key, String type) { } - static class DateAsEpochBucketKeyExtractor implements BucketKeyExtractor { + static class DatesAsEpochBucketKeyExtractor implements BucketKeyExtractor { @Override public Object value(Object key, String type) { From d59abcda114a2c929a91abc4bdb169d01895e7bf Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Fri, 4 Dec 2020 12:59:44 +0100 Subject: [PATCH 10/15] add comment about date_nanos --- .../transform/transforms/pivot/AggregationResultUtils.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/AggregationResultUtils.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/AggregationResultUtils.java index b9ca06a5b9a25..1a9dca744923f 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/AggregationResultUtils.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/pivot/AggregationResultUtils.java @@ -422,6 +422,8 @@ public Object value(Object key, String type) { return dropFloatingPointComponentIfTypeRequiresIt(type, (Double) key); } else if ((DateFieldMapper.CONTENT_TYPE.equals(type) || DateFieldMapper.DATE_NANOS_CONTENT_TYPE.equals(type)) && key instanceof Long) { + // date_histogram return bucket keys with milliseconds since epoch precision, therefore we don't need a + // nanosecond formatter, for the parser on indexing side, time is optional (only the date part is mandatory) return DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.formatMillis((Long) key); } From bc2575780304de3a6440bb968d36c462f4c20d08 Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Fri, 4 Dec 2020 13:07:06 +0100 Subject: [PATCH 11/15] checkstyle --- .../transform/transforms/TransformConfig.java | 51 ++++++++++--------- .../transforms/TransformIndexer.java | 6 +-- 2 files changed, 29 insertions(+), 28 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java index 7c28b8eb305ca..1ca611998dd6a 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java @@ -467,32 +467,34 @@ public static TransformConfig rewriteForUpdate(final TransformConfig transformCo } private static TransformConfig applyRewriteForUpdate(Builder builder) { - // 1. Move pivot.max_page_size_search to settings.max_page_size_search - if (builder.getPivotConfig() != null && builder.getPivotConfig().getMaxPageSearchSize() != null) { - - // find maxPageSearchSize value - Integer maxPageSearchSizeDeprecated = builder.getPivotConfig().getMaxPageSearchSize(); - Integer maxPageSearchSize = builder.getSettings().getMaxPageSearchSize() != null - ? builder.getSettings().getMaxPageSearchSize() - : maxPageSearchSizeDeprecated; - - // create a new pivot config but set maxPageSearchSize to null - builder.setPivotConfig( - new PivotConfig(builder.getPivotConfig().getGroupConfig(), builder.getPivotConfig().getAggregationConfig(), null) - ); - // create new settings with maxPageSearchSize - builder.setSettings( - new SettingsConfig( - maxPageSearchSize, - builder.getSettings().getDocsPerSecond(), - builder.getSettings().getDatesAsEpochMillis() - ) - ); - } + // 1. Move pivot.max_page_size_search to settings.max_page_size_search + if (builder.getPivotConfig() != null && builder.getPivotConfig().getMaxPageSearchSize() != null) { + + // find maxPageSearchSize value + Integer maxPageSearchSizeDeprecated = builder.getPivotConfig().getMaxPageSearchSize(); + Integer maxPageSearchSize = builder.getSettings().getMaxPageSearchSize() != null + ? builder.getSettings().getMaxPageSearchSize() + : maxPageSearchSizeDeprecated; + + // create a new pivot config but set maxPageSearchSize to null + builder.setPivotConfig( + new PivotConfig(builder.getPivotConfig().getGroupConfig(), builder.getPivotConfig().getAggregationConfig(), null) + ); + // create new settings with maxPageSearchSize + builder.setSettings( + new SettingsConfig( + maxPageSearchSize, + builder.getSettings().getDocsPerSecond(), + builder.getSettings().getDatesAsEpochMillis() + ) + ); + } // 2. set dates_as_epoch_millis to true for transforms < 7.11 to keep BWC if (builder.getVersion() != null && builder.getVersion().before(Version.V_8_0_0)) { // todo: V_7_11_0 - builder.setSettings(new SettingsConfig(builder.getSettings().getMaxPageSearchSize(), builder.getSettings().getDocsPerSecond(), true)); + builder.setSettings( + new SettingsConfig(builder.getSettings().getMaxPageSearchSize(), builder.getSettings().getDocsPerSecond(), true) + ); } return builder.setVersion(Version.CURRENT).build(); @@ -594,8 +596,7 @@ public Builder setHeaders(Map headers) { return this; } - public Map getHeaders() - { + public Map getHeaders() { return headers; } diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformIndexer.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformIndexer.java index bd916c6de8e86..e0df2b9194836 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformIndexer.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformIndexer.java @@ -1016,14 +1016,14 @@ private void configurePageSize(Integer newPageSize) { } private synchronized void startIndexerThreadShutdown() { - indexerThreadShuttingDown = true; + indexerThreadShuttingDown = true; stopCalledDuringIndexerThreadShutdown = false; } private synchronized void finishIndexerThreadShutdown() { - indexerThreadShuttingDown = false; + indexerThreadShuttingDown = false; if (stopCalledDuringIndexerThreadShutdown) { - doSaveState(IndexerState.STOPPED, getPosition(), () -> {}); + doSaveState(IndexerState.STOPPED, getPosition(), () -> {}); } } From 54024270d67c479471833d116b3ce6f4efca4096 Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Fri, 4 Dec 2020 14:44:01 +0100 Subject: [PATCH 12/15] add more tests --- .../DateHistogramGroupByOtherTimeFieldIT.java | 24 +++++++++++++++++-- .../pivot/AggregationResultUtilsTests.java | 21 ++++++++++++++++ 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/DateHistogramGroupByOtherTimeFieldIT.java b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/DateHistogramGroupByOtherTimeFieldIT.java index cdad2fe9232e0..2678e69a25939 100644 --- a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/DateHistogramGroupByOtherTimeFieldIT.java +++ b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/DateHistogramGroupByOtherTimeFieldIT.java @@ -4,6 +4,7 @@ import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.transform.transforms.DestConfig; +import org.elasticsearch.client.transform.transforms.SettingsConfig; import org.elasticsearch.client.transform.transforms.SourceConfig; import org.elasticsearch.client.transform.transforms.TransformConfig; import org.elasticsearch.client.transform.transforms.pivot.DateHistogramGroupSource; @@ -23,6 +24,8 @@ import org.elasticsearch.search.builder.SearchSourceBuilder; import java.io.IOException; +import java.time.Instant; +import java.time.ZoneId; import java.util.ArrayList; import java.util.HashMap; import java.util.Iterator; @@ -40,15 +43,20 @@ public class DateHistogramGroupByOtherTimeFieldIT extends ContinuousTestCase { private static final String NAME = "continuous-date-histogram-pivot-other-timefield-test"; private final boolean addGroupByTerms; + private final boolean datesAsEpochMillis; public DateHistogramGroupByOtherTimeFieldIT() { addGroupByTerms = randomBoolean(); + datesAsEpochMillis = randomBoolean(); } @Override public TransformConfig createConfig() { TransformConfig.Builder transformConfigBuilder = new TransformConfig.Builder(); addCommonBuilderParameters(transformConfigBuilder); + if (datesAsEpochMillis) { + transformConfigBuilder.setSettings(addCommonSetings(new SettingsConfig.Builder()).setDatesAsEpochMilli(true).build()); + } transformConfigBuilder.setSource(new SourceConfig(CONTINUOUS_EVENTS_SOURCE_INDEX)); transformConfigBuilder.setDest(new DestConfig(NAME, INGEST_PIPELINE)); transformConfigBuilder.setId(NAME); @@ -120,7 +128,13 @@ private void assertResultsGroupByDateHistogram(int iteration, SearchResponse res SearchHit searchHit = destIterator.next(); Map source = searchHit.getSourceAsMap(); - String transformBucketKey = (String) XContentMapValues.extractValue("second", source); + String transformBucketKey; + if (datesAsEpochMillis) { + transformBucketKey = ContinuousTestCase.STRICT_DATE_OPTIONAL_TIME_PRINTER_NANOS.withZone(ZoneId.of("UTC")) + .format(Instant.ofEpochMilli((Long) XContentMapValues.extractValue("second", source))); + } else { + transformBucketKey = (String) XContentMapValues.extractValue("second", source); + } // aggs return buckets with 0 doc_count while composite aggs skip over them while (bucket.getDocCount() == 0L) { @@ -185,7 +199,13 @@ private void assertResultsGroupByDateHistogramAndTerms(int iteration, SearchResp SearchHit searchHit = destIterator.next(); Map source = searchHit.getSourceAsMap(); - String transformBucketKey = (String) XContentMapValues.extractValue("second", source); + String transformBucketKey; + if (datesAsEpochMillis) { + transformBucketKey = ContinuousTestCase.STRICT_DATE_OPTIONAL_TIME_PRINTER_NANOS.withZone(ZoneId.of("UTC")) + .format(Instant.ofEpochMilli((Long) XContentMapValues.extractValue("second", source))); + } else { + transformBucketKey = (String) XContentMapValues.extractValue("second", source); + } // test correctness, the results from the aggregation and the results from the transform should be the same assertThat( diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/AggregationResultUtilsTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/AggregationResultUtilsTests.java index 27296f1765b6b..f1bc209e1ca47 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/AggregationResultUtilsTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/AggregationResultUtilsTests.java @@ -62,6 +62,7 @@ import org.elasticsearch.xpack.core.transform.TransformField; import org.elasticsearch.xpack.core.transform.transforms.TransformIndexerStats; import org.elasticsearch.xpack.core.transform.transforms.pivot.GroupConfig; +import org.elasticsearch.xpack.transform.transforms.pivot.AggregationResultUtils.BucketKeyExtractor; import java.io.IOException; import java.util.Arrays; @@ -871,6 +872,26 @@ public void testSingleBucketAggExtractor() { ); } + public void testDefaultBucketKeyExtractor() { + BucketKeyExtractor extractor = new AggregationResultUtils.DefaultBucketKeyExtractor(); + + assertThat(extractor.value(42.0, "long"), equalTo(42L)); + assertThat(extractor.value(42.2, "double"), equalTo(42.2)); + assertThat(extractor.value(1577836800000L, "date"), equalTo("2020-01-01T00:00:00.000Z")); + assertThat(extractor.value(1577836800000L, "date_nanos"), equalTo("2020-01-01T00:00:00.000Z")); + assertThat(extractor.value(1577836800000L, "long"), equalTo(1577836800000L)); + } + + public void testDatesAsEpochBucketKeyExtractor() { + BucketKeyExtractor extractor = new AggregationResultUtils.DatesAsEpochBucketKeyExtractor(); + + assertThat(extractor.value(42.0, "long"), equalTo(42L)); + assertThat(extractor.value(42.2, "double"), equalTo(42.2)); + assertThat(extractor.value(1577836800000L, "date"), equalTo(1577836800000L)); + assertThat(extractor.value(1577836800000L, "date_nanos"), equalTo(1577836800000L)); + assertThat(extractor.value(1577836800000L, "long"), equalTo(1577836800000L)); + } + private void executeTest( GroupConfig groups, Collection aggregationBuilders, From ee7103a30599baed6505eb0a596f53eadbb10ff8 Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Fri, 4 Dec 2020 17:22:57 +0100 Subject: [PATCH 13/15] fix order (alphabetical) --- docs/reference/rest-api/common-parms.asciidoc | 14 +++++++------- .../transform/apis/put-transform.asciidoc | 6 +++--- .../transform/apis/update-transform.asciidoc | 6 +++--- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/docs/reference/rest-api/common-parms.asciidoc b/docs/reference/rest-api/common-parms.asciidoc index 008fe7034123d..8c5da7f50bed8 100644 --- a/docs/reference/rest-api/common-parms.asciidoc +++ b/docs/reference/rest-api/common-parms.asciidoc @@ -949,6 +949,13 @@ tag::transform-settings[] Defines optional {transform} settings. end::transform-settings[] +tag::transform-settings-dates-as-epoch-milli[] +Defines if dates in the ouput should be written as ISO formatted string (default) +or as millis since epoch. `epoch_millis` has been the default for transforms created +before version `7.11`. For compatible output set this to `true`. +The default value is `false`. +end::transform-settings-dates-as-epoch-milli[] + tag::transform-settings-docs-per-second[] Specifies a limit on the number of input documents per second. This setting throttles the {transform} by adding a wait time between search requests. The @@ -962,13 +969,6 @@ adjusted to a lower value. The minimum value is `10` and the maximum is `10,000` The default value is `500`. end::transform-settings-max-page-search-size[] -tag::transform-settings-dates-as-epoch-milli[] -Defines if dates in the ouput should be written as ISO formatted string (default) -or as millis since epoch. `epoch_millis` has been the default for transforms created -before version `7.11`. For compatible output set this to `true`. -The default value is `false`. -end::transform-settings-dates-as-epoch-milli[] - tag::target-index[] ``:: + diff --git a/docs/reference/transform/apis/put-transform.asciidoc b/docs/reference/transform/apis/put-transform.asciidoc index 3d5b648f0a0bd..712fada7ebe7a 100644 --- a/docs/reference/transform/apis/put-transform.asciidoc +++ b/docs/reference/transform/apis/put-transform.asciidoc @@ -134,15 +134,15 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings] .Properties of `settings` [%collapsible%open] ==== +`dates_as_epoch_millis`::: +(Optional, boolean) +include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-dates-as-epoch-milli] `docs_per_second`::: (Optional, float) include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-docs-per-second] `max_page_search_size`::: (Optional, integer) include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-max-page-search-size] -`dates_as_epoch_millis`::: -(Optional, boolean) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-dates-as-epoch-milli] ==== //End settings diff --git a/docs/reference/transform/apis/update-transform.asciidoc b/docs/reference/transform/apis/update-transform.asciidoc index 3d4759e7a2ef9..537c8707963fc 100644 --- a/docs/reference/transform/apis/update-transform.asciidoc +++ b/docs/reference/transform/apis/update-transform.asciidoc @@ -110,15 +110,15 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings] .Properties of `settings` [%collapsible%open] ==== +`dates_as_epoch_millis`::: +(Optional, boolean) +include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-dates-as-epoch-milli] `docs_per_second`::: (Optional, float) include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-docs-per-second] `max_page_search_size`::: (Optional, integer) include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-max-page-search-size] -`dates_as_epoch_millis`::: -(Optional, boolean) -include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=transform-settings-dates-as-epoch-milli] ==== //End settings From 113db1b1c0e59c67255b9ba1e2013efcb1d2c44c Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Mon, 7 Dec 2020 09:30:35 +0100 Subject: [PATCH 14/15] fix parameter name and remove debug code --- .../transform/transforms/SettingsConfig.java | 10 +++++----- .../transforms/SettingsConfigTests.java | 2 +- .../transform/transforms/SettingsConfig.java | 16 ++++++++-------- .../transforms/SettingsConfigTests.java | 2 +- .../transforms/TransformConfigTests.java | 2 +- .../continuous/DateHistogramGroupByIT.java | 2 +- .../DateHistogramGroupByOtherTimeFieldIT.java | 2 +- 7 files changed, 18 insertions(+), 18 deletions(-) diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/SettingsConfig.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/SettingsConfig.java index 246a0e1ab1538..01b4e300eba6b 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/SettingsConfig.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/SettingsConfig.java @@ -141,7 +141,7 @@ public static Builder builder() { public static class Builder { private Integer maxPageSearchSize; private Float docsPerSecond; - private Integer datesAsEpochMilli; + private Integer datesAsEpochMillis; /** * Sets the paging maximum paging maxPageSearchSize that transform can use when @@ -179,16 +179,16 @@ public Builder setRequestsPerSecond(Float docsPerSecond) { * * An explicit `null` resets to default. * - * @param datesAsEpochMilli true if dates should be written as epoch_millis. + * @param datesAsEpochMillis true if dates should be written as epoch_millis. * @return the {@link Builder} with datesAsEpochMilli set. */ - public Builder setDatesAsEpochMilli(Boolean datesAsEpochMilli) { - this.datesAsEpochMilli = datesAsEpochMilli == null ? DEFAULT_DATES_AS_EPOCH_MILLIS : datesAsEpochMilli ? 1 : 0; + public Builder setDatesAsEpochMillis(Boolean datesAsEpochMillis) { + this.datesAsEpochMillis = datesAsEpochMillis == null ? DEFAULT_DATES_AS_EPOCH_MILLIS : datesAsEpochMillis ? 1 : 0; return this; } public SettingsConfig build() { - return new SettingsConfig(maxPageSearchSize, docsPerSecond, datesAsEpochMilli); + return new SettingsConfig(maxPageSearchSize, docsPerSecond, datesAsEpochMillis); } } } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/SettingsConfigTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/SettingsConfigTests.java index 9945d2575f4d0..bd435ea7c711c 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/SettingsConfigTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/SettingsConfigTests.java @@ -118,7 +118,7 @@ public void testExplicitNullOnWriteBuilder() throws IOException { assertNull(settingsAsMap.getOrDefault("docs_per_second", "not_set")); assertThat(settingsAsMap.getOrDefault("dates_as_epoch_millis", "not_set"), equalTo("not_set")); - config = new SettingsConfig.Builder().setDatesAsEpochMilli(null).build(); + config = new SettingsConfig.Builder().setDatesAsEpochMillis(null).build(); // returns false, however it's `null` as in "use default", checked next assertFalse(config.getDatesAsEpochMillis()); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/SettingsConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/SettingsConfig.java index 78d3f3cc4fa00..1686cc50cfbbc 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/SettingsConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/SettingsConfig.java @@ -161,7 +161,7 @@ public int hashCode() { @Override public String toString() { - return Strings.toString(this, true, true) + " wdaem: " + this.datesAsEpochMillis; + return Strings.toString(this, true, true); } public static SettingsConfig fromXContent(final XContentParser parser, boolean lenient) throws IOException { @@ -171,7 +171,7 @@ public static SettingsConfig fromXContent(final XContentParser parser, boolean l public static class Builder { private Integer maxPageSearchSize; private Float docsPerSecond; - private Integer dateAsEpochMilli; + private Integer datesAsEpochMillis; /** * Default builder @@ -186,7 +186,7 @@ public Builder() {} public Builder(SettingsConfig base) { this.maxPageSearchSize = base.maxPageSearchSize; this.docsPerSecond = base.docsPerSecond; - this.dateAsEpochMilli = base.datesAsEpochMillis; + this.datesAsEpochMillis = base.datesAsEpochMillis; } /** @@ -225,11 +225,11 @@ public Builder setRequestsPerSecond(Float docsPerSecond) { * * An explicit `null` resets to default. * - * @param datesAsEpochMilli true if dates should be written as epoch_millis. + * @param datesAsEpochMillis true if dates should be written as epoch_millis. * @return the {@link Builder} with datesAsEpochMilli set. */ - public Builder setDatesAsEpochMilli(Boolean datesAsEpochMilli) { - this.dateAsEpochMilli = datesAsEpochMilli == null ? DEFAULT_DATES_AS_EPOCH_MILLIS : datesAsEpochMilli ? 1 : 0; + public Builder setDatesAsEpochMillis(Boolean datesAsEpochMillis) { + this.datesAsEpochMillis = datesAsEpochMillis == null ? DEFAULT_DATES_AS_EPOCH_MILLIS : datesAsEpochMillis ? 1 : 0; return this; } @@ -251,7 +251,7 @@ public Builder update(SettingsConfig update) { : update.getMaxPageSearchSize(); } if (update.getDatesAsEpochMillisForUpdate() != null) { - this.dateAsEpochMilli = update.getDatesAsEpochMillisForUpdate().equals(DEFAULT_DATES_AS_EPOCH_MILLIS) + this.datesAsEpochMillis = update.getDatesAsEpochMillisForUpdate().equals(DEFAULT_DATES_AS_EPOCH_MILLIS) ? null : update.getDatesAsEpochMillisForUpdate(); } @@ -260,7 +260,7 @@ public Builder update(SettingsConfig update) { } public SettingsConfig build() { - return new SettingsConfig(maxPageSearchSize, docsPerSecond, dateAsEpochMilli); + return new SettingsConfig(maxPageSearchSize, docsPerSecond, datesAsEpochMillis); } } } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/SettingsConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/SettingsConfigTests.java index b315b3098ca92..646a8c360060b 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/SettingsConfigTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/SettingsConfigTests.java @@ -146,7 +146,7 @@ public void testOmmitDefaultsOnWriteBuilder() throws IOException { settingsAsMap = xContentToMap(config); assertTrue(settingsAsMap.isEmpty()); - config = new SettingsConfig.Builder().setDatesAsEpochMilli(null).build(); + config = new SettingsConfig.Builder().setDatesAsEpochMillis(null).build(); assertThat(config.getDatesAsEpochMillisForUpdate(), equalTo(-1)); settingsAsMap = xContentToMap(config); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigTests.java index 3320b5f7c9098..a23a3a26d44ab 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigTests.java @@ -420,7 +420,7 @@ public void testRewriteForBWCOfDateNormalization() throws IOException { assertEquals(Version.CURRENT, transformConfigRewritten.getVersion()); TransformConfig explicitTrueAfter711 = new TransformConfig.Builder(transformConfig).setSettings( - new SettingsConfig.Builder(transformConfigRewritten.getSettings()).setDatesAsEpochMilli(true).build() + new SettingsConfig.Builder(transformConfigRewritten.getSettings()).setDatesAsEpochMillis(true).build() ).setVersion(Version.V_8_0_0).build(); // todo: V_7_11_0 transformConfigRewritten = TransformConfig.rewriteForUpdate(explicitTrueAfter711); diff --git a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/DateHistogramGroupByIT.java b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/DateHistogramGroupByIT.java index 6dec6ce8260c8..f264d016c3684 100644 --- a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/DateHistogramGroupByIT.java +++ b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/DateHistogramGroupByIT.java @@ -55,7 +55,7 @@ public TransformConfig createConfig() { TransformConfig.Builder transformConfigBuilder = new TransformConfig.Builder(); addCommonBuilderParameters(transformConfigBuilder); if (datesAsEpochMillis) { - transformConfigBuilder.setSettings(addCommonSetings(new SettingsConfig.Builder()).setDatesAsEpochMilli(true).build()); + transformConfigBuilder.setSettings(addCommonSetings(new SettingsConfig.Builder()).setDatesAsEpochMillis(true).build()); } transformConfigBuilder.setSource(new SourceConfig(CONTINUOUS_EVENTS_SOURCE_INDEX)); diff --git a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/DateHistogramGroupByOtherTimeFieldIT.java b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/DateHistogramGroupByOtherTimeFieldIT.java index 2678e69a25939..0bbd0b264bf71 100644 --- a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/DateHistogramGroupByOtherTimeFieldIT.java +++ b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/DateHistogramGroupByOtherTimeFieldIT.java @@ -55,7 +55,7 @@ public TransformConfig createConfig() { TransformConfig.Builder transformConfigBuilder = new TransformConfig.Builder(); addCommonBuilderParameters(transformConfigBuilder); if (datesAsEpochMillis) { - transformConfigBuilder.setSettings(addCommonSetings(new SettingsConfig.Builder()).setDatesAsEpochMilli(true).build()); + transformConfigBuilder.setSettings(addCommonSetings(new SettingsConfig.Builder()).setDatesAsEpochMillis(true).build()); } transformConfigBuilder.setSource(new SourceConfig(CONTINUOUS_EVENTS_SOURCE_INDEX)); transformConfigBuilder.setDest(new DestConfig(NAME, INGEST_PIPELINE)); From 23301ba7c02c70848fb818a7cf8cd7b6a0b763cb Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Mon, 7 Dec 2020 13:47:24 +0100 Subject: [PATCH 15/15] apply review suggestion --- .../xpack/core/transform/transforms/SettingsConfig.java | 4 +--- .../xpack/core/transform/transforms/TransformConfig.java | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/SettingsConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/SettingsConfig.java index 1686cc50cfbbc..ab4d297fd2d06 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/SettingsConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/SettingsConfig.java @@ -60,9 +60,7 @@ public SettingsConfig() { } public SettingsConfig(Integer maxPageSearchSize, Float docsPerSecond, Boolean datesAsEpochMillis) { - this.maxPageSearchSize = maxPageSearchSize; - this.docsPerSecond = docsPerSecond; - this.datesAsEpochMillis = datesAsEpochMillis == null ? null : datesAsEpochMillis ? 1 : 0; + this(maxPageSearchSize, docsPerSecond, datesAsEpochMillis == null ? null : datesAsEpochMillis ? 1 : 0); } public SettingsConfig(Integer maxPageSearchSize, Float docsPerSecond, Integer datesAsEpochMillis) { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java index 1ca611998dd6a..27f16f854d8bc 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java @@ -452,7 +452,7 @@ public static TransformConfig fromXContent(final XContentParser parser, @Nullabl */ public static TransformConfig rewriteForUpdate(final TransformConfig transformConfig) { - // quick checks(optimization) if a rewrite is required, if none found just return the original + // quick check if a rewrite is required, if none found just return the original // a failing quick check, does not mean a rewrite is necessary if (transformConfig.getVersion() != null && transformConfig.getVersion().onOrAfter(Version.V_8_0_0) // todo: V_7_11_0