From 9118761390f87134347d27810193b555d98e9fba Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Wed, 20 Jan 2021 10:37:23 +0100 Subject: [PATCH 01/39] create config classes for retention policy configuration --- .../TransformNamedXContentProvider.java | 12 +- .../transforms/RetentionPolicyConfig.java | 30 + .../transforms/TimeRetentionPolicyConfig.java | 94 +++ .../transform/transforms/TransformConfig.java | 71 +- .../client/RestHighLevelClientTests.java | 5 +- .../TimeRetentionPolicyConfigTests.java | 29 + .../transforms/TransformConfigTests.java | 5 + .../hlrc/TimeRetentionPolicyConfigTests.java | 69 ++ .../search/AsyncSearchResponseTests.java | 2 +- .../xpack/core/XPackClientPlugin.java | 9 +- .../xpack/core/transform/TransformField.java | 4 +- .../TransformNamedXContentProvider.java | 10 +- .../transforms/RetentionPolicyConfig.java | 14 + .../transforms/TimeRetentionPolicyConfig.java | 115 +++ .../transform/transforms/TimeSyncConfig.java | 2 +- .../transform/transforms/TransformConfig.java | 54 +- .../transforms/TransformConfig.java.orig | 662 ++++++++++++++++++ .../AbstractSerializingTransformTestCase.java | 49 +- .../MockDeprecatedAggregationBuilder.java | 2 +- .../MockDeprecatedQueryBuilder.java | 2 +- ...tractWireSerializingTransformTestCase.java | 7 +- .../PreviewTransformActionRequestTests.java | 2 + .../PutTransformActionRequestTests.java | 24 +- .../UpdateTransformsActionResponseTests.java | 1 + .../AbstractSerializingTransformTestCase.java | 76 -- .../transform/transforms/DestConfigTests.java | 1 + .../transforms/QueryConfigTests.java | 2 + .../transforms/SettingsConfigTests.java | 1 + .../transforms/SourceConfigTests.java | 1 + .../TimeRetentionPolicyConfigTests.java | 30 + .../TransformCheckpointStatsTests.java | 1 + .../transforms/TransformCheckpointTests.java | 1 + .../TransformCheckpointingInfoTests.java | 1 + .../transforms/TransformConfigTests.java | 57 +- .../TransformConfigUpdateTests.java | 22 +- .../TransformDestIndexSettingsTests.java | 1 + .../transforms/TransformProgressTests.java | 1 + .../transforms/TransformStoredDocTests.java | 1 + .../transform/transforms/TransformTests.java | 1 + .../transforms/latest/LatestConfigTests.java | 2 +- .../pivot/AggregationConfigTests.java | 4 +- .../transforms/pivot/PivotConfigTests.java | 2 +- .../transforms/pivot/ScriptConfigTests.java | 2 +- 43 files changed, 1329 insertions(+), 152 deletions(-) create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/RetentionPolicyConfig.java create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/TimeRetentionPolicyConfig.java create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/TimeRetentionPolicyConfigTests.java create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/hlrc/TimeRetentionPolicyConfigTests.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/RetentionPolicyConfig.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TimeRetentionPolicyConfig.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java.orig rename x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/{action => }/AbstractSerializingTransformTestCase.java (60%) rename x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/{transforms => }/MockDeprecatedAggregationBuilder.java (98%) rename x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/{transforms => }/MockDeprecatedQueryBuilder.java (98%) delete mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/AbstractSerializingTransformTestCase.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TimeRetentionPolicyConfigTests.java diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/TransformNamedXContentProvider.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/TransformNamedXContentProvider.java index ad21119b929a6..8b7dcdcbe18e0 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/TransformNamedXContentProvider.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/TransformNamedXContentProvider.java @@ -8,7 +8,9 @@ package org.elasticsearch.client.transform; +import org.elasticsearch.client.transform.transforms.RetentionPolicyConfig; import org.elasticsearch.client.transform.transforms.SyncConfig; +import org.elasticsearch.client.transform.transforms.TimeRetentionPolicyConfig; import org.elasticsearch.client.transform.transforms.TimeSyncConfig; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.xcontent.NamedXContentRegistry; @@ -22,9 +24,13 @@ public class TransformNamedXContentProvider implements NamedXContentProvider { @Override public List getNamedXContentParsers() { return Arrays.asList( - new NamedXContentRegistry.Entry(SyncConfig.class, - new ParseField(TimeSyncConfig.NAME), - TimeSyncConfig::fromXContent)); + new NamedXContentRegistry.Entry(SyncConfig.class, new ParseField(TimeSyncConfig.NAME), TimeSyncConfig::fromXContent), + new NamedXContentRegistry.Entry( + RetentionPolicyConfig.class, + new ParseField(TimeRetentionPolicyConfig.NAME), + TimeRetentionPolicyConfig::fromXContent + ) + ); } } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/RetentionPolicyConfig.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/RetentionPolicyConfig.java new file mode 100644 index 0000000000000..639ca82b877f3 --- /dev/null +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/RetentionPolicyConfig.java @@ -0,0 +1,30 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client.transform.transforms; + +import org.elasticsearch.common.xcontent.ToXContentObject; + +public interface RetentionPolicyConfig extends ToXContentObject { + + /** + * Returns the name of the writeable object + */ + String getName(); +} diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/TimeRetentionPolicyConfig.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/TimeRetentionPolicyConfig.java new file mode 100644 index 0000000000000..78e13bd51cf9d --- /dev/null +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/TimeRetentionPolicyConfig.java @@ -0,0 +1,94 @@ +package org.elasticsearch.client.transform.transforms; + +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.io.IOException; +import java.util.Objects; + +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg; + +public class TimeRetentionPolicyConfig implements RetentionPolicyConfig { + + public static final String NAME = "time"; + + private static final ParseField FIELD = new ParseField("field"); + private static final ParseField MAX_AGE = new ParseField("max_age"); + + private final String field; + private final TimeValue maxAge; + + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "time_retention_policy_config", + true, + args -> new TimeRetentionPolicyConfig((String) args[0], args[1] != null ? (TimeValue) args[1] : TimeValue.ZERO) + ); + + static { + PARSER.declareString(constructorArg(), FIELD); + PARSER.declareField( + constructorArg(), + (p, c) -> TimeValue.parseTimeValue(p.text(), MAX_AGE.getPreferredName()), + MAX_AGE, + ObjectParser.ValueType.STRING + ); + } + + public static TimeRetentionPolicyConfig fromXContent(XContentParser parser) { + return PARSER.apply(parser, null); + } + + public TimeRetentionPolicyConfig(String field, TimeValue maxAge) { + this.field = field; + this.maxAge = maxAge; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(FIELD.getPreferredName(), field); + if (maxAge.duration() > 0) { + builder.field(MAX_AGE.getPreferredName(), maxAge.getStringRep()); + } + builder.endObject(); + return builder; + } + + public String getField() { + return field; + } + + public TimeValue getMaxAge() { + return maxAge; + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + + if (other == null || getClass() != other.getClass()) { + return false; + } + + final TimeRetentionPolicyConfig that = (TimeRetentionPolicyConfig) other; + + return Objects.equals(this.field, that.field) && Objects.equals(this.maxAge, that.maxAge); + } + + @Override + public int hashCode() { + return Objects.hash(field, maxAge); + } + + @Override + public String getName() { + return NAME; + } +} diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/TransformConfig.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/TransformConfig.java index 21f4e6d95f0f1..44ce44696705a 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/TransformConfig.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/TransformConfig.java @@ -41,6 +41,7 @@ public class TransformConfig implements ToXContentObject { public static final ParseField SETTINGS = new ParseField("settings"); public static final ParseField VERSION = new ParseField("version"); public static final ParseField CREATE_TIME = new ParseField("create_time"); + public static final ParseField RETENTION_POLICY = new ParseField("retention_policy"); // types of transforms public static final ParseField PIVOT_TRANSFORM = new ParseField("pivot"); public static final ParseField LATEST_TRANSFORM = new ParseField("latest"); @@ -54,6 +55,7 @@ public class TransformConfig implements ToXContentObject { private final PivotConfig pivotConfig; private final LatestConfig latestConfig; private final String description; + private final RetentionPolicyConfig retentionPolicyConfig; private final Version transformVersion; private final Instant createTime; @@ -70,8 +72,9 @@ public class TransformConfig implements ToXContentObject { LatestConfig latestConfig = (LatestConfig) args[6]; String description = (String) args[7]; SettingsConfig settings = (SettingsConfig) args[8]; - Instant createTime = (Instant) args[9]; - String transformVersion = (String) args[10]; + RetentionPolicyConfig retentionPolicyConfig = (RetentionPolicyConfig) args[9]; + Instant createTime = (Instant) args[10]; + String transformVersion = (String) args[11]; return new TransformConfig( id, source, @@ -82,6 +85,7 @@ public class TransformConfig implements ToXContentObject { latestConfig, description, settings, + retentionPolicyConfig, createTime, transformVersion ); @@ -103,6 +107,7 @@ public class TransformConfig implements ToXContentObject { PARSER.declareObject(optionalConstructorArg(), (p, c) -> LatestConfig.fromXContent(p), LATEST_TRANSFORM); PARSER.declareString(optionalConstructorArg(), DESCRIPTION); PARSER.declareObject(optionalConstructorArg(), (p, c) -> SettingsConfig.fromXContent(p), SETTINGS); + PARSER.declareObject(optionalConstructorArg(), (p, c) -> parseRentionPolicyConfig(p), RETENTION_POLICY); PARSER.declareField( optionalConstructorArg(), p -> TimeUtil.parseTimeFieldToInstant(p, CREATE_TIME.getPreferredName()), @@ -110,6 +115,7 @@ public class TransformConfig implements ToXContentObject { ObjectParser.ValueType.VALUE ); PARSER.declareString(optionalConstructorArg(), VERSION); + PARSER.declareObject(optionalConstructorArg(), (p, c) -> parseRentionPolicyConfig(p), RETENTION_POLICY); } private static SyncConfig parseSyncConfig(XContentParser parser) throws IOException { @@ -120,6 +126,14 @@ private static SyncConfig parseSyncConfig(XContentParser parser) throws IOExcept return syncConfig; } + private static RetentionPolicyConfig parseRentionPolicyConfig(XContentParser parser) throws IOException { + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser); + XContentParserUtils.ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.nextToken(), parser); + RetentionPolicyConfig retentionPolicyConfig = parser.namedObject(RetentionPolicyConfig.class, parser.currentName(), true); + XContentParserUtils.ensureExpectedToken(XContentParser.Token.END_OBJECT, parser.nextToken(), parser); + return retentionPolicyConfig; + } + public static TransformConfig fromXContent(final XContentParser parser) { return PARSER.apply(parser, null); } @@ -136,7 +150,7 @@ public static TransformConfig fromXContent(final XContentParser parser) { * @return A TransformConfig to preview, NOTE it will have a {@code null} id, destination and index. */ public static TransformConfig forPreview(final SourceConfig source, final PivotConfig pivotConfig) { - return new TransformConfig(null, source, null, null, null, pivotConfig, null, null, null, null, null); + return new TransformConfig(null, source, null, null, null, pivotConfig, null, null, null, null, null, null); } /** @@ -151,7 +165,7 @@ public static TransformConfig forPreview(final SourceConfig source, final PivotC * @return A TransformConfig to preview, NOTE it will have a {@code null} id, destination and index. */ public static TransformConfig forPreview(final SourceConfig source, final LatestConfig latestConfig) { - return new TransformConfig(null, source, null, null, null, null, latestConfig, null, null, null, null); + return new TransformConfig(null, source, null, null, null, null, latestConfig, null, null, null, null, null); } TransformConfig( @@ -164,6 +178,7 @@ public static TransformConfig forPreview(final SourceConfig source, final Latest final LatestConfig latestConfig, final String description, final SettingsConfig settings, + final RetentionPolicyConfig retentionPolicyConfig, final Instant createTime, final String version ) { @@ -176,6 +191,7 @@ public static TransformConfig forPreview(final SourceConfig source, final Latest this.latestConfig = latestConfig; this.description = description; this.settings = settings; + this.retentionPolicyConfig = retentionPolicyConfig; this.createTime = createTime == null ? null : Instant.ofEpochMilli(createTime.toEpochMilli()); this.transformVersion = version == null ? null : Version.fromString(version); } @@ -226,6 +242,11 @@ public SettingsConfig getSettings() { return settings; } + @Nullable + public RetentionPolicyConfig getRetentionPolicyConfig() { + return retentionPolicyConfig; + } + @Override public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException { builder.startObject(); @@ -258,6 +279,11 @@ public XContentBuilder toXContent(final XContentBuilder builder, final Params pa if (settings != null) { builder.field(SETTINGS.getPreferredName(), settings); } + if (retentionPolicyConfig != null) { + builder.startObject(RETENTION_POLICY.getPreferredName()); + builder.field(retentionPolicyConfig.getName(), retentionPolicyConfig); + builder.endObject(); + } if (createTime != null) { builder.timeField(CREATE_TIME.getPreferredName(), CREATE_TIME.getPreferredName() + "_string", createTime.toEpochMilli()); } @@ -290,13 +316,26 @@ public boolean equals(Object other) { && Objects.equals(this.settings, that.settings) && Objects.equals(this.createTime, that.createTime) && Objects.equals(this.pivotConfig, that.pivotConfig) - && Objects.equals(this.latestConfig, that.latestConfig); + && Objects.equals(this.latestConfig, that.latestConfig) + && Objects.equals(this.retentionPolicyConfig, that.retentionPolicyConfig); } @Override public int hashCode() { return Objects.hash( - id, source, dest, frequency, syncConfig, settings, createTime, transformVersion, pivotConfig, latestConfig, description); + id, + source, + dest, + frequency, + syncConfig, + settings, + createTime, + transformVersion, + pivotConfig, + latestConfig, + description, + retentionPolicyConfig + ); } @Override @@ -319,6 +358,7 @@ public static class Builder { private LatestConfig latestConfig; private SettingsConfig settings; private String description; + private RetentionPolicyConfig retentionPolicyConfig; public Builder setId(String id) { this.id = id; @@ -365,9 +405,26 @@ public Builder setSettings(SettingsConfig settings) { return this; } + public Builder setRetentionPolicyConfig(RetentionPolicyConfig retentionPolicyConfig) { + this.retentionPolicyConfig = retentionPolicyConfig; + return this; + } + public TransformConfig build() { return new TransformConfig( - id, source, dest, frequency, syncConfig, pivotConfig, latestConfig, description, settings, null, null); + id, + source, + dest, + frequency, + syncConfig, + pivotConfig, + latestConfig, + description, + settings, + retentionPolicyConfig, + null, + null + ); } } } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java index eb11fafcc4f8b..1f3bc0dd66e3b 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java @@ -9,6 +9,7 @@ package org.elasticsearch.client; import com.fasterxml.jackson.core.JsonParseException; + import org.apache.http.HttpEntity; import org.apache.http.HttpHost; import org.apache.http.HttpResponse; @@ -697,7 +698,7 @@ public void testDefaultNamedXContents() { public void testProvidedNamedXContents() { List namedXContents = RestHighLevelClient.getProvidedNamedXContents(); - assertEquals(76, namedXContents.size()); + assertEquals(77, namedXContents.size()); Map, Integer> categories = new HashMap<>(); List names = new ArrayList<>(); for (NamedXContentRegistry.Entry namedXContent : namedXContents) { @@ -707,7 +708,7 @@ public void testProvidedNamedXContents() { categories.put(namedXContent.categoryClass, counter + 1); } } - assertEquals("Had: " + categories, 14, categories.size()); + assertEquals("Had: " + categories, 15, categories.size()); assertEquals(Integer.valueOf(3), categories.get(Aggregation.class)); assertTrue(names.contains(ChildrenAggregationBuilder.NAME)); assertTrue(names.contains(MatrixStatsAggregationBuilder.NAME)); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/TimeRetentionPolicyConfigTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/TimeRetentionPolicyConfigTests.java new file mode 100644 index 0000000000000..86ff0447f4eff --- /dev/null +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/TimeRetentionPolicyConfigTests.java @@ -0,0 +1,29 @@ +package org.elasticsearch.client.transform.transforms; + +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.test.AbstractXContentTestCase; + +import java.io.IOException; + +public class TimeRetentionPolicyConfigTests extends AbstractXContentTestCase { + + public static TimeRetentionPolicyConfig randomTimeRetentionPolicyConfig() { + return new TimeRetentionPolicyConfig(randomAlphaOfLengthBetween(1, 10), new TimeValue(randomNonNegativeLong())); + } + + @Override + protected TimeRetentionPolicyConfig createTestInstance() { + return randomTimeRetentionPolicyConfig(); + } + + @Override + protected TimeRetentionPolicyConfig doParseInstance(XContentParser parser) throws IOException { + return TimeRetentionPolicyConfig.fromXContent(parser); + } + + @Override + protected boolean supportsUnknownFields() { + return true; + } +} diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/TransformConfigTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/TransformConfigTests.java index 30daddc0d9e68..1579d29df7a2b 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/TransformConfigTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/TransformConfigTests.java @@ -52,6 +52,7 @@ public static TransformConfig randomTransformConfig() { latestConfig, randomBoolean() ? null : randomAlphaOfLengthBetween(1, 100), SettingsConfigTests.randomSettingsConfig(), + randomBoolean() ? null : randomRetentionPolicyConfig(), randomBoolean() ? null : Instant.now(), randomBoolean() ? null : Version.CURRENT.toString() ); @@ -61,6 +62,10 @@ public static SyncConfig randomSyncConfig() { return TimeSyncConfigTests.randomTimeSyncConfig(); } + public static RetentionPolicyConfig randomRetentionPolicyConfig() { + return TimeRetentionPolicyConfigTests.randomTimeRetentionPolicyConfig(); + } + @Override protected TransformConfig createTestInstance() { return randomTransformConfig(); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/hlrc/TimeRetentionPolicyConfigTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/hlrc/TimeRetentionPolicyConfigTests.java new file mode 100644 index 0000000000000..913de99053ee0 --- /dev/null +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/hlrc/TimeRetentionPolicyConfigTests.java @@ -0,0 +1,69 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client.transform.transforms.hlrc; + +import org.elasticsearch.client.AbstractResponseTestCase; +import org.elasticsearch.client.transform.transforms.TimeRetentionPolicyConfig; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; + +import java.io.IOException; + +public class TimeRetentionPolicyConfigTests extends AbstractResponseTestCase< + org.elasticsearch.xpack.core.transform.transforms.TimeRetentionPolicyConfig, + TimeRetentionPolicyConfig> { + + public static org.elasticsearch.xpack.core.transform.transforms.TimeRetentionPolicyConfig randomTimeSyncConfig() { + return new org.elasticsearch.xpack.core.transform.transforms.TimeRetentionPolicyConfig( + randomAlphaOfLengthBetween(1, 10), + new TimeValue(randomNonNegativeLong()) + ); + } + + public static void assertHlrcEquals( + org.elasticsearch.xpack.core.transform.transforms.TimeRetentionPolicyConfig serverTestInstance, + TimeRetentionPolicyConfig clientInstance + ) { + assertEquals(serverTestInstance.getField(), clientInstance.getField()); + assertEquals(serverTestInstance.getMaxAge(), clientInstance.getMaxAge()); + } + + @Override + protected org.elasticsearch.xpack.core.transform.transforms.TimeRetentionPolicyConfig createServerTestInstance( + XContentType xContentType + ) { + return randomTimeSyncConfig(); + } + + @Override + protected TimeRetentionPolicyConfig doParseToClientInstance(XContentParser parser) throws IOException { + return TimeRetentionPolicyConfig.fromXContent(parser); + } + + @Override + protected void assertInstances( + org.elasticsearch.xpack.core.transform.transforms.TimeRetentionPolicyConfig serverTestInstance, + TimeRetentionPolicyConfig clientInstance + ) { + assertHlrcEquals(serverTestInstance, clientInstance); + } + +} diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchResponseTests.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchResponseTests.java index 044820d4bb158..21ee11a653913 100644 --- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchResponseTests.java +++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchResponseTests.java @@ -48,7 +48,7 @@ public void registerNamedObjects() { SearchModule searchModule = new SearchModule(Settings.EMPTY, emptyList()); List namedWriteables = searchModule.getNamedWriteables(); - namedWriteables.add(new NamedWriteableRegistry.Entry(SyncConfig.class, TransformField.TIME_BASED_SYNC.getPreferredName(), + namedWriteables.add(new NamedWriteableRegistry.Entry(SyncConfig.class, TransformField.TIME.getPreferredName(), TimeSyncConfig::new)); List namedXContents = searchModule.getNamedXContents(); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java index 5965162676d02..458e7b24f9787 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java @@ -93,7 +93,6 @@ import org.elasticsearch.xpack.core.ml.action.ExplainDataFrameAnalyticsAction; import org.elasticsearch.xpack.core.ml.action.FinalizeJobExecutionAction; import org.elasticsearch.xpack.core.rollup.action.RollupIndexerAction; -import org.elasticsearch.xpack.core.textstructure.action.FindStructureAction; import org.elasticsearch.xpack.core.ml.action.FlushJobAction; import org.elasticsearch.xpack.core.ml.action.ForecastJobAction; import org.elasticsearch.xpack.core.ml.action.GetBucketsAction; @@ -153,12 +152,12 @@ import org.elasticsearch.xpack.core.rollup.action.GetRollupCapsAction; import org.elasticsearch.xpack.core.rollup.action.GetRollupJobsAction; import org.elasticsearch.xpack.core.rollup.action.PutRollupJobAction; +import org.elasticsearch.xpack.core.rollup.action.RollupAction; import org.elasticsearch.xpack.core.rollup.action.RollupSearchAction; import org.elasticsearch.xpack.core.rollup.action.StartRollupJobAction; import org.elasticsearch.xpack.core.rollup.action.StopRollupJobAction; import org.elasticsearch.xpack.core.rollup.job.RollupJob; import org.elasticsearch.xpack.core.rollup.job.RollupJobStatus; -import org.elasticsearch.xpack.core.rollup.action.RollupAction; import org.elasticsearch.xpack.core.runtimefields.RuntimeFieldsFeatureSetUsage; import org.elasticsearch.xpack.core.search.action.GetAsyncSearchAction; import org.elasticsearch.xpack.core.search.action.SubmitAsyncSearchAction; @@ -203,6 +202,7 @@ import org.elasticsearch.xpack.core.spatial.SpatialFeatureSetUsage; import org.elasticsearch.xpack.core.sql.SqlFeatureSetUsage; import org.elasticsearch.xpack.core.ssl.action.GetCertificateInfoAction; +import org.elasticsearch.xpack.core.textstructure.action.FindStructureAction; import org.elasticsearch.xpack.core.transform.TransformFeatureSetUsage; import org.elasticsearch.xpack.core.transform.TransformField; import org.elasticsearch.xpack.core.transform.action.DeleteTransformAction; @@ -212,7 +212,9 @@ import org.elasticsearch.xpack.core.transform.action.PutTransformAction; import org.elasticsearch.xpack.core.transform.action.StartTransformAction; import org.elasticsearch.xpack.core.transform.action.StopTransformAction; +import org.elasticsearch.xpack.core.transform.transforms.RetentionPolicyConfig; import org.elasticsearch.xpack.core.transform.transforms.SyncConfig; +import org.elasticsearch.xpack.core.transform.transforms.TimeRetentionPolicyConfig; import org.elasticsearch.xpack.core.transform.transforms.TimeSyncConfig; import org.elasticsearch.xpack.core.transform.transforms.TransformState; import org.elasticsearch.xpack.core.transform.transforms.TransformTaskParams; @@ -504,7 +506,8 @@ public List getNamedWriteables() { new NamedWriteableRegistry.Entry(PersistentTaskParams.class, TransformField.TASK_NAME, TransformTaskParams::new), new NamedWriteableRegistry.Entry(Task.Status.class, TransformField.TASK_NAME, TransformState::new), new NamedWriteableRegistry.Entry(PersistentTaskState.class, TransformField.TASK_NAME, TransformState::new), - new NamedWriteableRegistry.Entry(SyncConfig.class, TransformField.TIME_BASED_SYNC.getPreferredName(), TimeSyncConfig::new), + new NamedWriteableRegistry.Entry(SyncConfig.class, TransformField.TIME.getPreferredName(), TimeSyncConfig::new), + new NamedWriteableRegistry.Entry(RetentionPolicyConfig.class, TransformField.TIME.getPreferredName(), TimeRetentionPolicyConfig::new), // Flattened for backward compatibility with 7.x new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.FLATTENED, FlattenedFeatureSetUsage::new), // Vectors diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformField.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformField.java index f5cf48ead7859..2870bba177b65 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformField.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformField.java @@ -39,9 +39,11 @@ public final class TransformField { public static final ParseField DATES_AS_EPOCH_MILLIS = new ParseField("dates_as_epoch_millis"); public static final ParseField FIELD = new ParseField("field"); public static final ParseField SYNC = new ParseField("sync"); - public static final ParseField TIME_BASED_SYNC = new ParseField("time"); + public static final ParseField TIME = new ParseField("time"); public static final ParseField DELAY = new ParseField("delay"); public static final ParseField DEFER_VALIDATION = new ParseField("defer_validation"); + public static final ParseField RETENTION_POLICY = new ParseField("retention_policy"); + public static final ParseField MAX_AGE = new ParseField("max_age"); public static final ParseField ALLOW_NO_MATCH = new ParseField("allow_no_match"); /** diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformNamedXContentProvider.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformNamedXContentProvider.java index 35a6bbdf15614..e4611b04fdad4 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformNamedXContentProvider.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformNamedXContentProvider.java @@ -9,7 +9,9 @@ import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.plugins.spi.NamedXContentProvider; +import org.elasticsearch.xpack.core.transform.transforms.RetentionPolicyConfig; import org.elasticsearch.xpack.core.transform.transforms.SyncConfig; +import org.elasticsearch.xpack.core.transform.transforms.TimeRetentionPolicyConfig; import org.elasticsearch.xpack.core.transform.transforms.TimeSyncConfig; import java.util.Arrays; @@ -21,7 +23,11 @@ public class TransformNamedXContentProvider implements NamedXContentProvider { public List getNamedXContentParsers() { return Arrays.asList( new NamedXContentRegistry.Entry(SyncConfig.class, - TransformField.TIME_BASED_SYNC, - TimeSyncConfig::parse)); + TransformField.TIME, + TimeSyncConfig::parse), + new NamedXContentRegistry.Entry(RetentionPolicyConfig.class, + TransformField.TIME, + TimeRetentionPolicyConfig::parse) + ); } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/RetentionPolicyConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/RetentionPolicyConfig.java new file mode 100644 index 0000000000000..08a4f853976f9 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/RetentionPolicyConfig.java @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core.transform.transforms; + +import org.elasticsearch.common.io.stream.NamedWriteable; +import org.elasticsearch.common.xcontent.ToXContentObject; + +public interface RetentionPolicyConfig extends ToXContentObject, NamedWriteable { + +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TimeRetentionPolicyConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TimeRetentionPolicyConfig.java new file mode 100644 index 0000000000000..fae32524bdb18 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TimeRetentionPolicyConfig.java @@ -0,0 +1,115 @@ +package org.elasticsearch.xpack.core.transform.transforms; + +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.xpack.core.transform.TransformField; +import org.elasticsearch.xpack.core.transform.utils.ExceptionsHelper; + +import java.io.IOException; +import java.util.Objects; + +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; + +public class TimeRetentionPolicyConfig implements RetentionPolicyConfig { + + private static final String NAME = "transform_retention_policy_time"; + + private final String field; + private final TimeValue maxAge; + + private static final ConstructingObjectParser STRICT_PARSER = createParser(false); + private static final ConstructingObjectParser LENIENT_PARSER = createParser(true); + + private static ConstructingObjectParser createParser(boolean lenient) { + ConstructingObjectParser parser = new ConstructingObjectParser<>(NAME, lenient, args -> { + String field = (String) args[0]; + TimeValue maxAge = (TimeValue) args[1]; + return new TimeRetentionPolicyConfig(field, maxAge); + }); + parser.declareString(constructorArg(), TransformField.FIELD); + parser.declareField( + constructorArg(), + (p, c) -> TimeValue.parseTimeValue(p.text(), TransformField.MAX_AGE.getPreferredName()), + TransformField.MAX_AGE, + ObjectParser.ValueType.STRING + ); + return parser; + } + + public TimeRetentionPolicyConfig(final String field, final TimeValue maxAge) { + this.field = ExceptionsHelper.requireNonNull(field, TransformField.FIELD.getPreferredName()); + this.maxAge = ExceptionsHelper.requireNonNull(maxAge, TransformField.MAX_AGE.getPreferredName()); + } + + public TimeRetentionPolicyConfig(StreamInput in) throws IOException { + this.field = in.readString(); + this.maxAge = in.readTimeValue(); + } + + public String getField() { + return field; + } + + public TimeValue getMaxAge() { + return maxAge; + } + + @Override + public void writeTo(final StreamOutput out) throws IOException { + out.writeString(field); + out.writeTimeValue(maxAge); + } + + @Override + public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException { + builder.startObject(); + builder.field(TransformField.FIELD.getPreferredName(), field); + builder.field(TransformField.MAX_AGE.getPreferredName(), maxAge.getStringRep()); + builder.endObject(); + return builder; + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + + if (other == null || getClass() != other.getClass()) { + return false; + } + + final TimeRetentionPolicyConfig that = (TimeRetentionPolicyConfig) other; + + return Objects.equals(this.field, that.field) && Objects.equals(this.maxAge, that.maxAge); + } + + @Override + public int hashCode() { + return Objects.hash(field, maxAge); + } + + @Override + public String toString() { + return Strings.toString(this, true, true); + } + + public static TimeRetentionPolicyConfig parse(final XContentParser parser) { + return LENIENT_PARSER.apply(parser, null); + } + + public static TimeRetentionPolicyConfig fromXContent(final XContentParser parser, boolean lenient) throws IOException { + return lenient ? LENIENT_PARSER.apply(parser, null) : STRICT_PARSER.apply(parser, null); + } + + @Override + public String getWriteableName() { + return TransformField.TIME.getPreferredName(); + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TimeSyncConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TimeSyncConfig.java index 8ab21573fe608..956018a243eb7 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TimeSyncConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TimeSyncConfig.java @@ -131,7 +131,7 @@ public static TimeSyncConfig fromXContent(final XContentParser parser, boolean l @Override public String getWriteableName() { - return TransformField.TIME_BASED_SYNC.getPreferredName(); + return TransformField.TIME.getPreferredName(); } @Override diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java index e56a107465235..b4577ca8985fa 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java @@ -61,6 +61,7 @@ public class TransformConfig extends AbstractDiffable implement private final TimeValue frequency; private final SyncConfig syncConfig; private final SettingsConfig settings; + private final RetentionPolicyConfig retentionPolicyConfig; private final String description; // headers store the user context from the creating user, which allows us to run the transform as this user // the header only contains name, groups and other context but no authorization keys @@ -103,8 +104,8 @@ private static ConstructingObjectParser createParser(bo if (lenient == false) { // on strict parsing do not allow injection of headers, transform version, or create time validateStrictParsingParams(args[6], HEADERS.getPreferredName()); - validateStrictParsingParams(args[11], TransformField.CREATE_TIME.getPreferredName()); - validateStrictParsingParams(args[12], TransformField.VERSION.getPreferredName()); + validateStrictParsingParams(args[12], TransformField.CREATE_TIME.getPreferredName()); + validateStrictParsingParams(args[13], TransformField.VERSION.getPreferredName()); // exactly one function must be defined if ((args[7] == null) == (args[8] == null)) { throw new IllegalArgumentException(TransformMessages.TRANSFORM_CONFIGURATION_BAD_FUNCTION_COUNT); @@ -118,6 +119,8 @@ private static ConstructingObjectParser createParser(bo LatestConfig latestConfig = (LatestConfig) args[8]; String description = (String) args[9]; SettingsConfig settings = (SettingsConfig) args[10]; + RetentionPolicyConfig retentionPolicyConfig = (RetentionPolicyConfig) args[11]; + return new TransformConfig( id, source, @@ -129,8 +132,9 @@ private static ConstructingObjectParser createParser(bo latestConfig, description, settings, - (Instant) args[11], - (String) args[12] + retentionPolicyConfig, + (Instant) args[12], + (String) args[13] ); }); @@ -145,6 +149,7 @@ private static ConstructingObjectParser createParser(bo parser.declareObject(optionalConstructorArg(), (p, c) -> LatestConfig.fromXContent(p, lenient), LATEST_TRANSFORM); parser.declareString(optionalConstructorArg(), TransformField.DESCRIPTION); parser.declareObject(optionalConstructorArg(), (p, c) -> SettingsConfig.fromXContent(p, lenient), TransformField.SETTINGS); + parser.declareObject(optionalConstructorArg(), (p, c) -> parseRentionPolicyConfig(p), TransformField.RETENTION_POLICY); parser.declareField( optionalConstructorArg(), p -> TimeUtils.parseTimeFieldToInstant(p, TransformField.CREATE_TIME.getPreferredName()), @@ -163,6 +168,14 @@ private static SyncConfig parseSyncConfig(XContentParser parser, boolean ignoreU return syncConfig; } + private static RetentionPolicyConfig parseRentionPolicyConfig(XContentParser parser) throws IOException { + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser); + XContentParserUtils.ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.nextToken(), parser); + RetentionPolicyConfig retentionPolicyConfig = parser.namedObject(RetentionPolicyConfig.class, parser.currentName(), true); + XContentParserUtils.ensureExpectedToken(XContentParser.Token.END_OBJECT, parser.nextToken(), parser); + return retentionPolicyConfig; + } + public static String documentId(String transformId) { return NAME + "-" + transformId; } @@ -178,6 +191,7 @@ public TransformConfig( final LatestConfig latestConfig, final String description, final SettingsConfig settings, + final RetentionPolicyConfig retentionPolicyConfig, final Instant createTime, final String version ) { @@ -191,6 +205,7 @@ public TransformConfig( this.latestConfig = latestConfig; this.description = description; this.settings = settings == null ? new SettingsConfig() : settings; + this.retentionPolicyConfig = retentionPolicyConfig; if (this.description != null && this.description.length() > MAX_DESCRIPTION_LENGTH) { throw new IllegalArgumentException("[description] must be less than 1000 characters in length."); } @@ -225,6 +240,11 @@ public TransformConfig(final StreamInput in) throws IOException { } else { settings = new SettingsConfig(); } + if (in.getVersion().onOrAfter(Version.V_8_0_0)) { + retentionPolicyConfig = in.readOptionalNamedWriteable(RetentionPolicyConfig.class); + } else { + retentionPolicyConfig = null; + } } public String getId() { @@ -292,6 +312,11 @@ public SettingsConfig getSettings() { return settings; } + @Nullable + public RetentionPolicyConfig getRetentionPolicyConfig() { + return retentionPolicyConfig; + } + public ActionRequestValidationException validate(ActionRequestValidationException validationException) { if (pivotConfig != null) { validationException = pivotConfig.validate(validationException); @@ -344,6 +369,9 @@ public void writeTo(final StreamOutput out) throws IOException { if (out.getVersion().onOrAfter(Version.V_7_8_0)) { settings.writeTo(out); } + if (out.getVersion().onOrAfter(Version.V_8_0_0)) { + out.writeOptionalNamedWriteable(retentionPolicyConfig); + } } @Override @@ -392,6 +420,11 @@ public XContentBuilder toXContent(final XContentBuilder builder, final Params pa builder.field(TransformField.DESCRIPTION.getPreferredName(), description); } builder.field(TransformField.SETTINGS.getPreferredName(), settings); + if (retentionPolicyConfig != null) { + builder.startObject(TransformField.RETENTION_POLICY.getPreferredName()); + builder.field(retentionPolicyConfig.getWriteableName(), retentionPolicyConfig); + builder.endObject(); + } builder.endObject(); return builder; } @@ -418,6 +451,7 @@ public boolean equals(Object other) { && Objects.equals(this.latestConfig, that.latestConfig) && Objects.equals(this.description, that.description) && Objects.equals(this.settings, that.settings) + && Objects.equals(this.retentionPolicyConfig, that.retentionPolicyConfig) && Objects.equals(this.createTime, that.createTime) && Objects.equals(this.transformVersion, that.transformVersion); } @@ -435,6 +469,7 @@ public int hashCode() { latestConfig, description, settings, + retentionPolicyConfig, createTime, transformVersion ); @@ -446,7 +481,6 @@ public String toString() { } public static TransformConfig fromXContent(final XContentParser parser, @Nullable final String optionalTransformId, boolean lenient) { - return lenient ? LENIENT_PARSER.apply(parser, optionalTransformId) : STRICT_PARSER.apply(parser, optionalTransformId); } @@ -524,6 +558,7 @@ public static class Builder { private PivotConfig pivotConfig; private LatestConfig latestConfig; private SettingsConfig settings; + private RetentionPolicyConfig retentionPolicyConfig; public Builder() {} @@ -539,6 +574,7 @@ public Builder(TransformConfig config) { this.pivotConfig = config.pivotConfig; this.latestConfig = config.latestConfig; this.settings = config.settings; + this.retentionPolicyConfig = config.retentionPolicyConfig; } public Builder setId(String id) { @@ -640,6 +676,11 @@ Version getVersion() { return transformVersion; } + public Builder setRetentionPolicyConfig(RetentionPolicyConfig retentionPolicyConfig) { + this.retentionPolicyConfig = retentionPolicyConfig; + return this; + } + public TransformConfig build() { return new TransformConfig( id, @@ -652,6 +693,7 @@ public TransformConfig build() { latestConfig, description, settings, + retentionPolicyConfig, createTime, transformVersion == null ? null : transformVersion.toString() ); @@ -679,6 +721,7 @@ public boolean equals(Object other) { && Objects.equals(this.latestConfig, that.latestConfig) && Objects.equals(this.description, that.description) && Objects.equals(this.settings, that.settings) + && Objects.equals(this.retentionPolicyConfig, that.retentionPolicyConfig) && Objects.equals(this.createTime, that.createTime) && Objects.equals(this.transformVersion, that.transformVersion); } @@ -696,6 +739,7 @@ public int hashCode() { latestConfig, description, settings, + retentionPolicyConfig, createTime, transformVersion ); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java.orig b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java.orig new file mode 100644 index 0000000000000..105314bd43225 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java.orig @@ -0,0 +1,662 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core.transform.transforms; + +import org.elasticsearch.Version; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.cluster.AbstractDiffable; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentParserUtils; +import org.elasticsearch.xpack.core.common.time.TimeUtils; +import org.elasticsearch.xpack.core.transform.TransformField; +import org.elasticsearch.xpack.core.transform.TransformMessages; +import org.elasticsearch.xpack.core.transform.transforms.map.MapConfig; +import org.elasticsearch.xpack.core.transform.transforms.pivot.PivotConfig; +import org.elasticsearch.xpack.core.transform.utils.ExceptionsHelper; + +import java.io.IOException; +import java.time.Instant; +import java.util.Collections; +import java.util.Map; +import java.util.Objects; + +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg; + +/** + * This class holds the configuration details of a data frame transform + */ +public class TransformConfig extends AbstractDiffable implements Writeable, ToXContentObject { + + public static final String NAME = "data_frame_transform_config"; + public static final ParseField HEADERS = new ParseField("headers"); + + // types of transforms + public static final ParseField PIVOT_TRANSFORM = new ParseField("pivot"); + public static final ParseField MAP_TRANSFORM = new ParseField("map"); + + private static final ConstructingObjectParser STRICT_PARSER = createParser(false); + private static final ConstructingObjectParser LENIENT_PARSER = createParser(true); + static final int MAX_DESCRIPTION_LENGTH = 1_000; + + private final String id; + private final SourceConfig source; + private final DestConfig dest; + private final TimeValue frequency; + private final SyncConfig syncConfig; + private final SettingsConfig settings; + private final String description; + // headers store the user context from the creating user, which allows us to run the transform as this user + // the header only contains name, groups and other context but no authorization keys + private Map headers; + private Version transformVersion; + private Instant createTime; + + private final PivotConfig pivotConfig; + private final MapConfig mapConfig; + + private static void validateStrictParsingParams(Object arg, String parameterName) { + if (arg != null) { + throw new IllegalArgumentException("Found [" + parameterName + "], not allowed for strict parsing"); + } + } + + private static ConstructingObjectParser createParser(boolean lenient) { + ConstructingObjectParser parser = new ConstructingObjectParser<>(NAME, lenient, (args, optionalId) -> { + String id = (String) args[0]; + + // if the id has been specified in the body and the path, they must match + if (id == null) { + id = optionalId; + } else if (optionalId != null && id.equals(optionalId) == false) { + throw new IllegalArgumentException( + TransformMessages.getMessage(TransformMessages.REST_PUT_TRANSFORM_INCONSISTENT_ID, id, optionalId) + ); + } + + SourceConfig source = (SourceConfig) args[1]; + DestConfig dest = (DestConfig) args[2]; + + TimeValue frequency = args[3] == null + ? null + : TimeValue.parseTimeValue((String) args[3], TransformField.FREQUENCY.getPreferredName()); + + SyncConfig syncConfig = (SyncConfig) args[4]; + // ignored, only for internal storage: String docType = (String) args[5]; + + // on strict parsing do not allow injection of headers, transform version, or create time + if (lenient == false) { + validateStrictParsingParams(args[6], HEADERS.getPreferredName()); + validateStrictParsingParams(args[11], TransformField.CREATE_TIME.getPreferredName()); + validateStrictParsingParams(args[12], TransformField.VERSION.getPreferredName()); + } + + @SuppressWarnings("unchecked") + Map headers = (Map) args[6]; + + PivotConfig pivotConfig = (PivotConfig) args[7]; + MapConfig mapConfig = (MapConfig) args[8]; + + String description = (String) args[9]; + SettingsConfig settings = (SettingsConfig) args[10]; + return new TransformConfig( + id, + source, + dest, + frequency, + syncConfig, + headers, + pivotConfig, + mapConfig, + description, + settings, + (Instant) args[11], + (String) args[12] + ); + }); + + parser.declareString(optionalConstructorArg(), TransformField.ID); + parser.declareObject(constructorArg(), (p, c) -> SourceConfig.fromXContent(p, lenient), TransformField.SOURCE); + parser.declareObject(constructorArg(), (p, c) -> DestConfig.fromXContent(p, lenient), TransformField.DESTINATION); + parser.declareString(optionalConstructorArg(), TransformField.FREQUENCY); + parser.declareObject(optionalConstructorArg(), (p, c) -> parseSyncConfig(p, lenient), TransformField.SYNC); + parser.declareString(optionalConstructorArg(), TransformField.INDEX_DOC_TYPE); + parser.declareObject(optionalConstructorArg(), (p, c) -> p.mapStrings(), HEADERS); + parser.declareObject(optionalConstructorArg(), (p, c) -> PivotConfig.fromXContent(p, lenient), PIVOT_TRANSFORM); + parser.declareObject(optionalConstructorArg(), (p, c) -> MapConfig.fromXContent(p, lenient), MAP_TRANSFORM); + parser.declareString(optionalConstructorArg(), TransformField.DESCRIPTION); + parser.declareObject(optionalConstructorArg(), (p, c) -> SettingsConfig.fromXContent(p, lenient), TransformField.SETTINGS); + parser.declareField( + optionalConstructorArg(), + p -> TimeUtils.parseTimeFieldToInstant(p, TransformField.CREATE_TIME.getPreferredName()), + TransformField.CREATE_TIME, + ObjectParser.ValueType.VALUE + ); + parser.declareString(optionalConstructorArg(), TransformField.VERSION); + return parser; + } + + private static SyncConfig parseSyncConfig(XContentParser parser, boolean ignoreUnknownFields) throws IOException { + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser); + XContentParserUtils.ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.nextToken(), parser); + SyncConfig syncConfig = parser.namedObject(SyncConfig.class, parser.currentName(), ignoreUnknownFields); + XContentParserUtils.ensureExpectedToken(XContentParser.Token.END_OBJECT, parser.nextToken(), parser); + return syncConfig; + } + + public static String documentId(String transformId) { + return NAME + "-" + transformId; + } + + TransformConfig( + final String id, + final SourceConfig source, + final DestConfig dest, + final TimeValue frequency, + final SyncConfig syncConfig, + final Map headers, + final PivotConfig pivotConfig, + final MapConfig mapConfig, + final String description, + final SettingsConfig settings, + final Instant createTime, + final String version + ) { + this.id = ExceptionsHelper.requireNonNull(id, TransformField.ID.getPreferredName()); + this.source = ExceptionsHelper.requireNonNull(source, TransformField.SOURCE.getPreferredName()); + this.dest = ExceptionsHelper.requireNonNull(dest, TransformField.DESTINATION.getPreferredName()); + this.frequency = frequency; + this.syncConfig = syncConfig; + this.setHeaders(headers == null ? Collections.emptyMap() : headers); + this.pivotConfig = pivotConfig; + this.mapConfig = mapConfig; + this.description = description; + this.settings = settings == null ? new SettingsConfig() : settings; + + // at least one function must be defined + if (this.pivotConfig == null && this.mapConfig == null) { + throw new IllegalArgumentException(TransformMessages.TRANSFORM_CONFIGURATION_NO_TRANSFORM); + } + // check that not both are given + if (this.pivotConfig != null && this.mapConfig != null) { + throw new IllegalArgumentException("only 1 function allowed"); + } + + if (this.description != null && this.description.length() > MAX_DESCRIPTION_LENGTH) { + throw new IllegalArgumentException("[description] must be less than 1000 characters in length."); + } + this.createTime = createTime == null ? null : Instant.ofEpochMilli(createTime.toEpochMilli()); + this.transformVersion = version == null ? null : Version.fromString(version); + } + + public TransformConfig( + final String id, + final SourceConfig source, + final DestConfig dest, + final TimeValue frequency, + final SyncConfig syncConfig, + final Map headers, + final PivotConfig pivotConfig, + final String description, + final SettingsConfig settings + ) { + this(id, source, dest, frequency, syncConfig, headers, pivotConfig, null, description, settings, null, null); + } + + public TransformConfig(final StreamInput in) throws IOException { + id = in.readString(); + source = new SourceConfig(in); + dest = new DestConfig(in); + if (in.getVersion().onOrAfter(Version.V_7_3_0)) { + frequency = in.readOptionalTimeValue(); + } else { + frequency = null; + } + setHeaders(in.readMap(StreamInput::readString, StreamInput::readString)); + pivotConfig = in.readOptionalWriteable(PivotConfig::new); + description = in.readOptionalString(); + if (in.getVersion().onOrAfter(Version.V_7_3_0)) { + syncConfig = in.readOptionalNamedWriteable(SyncConfig.class); + createTime = in.readOptionalInstant(); + transformVersion = in.readBoolean() ? Version.readVersion(in) : null; + } else { + syncConfig = null; + createTime = null; + transformVersion = null; + } + if (in.getVersion().onOrAfter(Version.V_7_8_0)) { + settings = new SettingsConfig(in); + } else { + settings = new SettingsConfig(); + } + if (in.getVersion().onOrAfter(Version.V_8_0_0)) { // todo 7.9.0 + mapConfig = in.readOptionalWriteable(MapConfig::new); + } else { + mapConfig = null; + } + } + + public String getId() { + return id; + } + + public SourceConfig getSource() { + return source; + } + + public DestConfig getDestination() { + return dest; + } + + public TimeValue getFrequency() { + return frequency; + } + + public SyncConfig getSyncConfig() { + return syncConfig; + } + + public Map getHeaders() { + return headers; + } + + public TransformConfig setHeaders(Map headers) { + this.headers = headers; + return this; + } + + public Version getVersion() { + return transformVersion; + } + + public TransformConfig setVersion(Version transformVersion) { + this.transformVersion = transformVersion; + return this; + } + + public Instant getCreateTime() { + return createTime; + } + + public TransformConfig setCreateTime(Instant createTime) { + ExceptionsHelper.requireNonNull(createTime, TransformField.CREATE_TIME.getPreferredName()); + this.createTime = Instant.ofEpochMilli(createTime.toEpochMilli()); + return this; + } + + public PivotConfig getPivotConfig() { + return pivotConfig; + } + + public MapConfig getMapConfig() { + return mapConfig; + } + + @Nullable + public String getDescription() { + return description; + } + + public SettingsConfig getSettings() { + return settings; + } + + public ActionRequestValidationException validate(ActionRequestValidationException validationException) { + if (pivotConfig != null) { + validationException = pivotConfig.validate(validationException); + } + validationException = settings.validate(validationException); + + return validationException; + } + + public boolean isValid() { + if (pivotConfig != null && pivotConfig.isValid() == false) { + return false; + } + + if (syncConfig != null && syncConfig.isValid() == false) { + return false; + } + + return settings.isValid() && source.isValid() && dest.isValid(); + } + + @Override + public void writeTo(final StreamOutput out) throws IOException { + out.writeString(id); + source.writeTo(out); + dest.writeTo(out); + if (out.getVersion().onOrAfter(Version.V_7_3_0)) { + out.writeOptionalTimeValue(frequency); + } + out.writeMap(headers, StreamOutput::writeString, StreamOutput::writeString); + out.writeOptionalWriteable(pivotConfig); + out.writeOptionalString(description); + if (out.getVersion().onOrAfter(Version.V_7_3_0)) { + out.writeOptionalNamedWriteable(syncConfig); + out.writeOptionalInstant(createTime); + if (transformVersion != null) { + out.writeBoolean(true); + Version.writeVersion(transformVersion, out); + } else { + out.writeBoolean(false); + } + } + if (out.getVersion().onOrAfter(Version.V_7_8_0)) { + settings.writeTo(out); + } + if (out.getVersion().onOrAfter(Version.V_8_0_0)) { // todo 7.9.0 + out.writeOptionalWriteable(mapConfig); + } + } + + @Override + public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException { + final boolean excludeGenerated = params.paramAsBoolean(TransformField.EXCLUDE_GENERATED, false); + final boolean forInternalStorage = params.paramAsBoolean(TransformField.FOR_INTERNAL_STORAGE, false); + assert (forInternalStorage && excludeGenerated) == false: + "unsupported behavior, exclude_generated is true and for_internal_storage is true"; + builder.startObject(); + builder.field(TransformField.ID.getPreferredName(), id); + if (excludeGenerated == false) { + if (headers.isEmpty() == false && forInternalStorage) { + builder.field(HEADERS.getPreferredName(), headers); + } + if (transformVersion != null) { + builder.field(TransformField.VERSION.getPreferredName(), transformVersion); + } + if (createTime != null) { + builder.timeField( + TransformField.CREATE_TIME.getPreferredName(), + TransformField.CREATE_TIME.getPreferredName() + "_string", + createTime.toEpochMilli() + ); + } + if (forInternalStorage) { + builder.field(TransformField.INDEX_DOC_TYPE.getPreferredName(), NAME); + } + } + builder.field(TransformField.SOURCE.getPreferredName(), source, params); + builder.field(TransformField.DESTINATION.getPreferredName(), dest); + if (frequency != null) { + builder.field(TransformField.FREQUENCY.getPreferredName(), frequency.getStringRep()); + } + if (syncConfig != null) { + builder.startObject(TransformField.SYNC.getPreferredName()); + builder.field(syncConfig.getWriteableName(), syncConfig); + builder.endObject(); + } + if (pivotConfig != null) { + builder.field(PIVOT_TRANSFORM.getPreferredName(), pivotConfig); + } +<<<<<<< HEAD +======= + if (mapConfig != null) { + builder.field(MAP_TRANSFORM.getPreferredName(), mapConfig); + } + if (params.paramAsBoolean(TransformField.FOR_INTERNAL_STORAGE, false)) { + builder.field(TransformField.INDEX_DOC_TYPE.getPreferredName(), NAME); + } + if (headers.isEmpty() == false && params.paramAsBoolean(TransformField.FOR_INTERNAL_STORAGE, false)) { + builder.field(HEADERS.getPreferredName(), headers); + } +>>>>>>> implement map as function + if (description != null) { + builder.field(TransformField.DESCRIPTION.getPreferredName(), description); + } + builder.field(TransformField.SETTINGS.getPreferredName(), settings); + builder.endObject(); + return builder; + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + + if (other == null || getClass() != other.getClass()) { + return false; + } + + final TransformConfig that = (TransformConfig) other; + + return Objects.equals(this.id, that.id) + && Objects.equals(this.source, that.source) + && Objects.equals(this.dest, that.dest) + && Objects.equals(this.frequency, that.frequency) + && Objects.equals(this.syncConfig, that.syncConfig) + && Objects.equals(this.headers, that.headers) + && Objects.equals(this.pivotConfig, that.pivotConfig) + && Objects.equals(this.mapConfig, that.mapConfig) + && Objects.equals(this.description, that.description) + && Objects.equals(this.settings, that.settings) + && Objects.equals(this.createTime, that.createTime) + && Objects.equals(this.transformVersion, that.transformVersion); + } + + @Override + public int hashCode() { + return Objects.hash( + id, + source, + dest, + frequency, + syncConfig, + headers, + pivotConfig, + mapConfig, + description, + settings, + createTime, + transformVersion + ); + } + + @Override + public String toString() { + return Strings.toString(this, true, true); + } + + public static TransformConfig fromXContent(final XContentParser parser, @Nullable final String optionalTransformId, boolean lenient) { + + return lenient ? LENIENT_PARSER.apply(parser, optionalTransformId) : STRICT_PARSER.apply(parser, optionalTransformId); + } + + /** + * Rewrites the transform config according to the latest format, for example moving deprecated + * settings to its new place. + * + * @param transformConfig original config + * @return a rewritten transform config if a rewrite was necessary, otherwise the given transformConfig + */ + public static TransformConfig rewriteForUpdate(final TransformConfig transformConfig) { + + // quick checks for deprecated features, if none found just return the original + if (transformConfig.getPivotConfig() == null || transformConfig.getPivotConfig().getMaxPageSearchSize() == null) { + return transformConfig; + } + + Builder builder = new Builder(transformConfig); + + if (transformConfig.getPivotConfig() != null && transformConfig.getPivotConfig().getMaxPageSearchSize() != null) { + // create a new pivot config but set maxPageSearchSize to null + PivotConfig newPivotConfig = new PivotConfig( + transformConfig.getPivotConfig().getGroupConfig(), + transformConfig.getPivotConfig().getAggregationConfig(), + null + ); + builder.setPivotConfig(newPivotConfig); + + Integer maxPageSearchSizeDeprecated = transformConfig.getPivotConfig().getMaxPageSearchSize(); + Integer maxPageSearchSize = transformConfig.getSettings().getMaxPageSearchSize() != null + ? transformConfig.getSettings().getMaxPageSearchSize() + : maxPageSearchSizeDeprecated; + + builder.setSettings(new SettingsConfig(maxPageSearchSize, transformConfig.getSettings().getDocsPerSecond())); + } + return builder.setVersion(Version.CURRENT).build(); + } + + public static class Builder { + private String id; + private SourceConfig source; + private DestConfig dest; + private TimeValue frequency; + private SyncConfig syncConfig; + private String description; + private Map headers; + private Version transformVersion; + private Instant createTime; + private PivotConfig pivotConfig; + private MapConfig mapConfig; + private SettingsConfig settings; + + public Builder() {} + + public Builder(TransformConfig config) { + this.id = config.id; + this.source = config.source; + this.dest = config.dest; + this.frequency = config.frequency; + this.syncConfig = config.syncConfig; + this.description = config.description; + this.transformVersion = config.transformVersion; + this.createTime = config.createTime; + this.pivotConfig = config.pivotConfig; + this.settings = config.settings; + } + + public Builder setId(String id) { + this.id = id; + return this; + } + + public Builder setSource(SourceConfig source) { + this.source = source; + return this; + } + + public Builder setDest(DestConfig dest) { + this.dest = dest; + return this; + } + + public Builder setFrequency(TimeValue frequency) { + this.frequency = frequency; + return this; + } + + public Builder setSyncConfig(SyncConfig syncConfig) { + this.syncConfig = syncConfig; + return this; + } + + public Builder setDescription(String description) { + this.description = description; + return this; + } + + public Builder setSettings(SettingsConfig settings) { + this.settings = settings; + return this; + } + + public Builder setHeaders(Map headers) { + this.headers = headers; + return this; + } + + public Builder setPivotConfig(PivotConfig pivotConfig) { + this.pivotConfig = pivotConfig; + return this; + } + + public Builder setMapConfig(MapConfig mapConfig) { + this.mapConfig = mapConfig; + return this; + } + + Builder setVersion(Version version) { + this.transformVersion = version; + return this; + } + + public TransformConfig build() { + return new TransformConfig( + id, + source, + dest, + frequency, + syncConfig, + headers, + pivotConfig, + mapConfig, + description, + settings, + createTime, + transformVersion == null ? null : transformVersion.toString() + ); + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + + if (other == null || getClass() != other.getClass()) { + return false; + } + + final TransformConfig.Builder that = (TransformConfig.Builder) other; + + return Objects.equals(this.id, that.id) + && Objects.equals(this.source, that.source) + && Objects.equals(this.dest, that.dest) + && Objects.equals(this.frequency, that.frequency) + && Objects.equals(this.syncConfig, that.syncConfig) + && Objects.equals(this.headers, that.headers) + && Objects.equals(this.pivotConfig, that.pivotConfig) + && Objects.equals(this.mapConfig, that.mapConfig) + && Objects.equals(this.description, that.description) + && Objects.equals(this.settings, that.settings) + && Objects.equals(this.createTime, that.createTime) + && Objects.equals(this.transformVersion, that.transformVersion); + } + + @Override + public int hashCode() { + return Objects.hash( + id, + source, + dest, + frequency, + syncConfig, + headers, + pivotConfig, + mapConfig, + description, + settings, + createTime, + transformVersion + ); + } + } +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/AbstractSerializingTransformTestCase.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/AbstractSerializingTransformTestCase.java similarity index 60% rename from x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/AbstractSerializingTransformTestCase.java rename to x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/AbstractSerializingTransformTestCase.java index 781098f1e7601..4bc86f862d75c 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/AbstractSerializingTransformTestCase.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/AbstractSerializingTransformTestCase.java @@ -5,9 +5,10 @@ * 2.0. */ -package org.elasticsearch.xpack.core.transform.action; +package org.elasticsearch.xpack.core.transform; import org.elasticsearch.Version; +import org.elasticsearch.common.ParseField; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; @@ -16,21 +17,29 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.ToXContent.Params; +import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.search.SearchModule; +import org.elasticsearch.search.aggregations.AggregationBuilder; +import org.elasticsearch.search.aggregations.BaseAggregationBuilder; import org.elasticsearch.test.AbstractSerializingTestCase; -import org.elasticsearch.xpack.core.transform.TransformField; -import org.elasticsearch.xpack.core.transform.TransformNamedXContentProvider; +import org.elasticsearch.xpack.core.transform.transforms.RetentionPolicyConfig; import org.elasticsearch.xpack.core.transform.transforms.SyncConfig; +import org.elasticsearch.xpack.core.transform.transforms.TimeRetentionPolicyConfig; import org.elasticsearch.xpack.core.transform.transforms.TimeSyncConfig; import org.junit.Before; import java.io.IOException; +import java.util.Collections; import java.util.List; import static java.util.Collections.emptyList; public abstract class AbstractSerializingTransformTestCase extends AbstractSerializingTestCase { + protected static Params TO_XCONTENT_PARAMS = new ToXContent.MapParams( + Collections.singletonMap(TransformField.FOR_INTERNAL_STORAGE, "true")); + private NamedWriteableRegistry namedWriteableRegistry; private NamedXContentRegistry namedXContentRegistry; @@ -40,10 +49,42 @@ public void registerNamedObjects() { List namedWriteables = searchModule.getNamedWriteables(); namedWriteables.add( - new NamedWriteableRegistry.Entry(SyncConfig.class, TransformField.TIME_BASED_SYNC.getPreferredName(), TimeSyncConfig::new) + new NamedWriteableRegistry.Entry(QueryBuilder.class, MockDeprecatedQueryBuilder.NAME, MockDeprecatedQueryBuilder::new) + ); + namedWriteables.add( + new NamedWriteableRegistry.Entry( + AggregationBuilder.class, + MockDeprecatedAggregationBuilder.NAME, + MockDeprecatedAggregationBuilder::new + ) + ); + namedWriteables.add( + new NamedWriteableRegistry.Entry(SyncConfig.class, TransformField.TIME.getPreferredName(), TimeSyncConfig::new) + ); + namedWriteables.add( + new NamedWriteableRegistry.Entry( + RetentionPolicyConfig.class, + TransformField.TIME.getPreferredName(), + TimeRetentionPolicyConfig::new + ) ); List namedXContents = searchModule.getNamedXContents(); + namedXContents.add( + new NamedXContentRegistry.Entry( + QueryBuilder.class, + new ParseField(MockDeprecatedQueryBuilder.NAME), + (p, c) -> MockDeprecatedQueryBuilder.fromXContent(p) + ) + ); + namedXContents.add( + new NamedXContentRegistry.Entry( + BaseAggregationBuilder.class, + new ParseField(MockDeprecatedAggregationBuilder.NAME), + (p, c) -> MockDeprecatedAggregationBuilder.fromXContent(p) + ) + ); + namedXContents.addAll(new TransformNamedXContentProvider().getNamedXContentParsers()); namedWriteableRegistry = new NamedWriteableRegistry(namedWriteables); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/MockDeprecatedAggregationBuilder.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/MockDeprecatedAggregationBuilder.java similarity index 98% rename from x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/MockDeprecatedAggregationBuilder.java rename to x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/MockDeprecatedAggregationBuilder.java index a5b295ab1cda6..e8bbc331a31e0 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/MockDeprecatedAggregationBuilder.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/MockDeprecatedAggregationBuilder.java @@ -5,7 +5,7 @@ * 2.0. */ -package org.elasticsearch.xpack.core.transform.transforms; +package org.elasticsearch.xpack.core.transform; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/MockDeprecatedQueryBuilder.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/MockDeprecatedQueryBuilder.java similarity index 98% rename from x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/MockDeprecatedQueryBuilder.java rename to x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/MockDeprecatedQueryBuilder.java index 8fc28cb0bc320..db5329e701940 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/MockDeprecatedQueryBuilder.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/MockDeprecatedQueryBuilder.java @@ -5,7 +5,7 @@ * 2.0. */ -package org.elasticsearch.xpack.core.transform.transforms; +package org.elasticsearch.xpack.core.transform; import org.apache.lucene.search.Query; import org.elasticsearch.common.ParsingException; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/AbstractWireSerializingTransformTestCase.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/AbstractWireSerializingTransformTestCase.java index cdb11d0b5308e..458e32070d4a3 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/AbstractWireSerializingTransformTestCase.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/AbstractWireSerializingTransformTestCase.java @@ -19,7 +19,9 @@ import org.elasticsearch.test.AbstractWireSerializingTestCase; import org.elasticsearch.xpack.core.transform.TransformField; import org.elasticsearch.xpack.core.transform.TransformNamedXContentProvider; +import org.elasticsearch.xpack.core.transform.transforms.RetentionPolicyConfig; import org.elasticsearch.xpack.core.transform.transforms.SyncConfig; +import org.elasticsearch.xpack.core.transform.transforms.TimeRetentionPolicyConfig; import org.elasticsearch.xpack.core.transform.transforms.TimeSyncConfig; import org.junit.Before; @@ -41,7 +43,10 @@ public void registerNamedObjects() { List namedWriteables = searchModule.getNamedWriteables(); namedWriteables.add( - new NamedWriteableRegistry.Entry(SyncConfig.class, TransformField.TIME_BASED_SYNC.getPreferredName(), TimeSyncConfig::new) + new NamedWriteableRegistry.Entry(SyncConfig.class, TransformField.TIME.getPreferredName(), TimeSyncConfig::new) + ); + namedWriteables.add( + new NamedWriteableRegistry.Entry(RetentionPolicyConfig.class, TransformField.TIME.getPreferredName(), TimeRetentionPolicyConfig::new) ); List namedXContents = searchModule.getNamedXContents(); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/PreviewTransformActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/PreviewTransformActionRequestTests.java index aaa8370e8e775..8f8891af7789b 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/PreviewTransformActionRequestTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/PreviewTransformActionRequestTests.java @@ -12,6 +12,7 @@ import org.elasticsearch.common.xcontent.DeprecationHandler; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.elasticsearch.xpack.core.transform.AbstractSerializingTransformTestCase; import org.elasticsearch.xpack.core.transform.action.PreviewTransformAction.Request; import org.elasticsearch.xpack.core.transform.transforms.DestConfig; import org.elasticsearch.xpack.core.transform.transforms.TransformConfig; @@ -53,6 +54,7 @@ protected Request createTestInstance() { null, null, null, + null, null ); return new Request(config); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/PutTransformActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/PutTransformActionRequestTests.java index b3704059acefb..d226c475e5625 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/PutTransformActionRequestTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/PutTransformActionRequestTests.java @@ -7,24 +7,13 @@ package org.elasticsearch.xpack.core.transform.action; -import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.search.SearchModule; -import org.elasticsearch.test.AbstractWireSerializingTestCase; -import org.elasticsearch.xpack.core.transform.TransformField; import org.elasticsearch.xpack.core.transform.action.PutTransformAction.Request; import org.elasticsearch.xpack.core.transform.transforms.TransformConfig; import org.elasticsearch.xpack.core.transform.transforms.TransformConfigTests; -import org.elasticsearch.xpack.core.transform.transforms.SyncConfig; -import org.elasticsearch.xpack.core.transform.transforms.TimeSyncConfig; import org.junit.Before; -import java.util.List; - -import static java.util.Collections.emptyList; - -public class PutTransformActionRequestTests extends AbstractWireSerializingTestCase { +public class PutTransformActionRequestTests extends AbstractWireSerializingTransformTestCase { private String transformId; @Before @@ -42,15 +31,4 @@ protected Request createTestInstance() { TransformConfig config = TransformConfigTests.randomTransformConfigWithoutHeaders(transformId); return new Request(config, randomBoolean()); } - - @Override - protected NamedWriteableRegistry getNamedWriteableRegistry() { - SearchModule searchModule = new SearchModule(Settings.EMPTY, emptyList()); - - List namedWriteables = searchModule.getNamedWriteables(); - namedWriteables.add( - new NamedWriteableRegistry.Entry(SyncConfig.class, TransformField.TIME_BASED_SYNC.getPreferredName(), TimeSyncConfig::new) - ); - return new NamedWriteableRegistry(namedWriteables); - } } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/UpdateTransformsActionResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/UpdateTransformsActionResponseTests.java index 9d91522a5a186..7b650e5b64522 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/UpdateTransformsActionResponseTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/UpdateTransformsActionResponseTests.java @@ -10,6 +10,7 @@ import org.elasticsearch.Version; import org.elasticsearch.common.io.stream.Writeable.Reader; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.xpack.core.transform.AbstractSerializingTransformTestCase; import org.elasticsearch.xpack.core.transform.action.UpdateTransformAction.Response; import org.elasticsearch.xpack.core.transform.action.compat.UpdateTransformActionPre78; import org.elasticsearch.xpack.core.transform.transforms.TransformConfig; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/AbstractSerializingTransformTestCase.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/AbstractSerializingTransformTestCase.java deleted file mode 100644 index 9c219b3f78f5a..0000000000000 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/AbstractSerializingTransformTestCase.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.core.transform.transforms; - -import org.elasticsearch.common.ParseField; -import org.elasticsearch.common.io.stream.NamedWriteableRegistry; -import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.xcontent.NamedXContentRegistry; -import org.elasticsearch.common.xcontent.ToXContent; -import org.elasticsearch.common.xcontent.ToXContent.Params; -import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.search.SearchModule; -import org.elasticsearch.search.aggregations.AggregationBuilder; -import org.elasticsearch.search.aggregations.BaseAggregationBuilder; -import org.elasticsearch.test.AbstractSerializingTestCase; -import org.elasticsearch.xpack.core.transform.TransformField; -import org.elasticsearch.xpack.core.transform.TransformNamedXContentProvider; -import org.junit.Before; - -import java.util.Collections; -import java.util.List; - -import static java.util.Collections.emptyList; - -public abstract class AbstractSerializingTransformTestCase - extends AbstractSerializingTestCase { - - protected static Params TO_XCONTENT_PARAMS = new ToXContent.MapParams( - Collections.singletonMap(TransformField.FOR_INTERNAL_STORAGE, "true")); - - /** - * Test case that ensures aggregation named objects are registered - */ - private NamedWriteableRegistry namedWriteableRegistry; - private NamedXContentRegistry namedXContentRegistry; - - @Before - public void registerAggregationNamedObjects() throws Exception { - // register aggregations as NamedWriteable - SearchModule searchModule = new SearchModule(Settings.EMPTY, emptyList()); - - List namedWriteables = searchModule.getNamedWriteables(); - namedWriteables.add(new NamedWriteableRegistry.Entry(QueryBuilder.class, MockDeprecatedQueryBuilder.NAME, - MockDeprecatedQueryBuilder::new)); - namedWriteables.add(new NamedWriteableRegistry.Entry(AggregationBuilder.class, MockDeprecatedAggregationBuilder.NAME, - MockDeprecatedAggregationBuilder::new)); - namedWriteables.add(new NamedWriteableRegistry.Entry(SyncConfig.class, TransformField.TIME_BASED_SYNC.getPreferredName(), - TimeSyncConfig::new)); - - List namedXContents = searchModule.getNamedXContents(); - namedXContents.add(new NamedXContentRegistry.Entry(QueryBuilder.class, - new ParseField(MockDeprecatedQueryBuilder.NAME), (p, c) -> MockDeprecatedQueryBuilder.fromXContent(p))); - namedXContents.add(new NamedXContentRegistry.Entry(BaseAggregationBuilder.class, - new ParseField(MockDeprecatedAggregationBuilder.NAME), (p, c) -> MockDeprecatedAggregationBuilder.fromXContent(p))); - namedXContents.addAll(new TransformNamedXContentProvider().getNamedXContentParsers()); - - namedWriteableRegistry = new NamedWriteableRegistry(namedWriteables); - namedXContentRegistry = new NamedXContentRegistry(namedXContents); - } - - @Override - protected NamedWriteableRegistry getNamedWriteableRegistry() { - return namedWriteableRegistry; - } - - @Override - protected NamedXContentRegistry xContentRegistry() { - return namedXContentRegistry; - } -} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/DestConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/DestConfigTests.java index 744ac287ed0c3..1925ebd1a4499 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/DestConfigTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/DestConfigTests.java @@ -9,6 +9,7 @@ import org.elasticsearch.common.io.stream.Writeable.Reader; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.xpack.core.transform.AbstractSerializingTransformTestCase; import org.junit.Before; import java.io.IOException; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/QueryConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/QueryConfigTests.java index 870196bbb9db0..427a25df233df 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/QueryConfigTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/QueryConfigTests.java @@ -21,6 +21,8 @@ import org.elasticsearch.index.query.MatchNoneQueryBuilder; import org.elasticsearch.index.query.MatchQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.xpack.core.transform.AbstractSerializingTransformTestCase; +import org.elasticsearch.xpack.core.transform.MockDeprecatedQueryBuilder; import org.junit.Before; import java.io.IOException; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/SettingsConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/SettingsConfigTests.java index c2e74bfc03871..273a5c817bc1d 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/SettingsConfigTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/SettingsConfigTests.java @@ -16,6 +16,7 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.elasticsearch.xpack.core.transform.AbstractSerializingTransformTestCase; import org.elasticsearch.xpack.core.watcher.watch.Payload.XContent; import org.junit.Before; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/SourceConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/SourceConfigTests.java index c77728dcc59e5..b61ef9d947939 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/SourceConfigTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/SourceConfigTests.java @@ -9,6 +9,7 @@ import org.elasticsearch.common.io.stream.Writeable.Reader; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.xpack.core.transform.AbstractSerializingTransformTestCase; import org.junit.Before; import java.io.IOException; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TimeRetentionPolicyConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TimeRetentionPolicyConfigTests.java new file mode 100644 index 0000000000000..677fd9da72fb8 --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TimeRetentionPolicyConfigTests.java @@ -0,0 +1,30 @@ +package org.elasticsearch.xpack.core.transform.transforms; + +import org.elasticsearch.common.io.stream.Writeable.Reader; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.test.AbstractSerializingTestCase; + +import java.io.IOException; + +public class TimeRetentionPolicyConfigTests extends AbstractSerializingTestCase { + + public static TimeRetentionPolicyConfig randomTimeRetentionPolicyConfig() { + return new TimeRetentionPolicyConfig(randomAlphaOfLengthBetween(1, 10), new TimeValue(randomNonNegativeLong())); + } + + @Override + protected TimeRetentionPolicyConfig doParseInstance(XContentParser parser) throws IOException { + return TimeRetentionPolicyConfig.fromXContent(parser, false); + } + + @Override + protected TimeRetentionPolicyConfig createTestInstance() { + return randomTimeRetentionPolicyConfig(); + } + + @Override + protected Reader instanceReader() { + return TimeRetentionPolicyConfig::new; + } +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformCheckpointStatsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformCheckpointStatsTests.java index c6c46ae99b15e..ef1ef1fa8c0e8 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformCheckpointStatsTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformCheckpointStatsTests.java @@ -9,6 +9,7 @@ import org.elasticsearch.common.io.stream.Writeable.Reader; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.xpack.core.transform.AbstractSerializingTransformTestCase; import java.io.IOException; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformCheckpointTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformCheckpointTests.java index a0d6c21702203..f84fd8d8b5ce0 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformCheckpointTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformCheckpointTests.java @@ -12,6 +12,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.xpack.core.transform.AbstractSerializingTransformTestCase; import java.io.IOException; import java.util.ArrayList; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformCheckpointingInfoTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformCheckpointingInfoTests.java index d150b6756410e..bd3e087e161a9 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformCheckpointingInfoTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformCheckpointingInfoTests.java @@ -12,6 +12,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.Writeable.Reader; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.xpack.core.transform.AbstractSerializingTransformTestCase; import java.io.IOException; import java.time.Instant; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigTests.java index 91404bc27aa71..212265741bab0 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigTests.java @@ -17,7 +17,9 @@ import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.xpack.core.transform.AbstractSerializingTransformTestCase; import org.elasticsearch.xpack.core.transform.transforms.latest.LatestConfig; +import org.elasticsearch.xpack.core.transform.transforms.latest.LatestConfigTests; import org.elasticsearch.xpack.core.transform.transforms.pivot.PivotConfig; import org.elasticsearch.xpack.core.transform.transforms.pivot.PivotConfigTests; import org.junit.Before; @@ -43,6 +45,16 @@ public static TransformConfig randomTransformConfigWithoutHeaders() { } public static TransformConfig randomTransformConfigWithoutHeaders(String id) { + PivotConfig pivotConfig; + LatestConfig latestConfig; + if (randomBoolean()) { + pivotConfig = PivotConfigTests.randomPivotConfig(); + latestConfig = null; + } else { + pivotConfig = null; + latestConfig = LatestConfigTests.randomLatestConfig(); + } + return new TransformConfig( id, randomSourceConfig(), @@ -50,10 +62,11 @@ public static TransformConfig randomTransformConfigWithoutHeaders(String id) { randomBoolean() ? null : TimeValue.timeValueMillis(randomIntBetween(1_000, 3_600_000)), randomBoolean() ? null : randomSyncConfig(), null, - PivotConfigTests.randomPivotConfig(Version.CURRENT), - null, + pivotConfig, + latestConfig, randomBoolean() ? null : randomAlphaOfLengthBetween(1, 1000), SettingsConfigTests.randomSettingsConfig(), + randomBoolean() ? null : randomRetentionPolicyConfig(), null, null ); @@ -64,7 +77,17 @@ public static TransformConfig randomTransformConfig() { } public static TransformConfig randomTransformConfig(String id) { - return randomTransformConfig(id, PivotConfigTests.randomPivotConfig(Version.CURRENT), null); + PivotConfig pivotConfig; + LatestConfig latestConfig; + if (randomBoolean()) { + pivotConfig = PivotConfigTests.randomPivotConfig(); + latestConfig = null; + } else { + pivotConfig = null; + latestConfig = LatestConfigTests.randomLatestConfig(); + } + + return randomTransformConfig(id, pivotConfig, latestConfig); } public static TransformConfig randomTransformConfig(String id, PivotConfig pivotConfig, LatestConfig latestConfig) { @@ -79,6 +102,7 @@ public static TransformConfig randomTransformConfig(String id, PivotConfig pivot latestConfig, randomBoolean() ? null : randomAlphaOfLengthBetween(1, 1000), randomBoolean() ? null : SettingsConfigTests.randomSettingsConfig(), + randomBoolean() ? null : randomRetentionPolicyConfig(), randomBoolean() ? null : Instant.now(), randomBoolean() ? null : Version.CURRENT.toString() ); @@ -86,6 +110,16 @@ public static TransformConfig randomTransformConfig(String id, PivotConfig pivot public static TransformConfig randomInvalidTransformConfig() { if (randomBoolean()) { + PivotConfig pivotConfig; + LatestConfig latestConfig; + if (randomBoolean()) { + pivotConfig = PivotConfigTests.randomPivotConfig(); + latestConfig = null; + } else { + pivotConfig = null; + latestConfig = LatestConfigTests.randomLatestConfig(); + } + return new TransformConfig( randomAlphaOfLengthBetween(1, 10), randomInvalidSourceConfig(), @@ -93,10 +127,11 @@ public static TransformConfig randomInvalidTransformConfig() { null, randomBoolean() ? randomSyncConfig() : null, randomHeaders(), - PivotConfigTests.randomPivotConfig(), - null, + pivotConfig, + latestConfig, randomBoolean() ? null : randomAlphaOfLengthBetween(1, 1000), null, + randomBoolean() ? null : randomRetentionPolicyConfig(), null, null ); @@ -112,6 +147,7 @@ public static TransformConfig randomInvalidTransformConfig() { null, randomBoolean() ? null : randomAlphaOfLengthBetween(1, 1000), null, + randomBoolean() ? null : randomRetentionPolicyConfig(), null, null ); @@ -121,6 +157,10 @@ public static SyncConfig randomSyncConfig() { return TimeSyncConfigTests.randomTimeSyncConfig(); } + public static RetentionPolicyConfig randomRetentionPolicyConfig() { + return TimeRetentionPolicyConfigTests.randomTimeRetentionPolicyConfig(); + } + @Before public void setUpOptionalId() { transformId = randomAlphaOfLengthBetween(1, 10); @@ -187,10 +227,7 @@ public void testDefaultMatchAll() throws IOException { } public void testConstructor_NoFunctionProvided() throws IOException { - String json = "{" - + " \"source\": {\"index\": \"src\"}," - + " \"dest\": {\"index\": \"dest\"}" - + "}"; + String json = "{" + " \"source\": {\"index\": \"src\"}," + " \"dest\": {\"index\": \"dest\"}" + "}"; // Should parse with lenient parser createTransformConfigFromString(json, "dummy", true); @@ -319,6 +356,7 @@ public void testMaxLengthDescription() { randomAlphaOfLength(1001), null, null, + null, null ) ); @@ -336,6 +374,7 @@ public void testMaxLengthDescription() { description, null, null, + null, null ); assertThat(description, equalTo(config.getDescription())); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigUpdateTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigUpdateTests.java index 341824b624c7d..bb0705355d96b 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigUpdateTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigUpdateTests.java @@ -42,10 +42,6 @@ public static TransformConfigUpdate randomTransformConfigUpdate() { ); } - public static SyncConfig randomSyncConfig() { - return TimeSyncConfigTests.randomTimeSyncConfig(); - } - @Override protected TransformConfigUpdate createTestInstance() { return randomTransformConfigUpdate(); @@ -89,12 +85,13 @@ public void testApply() { randomSourceConfig(), randomDestConfig(), TimeValue.timeValueMillis(randomIntBetween(1_000, 3_600_000)), - TimeSyncConfigTests.randomTimeSyncConfig(), + randomSyncConfig(), Collections.singletonMap("key", "value"), PivotConfigTests.randomPivotConfig(), null, randomBoolean() ? null : randomAlphaOfLengthBetween(1, 1000), SettingsConfigTests.randomNonEmptySettingsConfig(), + randomRetentionPolicyConfig(), randomBoolean() ? null : Instant.now(), randomBoolean() ? null : Version.V_7_2_0.toString() ); @@ -129,12 +126,13 @@ public void testApplySettings() { randomSourceConfig(), randomDestConfig(), TimeValue.timeValueMillis(randomIntBetween(1_000, 3_600_000)), - TimeSyncConfigTests.randomTimeSyncConfig(), + randomSyncConfig(), Collections.singletonMap("key", "value"), PivotConfigTests.randomPivotConfig(), null, randomBoolean() ? null : randomAlphaOfLengthBetween(1, 1000), SettingsConfigTests.randomNonEmptySettingsConfig(), + randomRetentionPolicyConfig(), randomBoolean() ? null : Instant.now(), randomBoolean() ? null : Version.V_7_2_0.toString() ); @@ -183,6 +181,7 @@ public void testApplyWithSyncChange() { null, randomBoolean() ? null : randomAlphaOfLengthBetween(1, 1000), SettingsConfigTests.randomNonEmptySettingsConfig(), + randomRetentionPolicyConfig(), randomBoolean() ? null : Instant.now(), randomBoolean() ? null : Version.CURRENT.toString() ); @@ -200,12 +199,13 @@ public void testApplyWithSyncChange() { randomSourceConfig(), randomDestConfig(), TimeValue.timeValueMillis(randomIntBetween(1_000, 3_600_000)), - TimeSyncConfigTests.randomTimeSyncConfig(), + randomSyncConfig(), null, PivotConfigTests.randomPivotConfig(), null, randomBoolean() ? null : randomAlphaOfLengthBetween(1, 1000), SettingsConfigTests.randomNonEmptySettingsConfig(), + randomRetentionPolicyConfig(), randomBoolean() ? null : Instant.now(), randomBoolean() ? null : Version.CURRENT.toString() ); @@ -254,6 +254,14 @@ private void toXContent(TransformConfigUpdate update, XContentBuilder builder) t builder.endObject(); } + private static SyncConfig randomSyncConfig() { + return TimeSyncConfigTests.randomTimeSyncConfig(); + } + + private static RetentionPolicyConfig randomRetentionPolicyConfig() { + return TimeRetentionPolicyConfigTests.randomTimeRetentionPolicyConfig(); + } + static class FooSync implements SyncConfig { @Override diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformDestIndexSettingsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformDestIndexSettingsTests.java index 93e8f0b2e0bb0..8af85cf69913e 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformDestIndexSettingsTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformDestIndexSettingsTests.java @@ -11,6 +11,7 @@ import org.elasticsearch.common.io.stream.Writeable.Reader; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.xpack.core.transform.AbstractSerializingTransformTestCase; import java.io.IOException; import java.util.HashMap; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformProgressTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformProgressTests.java index 8c129d67822ec..d4165c682884a 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformProgressTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformProgressTests.java @@ -12,6 +12,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.Writeable.Reader; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.xpack.core.transform.AbstractSerializingTransformTestCase; import java.io.IOException; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformStoredDocTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformStoredDocTests.java index 83b0c4ac43d40..6c9e71e2b7844 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformStoredDocTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformStoredDocTests.java @@ -10,6 +10,7 @@ import org.elasticsearch.common.io.stream.Writeable.Reader; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.xpack.core.transform.AbstractSerializingTransformTestCase; import org.elasticsearch.xpack.core.transform.TransformField; import java.io.IOException; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformTests.java index e43bb82f98627..fb6831456b3e3 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformTests.java @@ -13,6 +13,7 @@ import org.elasticsearch.common.io.stream.Writeable.Reader; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.xpack.core.transform.AbstractSerializingTransformTestCase; import java.io.IOException; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/latest/LatestConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/latest/LatestConfigTests.java index 6946b916b8465..c0a8114ba9628 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/latest/LatestConfigTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/latest/LatestConfigTests.java @@ -13,7 +13,7 @@ import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.search.sort.SortBuilders; import org.elasticsearch.search.sort.SortOrder; -import org.elasticsearch.xpack.core.transform.transforms.AbstractSerializingTransformTestCase; +import org.elasticsearch.xpack.core.transform.AbstractSerializingTransformTestCase; import java.io.IOException; import java.util.ArrayList; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/pivot/AggregationConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/pivot/AggregationConfigTests.java index 4685c4621410a..1811b9916cdc0 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/pivot/AggregationConfigTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/pivot/AggregationConfigTests.java @@ -20,8 +20,8 @@ import org.elasticsearch.search.aggregations.AggregationBuilder; import org.elasticsearch.search.aggregations.AggregationBuilders; import org.elasticsearch.search.aggregations.AggregatorFactories; -import org.elasticsearch.xpack.core.transform.transforms.AbstractSerializingTransformTestCase; -import org.elasticsearch.xpack.core.transform.transforms.MockDeprecatedAggregationBuilder; +import org.elasticsearch.xpack.core.transform.AbstractSerializingTransformTestCase; +import org.elasticsearch.xpack.core.transform.MockDeprecatedAggregationBuilder; import org.junit.Before; import java.io.IOException; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/pivot/PivotConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/pivot/PivotConfigTests.java index fff66deb8460f..77dad8f3fc1fc 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/pivot/PivotConfigTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/pivot/PivotConfigTests.java @@ -13,7 +13,7 @@ import org.elasticsearch.common.xcontent.DeprecationHandler; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.xpack.core.transform.transforms.AbstractSerializingTransformTestCase; +import org.elasticsearch.xpack.core.transform.AbstractSerializingTransformTestCase; import java.io.IOException; import java.util.Arrays; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/pivot/ScriptConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/pivot/ScriptConfigTests.java index 7d5851ea416b5..d27741707b7bf 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/pivot/ScriptConfigTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/pivot/ScriptConfigTests.java @@ -19,7 +19,7 @@ import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptType; -import org.elasticsearch.xpack.core.transform.transforms.AbstractSerializingTransformTestCase; +import org.elasticsearch.xpack.core.transform.AbstractSerializingTransformTestCase; import org.junit.Before; import java.io.IOException; From efd718c972855861d8e98794489f8bbc894e704e Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Thu, 21 Jan 2021 16:39:13 +0100 Subject: [PATCH 02/39] apply spotless --- .../transforms/TimeRetentionPolicyConfig.java | 1 - .../TransformNamedXContentProvider.java | 10 +--- .../transform/transforms/TransformConfig.java | 4 +- .../AbstractSerializingTransformTestCase.java | 3 +- .../MockDeprecatedAggregationBuilder.java | 20 ++++--- .../transform/MockDeprecatedQueryBuilder.java | 6 +- ...tractWireSerializingTransformTestCase.java | 6 +- .../UpdateTransformsActionResponseTests.java | 56 ++++++++++++------- .../transform/transforms/DestConfigTests.java | 3 +- .../transforms/QueryConfigTests.java | 2 +- .../transforms/SourceConfigTests.java | 47 +++++++++++----- .../transforms/TransformProgressTests.java | 6 +- .../transform/transforms/TransformTests.java | 8 ++- .../transforms/latest/LatestConfigTests.java | 39 ++++--------- .../pivot/AggregationConfigTests.java | 28 +++++----- 15 files changed, 134 insertions(+), 105 deletions(-) diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/TimeRetentionPolicyConfig.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/TimeRetentionPolicyConfig.java index 78e13bd51cf9d..54dd7af4fb4c8 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/TimeRetentionPolicyConfig.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/TimeRetentionPolicyConfig.java @@ -11,7 +11,6 @@ import java.util.Objects; import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; -import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg; public class TimeRetentionPolicyConfig implements RetentionPolicyConfig { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformNamedXContentProvider.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformNamedXContentProvider.java index e4611b04fdad4..447575c4de9f4 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformNamedXContentProvider.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformNamedXContentProvider.java @@ -22,12 +22,8 @@ public class TransformNamedXContentProvider implements NamedXContentProvider { @Override public List getNamedXContentParsers() { return Arrays.asList( - new NamedXContentRegistry.Entry(SyncConfig.class, - TransformField.TIME, - TimeSyncConfig::parse), - new NamedXContentRegistry.Entry(RetentionPolicyConfig.class, - TransformField.TIME, - TimeRetentionPolicyConfig::parse) - ); + new NamedXContentRegistry.Entry(SyncConfig.class, TransformField.TIME, TimeSyncConfig::parse), + new NamedXContentRegistry.Entry(RetentionPolicyConfig.class, TransformField.TIME, TimeRetentionPolicyConfig::parse) + ); } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java index b4577ca8985fa..9a4ad942a08d0 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java @@ -378,8 +378,8 @@ public void writeTo(final StreamOutput out) throws IOException { public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException { final boolean excludeGenerated = params.paramAsBoolean(TransformField.EXCLUDE_GENERATED, false); final boolean forInternalStorage = params.paramAsBoolean(TransformField.FOR_INTERNAL_STORAGE, false); - assert (forInternalStorage - && excludeGenerated) == false : "unsupported behavior, exclude_generated is true and for_internal_storage is true"; + assert (forInternalStorage && excludeGenerated) == false + : "unsupported behavior, exclude_generated is true and for_internal_storage is true"; builder.startObject(); builder.field(TransformField.ID.getPreferredName(), id); if (excludeGenerated == false) { diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/AbstractSerializingTransformTestCase.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/AbstractSerializingTransformTestCase.java index 4bc86f862d75c..2bab028adf052 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/AbstractSerializingTransformTestCase.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/AbstractSerializingTransformTestCase.java @@ -38,7 +38,8 @@ public abstract class AbstractSerializingTransformTestCase extends AbstractSerializingTestCase { protected static Params TO_XCONTENT_PARAMS = new ToXContent.MapParams( - Collections.singletonMap(TransformField.FOR_INTERNAL_STORAGE, "true")); + Collections.singletonMap(TransformField.FOR_INTERNAL_STORAGE, "true") + ); private NamedWriteableRegistry namedWriteableRegistry; private NamedXContentRegistry namedXContentRegistry; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/MockDeprecatedAggregationBuilder.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/MockDeprecatedAggregationBuilder.java index e8bbc331a31e0..47ee429eaa391 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/MockDeprecatedAggregationBuilder.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/MockDeprecatedAggregationBuilder.java @@ -34,8 +34,11 @@ public class MockDeprecatedAggregationBuilder extends ValuesSourceAggregationBui private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(MockDeprecatedAggregationBuilder.class); - protected MockDeprecatedAggregationBuilder(MockDeprecatedAggregationBuilder clone, Builder factoriesBuilder, - Map metadata) { + protected MockDeprecatedAggregationBuilder( + MockDeprecatedAggregationBuilder clone, + Builder factoriesBuilder, + Map metadata + ) { super(clone, factoriesBuilder, metadata); } @@ -71,8 +74,7 @@ protected ValuesSourceRegistry.RegistryKey getRegistryKey() { } @Override - protected void innerWriteTo(StreamOutput out) throws IOException { - } + protected void innerWriteTo(StreamOutput out) throws IOException {} @Override public BucketCardinality bucketCardinality() { @@ -80,10 +82,12 @@ public BucketCardinality bucketCardinality() { } @Override - protected ValuesSourceAggregatorFactory innerBuild(AggregationContext context, - ValuesSourceConfig config, - AggregatorFactory parent, - Builder subFactoriesBuilder) throws IOException { + protected ValuesSourceAggregatorFactory innerBuild( + AggregationContext context, + ValuesSourceConfig config, + AggregatorFactory parent, + Builder subFactoriesBuilder + ) throws IOException { return null; } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/MockDeprecatedQueryBuilder.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/MockDeprecatedQueryBuilder.java index db5329e701940..622f0d00c6a77 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/MockDeprecatedQueryBuilder.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/MockDeprecatedQueryBuilder.java @@ -38,8 +38,7 @@ public class MockDeprecatedQueryBuilder extends AbstractQueryBuilder namedXContents = searchModule.getNamedXContents(); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/UpdateTransformsActionResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/UpdateTransformsActionResponseTests.java index 7b650e5b64522..db8f8c66394c8 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/UpdateTransformsActionResponseTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/UpdateTransformsActionResponseTests.java @@ -42,9 +42,7 @@ protected Response doParseInstance(XContentParser parser) throws IOException { } public void testBWCPre78() throws IOException { - Response newResponse = new Response( - TransformConfigTests.randomTransformConfigWithoutHeaders(randomAlphaOfLengthBetween(1, 10)) - ); + Response newResponse = new Response(TransformConfigTests.randomTransformConfigWithoutHeaders(randomAlphaOfLengthBetween(1, 10))); UpdateTransformActionPre78.Response oldResponse = writeAndReadBWCObject( newResponse, getNamedWriteableRegistry(), @@ -59,12 +57,19 @@ public void testBWCPre78() throws IOException { assertEquals(newResponse.getConfig().getFrequency(), oldResponse.getConfig().getFrequency()); assertEquals( newResponse.getConfig().getPivotConfig().getGroupConfig().getGroups().keySet(), - oldResponse.getConfig().getPivotConfig().getGroupConfig().getGroups().keySet()); - for (Map.Entry oldResponseGroupEntry - : oldResponse.getConfig().getPivotConfig().getGroupConfig().getGroups().entrySet()) { + oldResponse.getConfig().getPivotConfig().getGroupConfig().getGroups().keySet() + ); + for (Map.Entry oldResponseGroupEntry : oldResponse.getConfig() + .getPivotConfig() + .getGroupConfig() + .getGroups() + .entrySet()) { SingleGroupSource oldResponseGroup = oldResponseGroupEntry.getValue(); - SingleGroupSource newResponseGroup = - newResponse.getConfig().getPivotConfig().getGroupConfig().getGroups().get(oldResponseGroupEntry.getKey()); + SingleGroupSource newResponseGroup = newResponse.getConfig() + .getPivotConfig() + .getGroupConfig() + .getGroups() + .get(oldResponseGroupEntry.getKey()); assertThat(oldResponseGroup.getField(), is(equalTo(newResponseGroup.getField()))); assertThat(oldResponseGroup.getScriptConfig(), is(equalTo(newResponseGroup.getScriptConfig()))); // missing_bucket was added in 7.10 so it is always false after deserializing from 7.7 @@ -72,15 +77,18 @@ public void testBWCPre78() throws IOException { } assertEquals( newResponse.getConfig().getPivotConfig().getAggregationConfig(), - oldResponse.getConfig().getPivotConfig().getAggregationConfig()); + oldResponse.getConfig().getPivotConfig().getAggregationConfig() + ); assertEquals( newResponse.getConfig().getPivotConfig().getMaxPageSearchSize(), - oldResponse.getConfig().getPivotConfig().getMaxPageSearchSize()); + oldResponse.getConfig().getPivotConfig().getMaxPageSearchSize() + ); if (newResponse.getConfig().getSource() != null) { assertThat(newResponse.getConfig().getSource().getIndex(), is(equalTo(newResponse.getConfig().getSource().getIndex()))); assertThat( newResponse.getConfig().getSource().getQueryConfig(), - is(equalTo(newResponse.getConfig().getSource().getQueryConfig()))); + is(equalTo(newResponse.getConfig().getSource().getQueryConfig())) + ); // runtime_mappings was added in 7.12 so it is always empty after deserializing from 7.7 assertThat(oldResponse.getConfig().getSource().getRuntimeMappings(), is(anEmptyMap())); } @@ -103,12 +111,19 @@ public void testBWCPre78() throws IOException { assertEquals(newResponse.getConfig().getFrequency(), newRequestFromOld.getConfig().getFrequency()); assertEquals( newResponse.getConfig().getPivotConfig().getGroupConfig().getGroups().keySet(), - newRequestFromOld.getConfig().getPivotConfig().getGroupConfig().getGroups().keySet()); - for (Map.Entry newRequestFromOldGroupEntry - : newRequestFromOld.getConfig().getPivotConfig().getGroupConfig().getGroups().entrySet()) { + newRequestFromOld.getConfig().getPivotConfig().getGroupConfig().getGroups().keySet() + ); + for (Map.Entry newRequestFromOldGroupEntry : newRequestFromOld.getConfig() + .getPivotConfig() + .getGroupConfig() + .getGroups() + .entrySet()) { SingleGroupSource newRequestFromOldGroup = newRequestFromOldGroupEntry.getValue(); - SingleGroupSource newResponseGroup = - newResponse.getConfig().getPivotConfig().getGroupConfig().getGroups().get(newRequestFromOldGroupEntry.getKey()); + SingleGroupSource newResponseGroup = newResponse.getConfig() + .getPivotConfig() + .getGroupConfig() + .getGroups() + .get(newRequestFromOldGroupEntry.getKey()); assertThat(newRequestFromOldGroup.getField(), is(equalTo(newResponseGroup.getField()))); assertThat(newRequestFromOldGroup.getScriptConfig(), is(equalTo(newResponseGroup.getScriptConfig()))); // missing_bucket was added in 7.10 so it is always false after deserializing from 7.7 @@ -116,15 +131,18 @@ public void testBWCPre78() throws IOException { } assertEquals( newResponse.getConfig().getPivotConfig().getAggregationConfig(), - newRequestFromOld.getConfig().getPivotConfig().getAggregationConfig()); + newRequestFromOld.getConfig().getPivotConfig().getAggregationConfig() + ); assertEquals( newResponse.getConfig().getPivotConfig().getMaxPageSearchSize(), - newRequestFromOld.getConfig().getPivotConfig().getMaxPageSearchSize()); + newRequestFromOld.getConfig().getPivotConfig().getMaxPageSearchSize() + ); if (newResponse.getConfig().getSource() != null) { assertThat(newRequestFromOld.getConfig().getSource().getIndex(), is(equalTo(newResponse.getConfig().getSource().getIndex()))); assertThat( newRequestFromOld.getConfig().getSource().getQueryConfig(), - is(equalTo(newResponse.getConfig().getSource().getQueryConfig()))); + is(equalTo(newResponse.getConfig().getSource().getQueryConfig())) + ); // runtime_mappings was added in 7.12 so it is always empty after deserializing from 7.7 assertThat(newRequestFromOld.getConfig().getSource().getRuntimeMappings(), is(anEmptyMap())); } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/DestConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/DestConfigTests.java index 1925ebd1a4499..1697311da2de1 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/DestConfigTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/DestConfigTests.java @@ -19,8 +19,7 @@ public class DestConfigTests extends AbstractSerializingTransformTestCase) XContentHelper.convertToMap(BytesReference.bytes(content), true, XContentType.JSON) - .v2(); + .v2(); } catch (IOException e) { // should not happen fail("failed to create random query config"); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/SourceConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/SourceConfigTests.java index b61ef9d947939..cca201acfb2dc 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/SourceConfigTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/SourceConfigTests.java @@ -32,7 +32,8 @@ public static SourceConfig randomSourceConfig() { return new SourceConfig( generateRandomStringArray(10, 10, false, false), QueryConfigTests.randomQueryConfig(), - randomRuntimeMappings()); + randomRuntimeMappings() + ); } public static SourceConfig randomInvalidSourceConfig() { @@ -40,7 +41,8 @@ public static SourceConfig randomInvalidSourceConfig() { return new SourceConfig( generateRandomStringArray(10, 10, false, false), QueryConfigTests.randomInvalidQueryConfig(), - randomRuntimeMappings()); + randomRuntimeMappings() + ); } private static Map randomRuntimeMappings() { @@ -112,19 +114,38 @@ public void testGetRuntimeMappings_NonEmptyRuntimeMappings() { } public void testRequiresRemoteCluster() { - assertFalse(new SourceConfig(new String [] {"index1", "index2", "index3"}, - QueryConfigTests.randomQueryConfig(), randomRuntimeMappings()).requiresRemoteCluster()); + assertFalse( + new SourceConfig(new String[] { "index1", "index2", "index3" }, QueryConfigTests.randomQueryConfig(), randomRuntimeMappings()) + .requiresRemoteCluster() + ); - assertTrue(new SourceConfig(new String [] {"index1", "remote2:index2", "index3"}, - QueryConfigTests.randomQueryConfig(), randomRuntimeMappings()).requiresRemoteCluster()); - - assertTrue(new SourceConfig(new String [] {"index1", "index2", "remote3:index3"}, - QueryConfigTests.randomQueryConfig(), randomRuntimeMappings()).requiresRemoteCluster()); + assertTrue( + new SourceConfig( + new String[] { "index1", "remote2:index2", "index3" }, + QueryConfigTests.randomQueryConfig(), + randomRuntimeMappings() + ).requiresRemoteCluster() + ); - assertTrue(new SourceConfig(new String [] {"index1", "remote2:index2", "remote3:index3"}, - QueryConfigTests.randomQueryConfig(), randomRuntimeMappings()).requiresRemoteCluster()); + assertTrue( + new SourceConfig( + new String[] { "index1", "index2", "remote3:index3" }, + QueryConfigTests.randomQueryConfig(), + randomRuntimeMappings() + ).requiresRemoteCluster() + ); - assertTrue(new SourceConfig(new String [] {"remote1:index1"}, - QueryConfigTests.randomQueryConfig(), randomRuntimeMappings()).requiresRemoteCluster()); + assertTrue( + new SourceConfig( + new String[] { "index1", "remote2:index2", "remote3:index3" }, + QueryConfigTests.randomQueryConfig(), + randomRuntimeMappings() + ).requiresRemoteCluster() + ); + + assertTrue( + new SourceConfig(new String[] { "remote1:index1" }, QueryConfigTests.randomQueryConfig(), randomRuntimeMappings()) + .requiresRemoteCluster() + ); } } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformProgressTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformProgressTests.java index d4165c682884a..541f1c8fd7639 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformProgressTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformProgressTests.java @@ -27,7 +27,8 @@ public static TransformProgress randomTransformProgress() { return new TransformProgress( randomBoolean() ? null : randomLongBetween(0, 10000), randomBoolean() ? null : randomLongBetween(0, 10000), - randomBoolean() ? null : randomLongBetween(1, 10000)); + randomBoolean() ? null : randomLongBetween(1, 10000) + ); } @Override @@ -66,8 +67,7 @@ public void testPercentComplete() { } public void testConstructor() { - IllegalArgumentException ex = - expectThrows(IllegalArgumentException.class, () -> new TransformProgress(-1L, null, null)); + IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () -> new TransformProgress(-1L, null, null)); assertThat(ex.getMessage(), equalTo("[total_docs] must be >0.")); ex = expectThrows(IllegalArgumentException.class, () -> new TransformProgress(1L, -1L, null)); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformTests.java index fb6831456b3e3..ba8969f81b7f9 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformTests.java @@ -28,8 +28,12 @@ protected TransformTaskParams doParseInstance(XContentParser parser) throws IOEx @Override protected TransformTaskParams createTestInstance() { - return new TransformTaskParams(randomAlphaOfLength(10), randomBoolean() ? null : Version.CURRENT, - randomBoolean() ? null : TimeValue.timeValueMillis(randomIntBetween(1_000, 3_600_000)), randomBoolean()); + return new TransformTaskParams( + randomAlphaOfLength(10), + randomBoolean() ? null : Version.CURRENT, + randomBoolean() ? null : TimeValue.timeValueMillis(randomIntBetween(1_000, 3_600_000)), + randomBoolean() + ); } @Override diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/latest/LatestConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/latest/LatestConfigTests.java index c0a8114ba9628..19a33964dc769 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/latest/LatestConfigTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/latest/LatestConfigTests.java @@ -49,10 +49,7 @@ protected Reader instanceReader() { } public void testValidate_ValidConfig() throws IOException { - String json = "{" - + " \"unique_key\": [ \"event1\", \"event2\", \"event3\" ]," - + " \"sort\": \"timestamp\"" - + "}"; + String json = "{" + " \"unique_key\": [ \"event1\", \"event2\", \"event3\" ]," + " \"sort\": \"timestamp\"" + "}"; LatestConfig config = createLatestConfigFromString(json); assertThat(config.validate(null), is(nullValue())); @@ -62,59 +59,47 @@ public void testValidate_ValidConfig() throws IOException { } public void testValidate_EmptyUniqueKey() throws IOException { - String json = "{" - + " \"unique_key\": []," - + " \"sort\": \"timestamp\"" - + "}"; + String json = "{" + " \"unique_key\": []," + " \"sort\": \"timestamp\"" + "}"; LatestConfig config = createLatestConfigFromString(json); assertThat(config.validate(null).validationErrors(), contains("latest.unique_key must be non-empty")); } public void testValidate_EmptyUniqueKeyElement() throws IOException { - String json = "{" - + " \"unique_key\": [ \"event1\", \"\", \"event2\", \"\", \"event3\" ]," - + " \"sort\": \"timestamp\"" - + "}"; + String json = "{" + " \"unique_key\": [ \"event1\", \"\", \"event2\", \"\", \"event3\" ]," + " \"sort\": \"timestamp\"" + "}"; LatestConfig config = createLatestConfigFromString(json); assertThat( config.validate(null).validationErrors(), - containsInAnyOrder("latest.unique_key[1] element must be non-empty", "latest.unique_key[3] element must be non-empty")); + containsInAnyOrder("latest.unique_key[1] element must be non-empty", "latest.unique_key[3] element must be non-empty") + ); } public void testValidate_DuplicateUniqueKeyElement() throws IOException { - String json = "{" - + " \"unique_key\": [ \"event1\", \"event2\", \"event1\" ]," - + " \"sort\": \"timestamp\"" - + "}"; + String json = "{" + " \"unique_key\": [ \"event1\", \"event2\", \"event1\" ]," + " \"sort\": \"timestamp\"" + "}"; LatestConfig config = createLatestConfigFromString(json); assertThat( config.validate(null).validationErrors(), - containsInAnyOrder("latest.unique_key elements must be unique, found duplicate element [event1]")); + containsInAnyOrder("latest.unique_key elements must be unique, found duplicate element [event1]") + ); } public void testValidate_EmptySort() throws IOException { - String json = "{" - + " \"unique_key\": [ \"event1\", \"event2\", \"event3\" ]," - + " \"sort\": \"\"" - + "}"; + String json = "{" + " \"unique_key\": [ \"event1\", \"event2\", \"event3\" ]," + " \"sort\": \"\"" + "}"; LatestConfig config = createLatestConfigFromString(json); assertThat(config.validate(null).validationErrors(), contains("latest.sort must be non-empty")); } public void testValidate_EmptyUniqueKeyAndSort() throws IOException { - String json = "{" - + " \"unique_key\": []," - + " \"sort\": \"\"" - + "}"; + String json = "{" + " \"unique_key\": []," + " \"sort\": \"\"" + "}"; LatestConfig config = createLatestConfigFromString(json); assertThat( config.validate(null).validationErrors(), - containsInAnyOrder("latest.unique_key must be non-empty", "latest.sort must be non-empty")); + containsInAnyOrder("latest.unique_key must be non-empty", "latest.sort must be non-empty") + ); } private LatestConfig createLatestConfigFromString(String json) throws IOException { diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/pivot/AggregationConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/pivot/AggregationConfigTests.java index 1811b9916cdc0..84f0e02fb8a3b 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/pivot/AggregationConfigTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/pivot/AggregationConfigTests.java @@ -105,12 +105,12 @@ public void testEmptyAggregation() throws IOException { } public void testFailOnStrictPassOnLenient() throws IOException { - String source = "{\n" + - " \"avg_rating\": { \"some_removed_agg\": { \"field\": \"rating\" } }\n" + - " },\n" + - " {\n" + - " \"max_rating\": { \"max_rating\" : { \"field\" : \"rating\" } }\n" + - " }"; + String source = "{\n" + + " \"avg_rating\": { \"some_removed_agg\": { \"field\": \"rating\" } }\n" + + " },\n" + + " {\n" + + " \"max_rating\": { \"max_rating\" : { \"field\" : \"rating\" } }\n" + + " }"; // lenient, passes but reports invalid try (XContentParser parser = createParser(JsonXContent.jsonXContent, source)) { @@ -136,14 +136,14 @@ public void testDeprecation() throws IOException { private static AggregationBuilder getRandomSupportedAggregation() { final int numberOfSupportedAggs = 4; switch (randomIntBetween(1, numberOfSupportedAggs)) { - case 1: - return AggregationBuilders.avg(randomAlphaOfLengthBetween(1, 10)).field(randomAlphaOfLengthBetween(1, 10)); - case 2: - return AggregationBuilders.min(randomAlphaOfLengthBetween(1, 10)).field(randomAlphaOfLengthBetween(1, 10)); - case 3: - return AggregationBuilders.max(randomAlphaOfLengthBetween(1, 10)).field(randomAlphaOfLengthBetween(1, 10)); - case 4: - return AggregationBuilders.sum(randomAlphaOfLengthBetween(1, 10)).field(randomAlphaOfLengthBetween(1, 10)); + case 1: + return AggregationBuilders.avg(randomAlphaOfLengthBetween(1, 10)).field(randomAlphaOfLengthBetween(1, 10)); + case 2: + return AggregationBuilders.min(randomAlphaOfLengthBetween(1, 10)).field(randomAlphaOfLengthBetween(1, 10)); + case 3: + return AggregationBuilders.max(randomAlphaOfLengthBetween(1, 10)).field(randomAlphaOfLengthBetween(1, 10)); + case 4: + return AggregationBuilders.sum(randomAlphaOfLengthBetween(1, 10)).field(randomAlphaOfLengthBetween(1, 10)); } return null; From a732dfde6888c2981d0a600c41156f5b48c0168e Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Thu, 21 Jan 2021 16:54:33 +0100 Subject: [PATCH 03/39] fix license headers --- .../transforms/TimeRetentionPolicyConfig.java | 19 +++++++++++++++++++ .../TimeRetentionPolicyConfigTests.java | 19 +++++++++++++++++++ .../transform/transforms/TransformConfig.java | 4 ++-- 3 files changed, 40 insertions(+), 2 deletions(-) diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/TimeRetentionPolicyConfig.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/TimeRetentionPolicyConfig.java index 54dd7af4fb4c8..f26d7d547ba22 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/TimeRetentionPolicyConfig.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/TimeRetentionPolicyConfig.java @@ -1,3 +1,22 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.elasticsearch.client.transform.transforms; import org.elasticsearch.common.ParseField; diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/TimeRetentionPolicyConfigTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/TimeRetentionPolicyConfigTests.java index 86ff0447f4eff..b2e23cf5fe6b3 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/TimeRetentionPolicyConfigTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/TimeRetentionPolicyConfigTests.java @@ -1,3 +1,22 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.elasticsearch.client.transform.transforms; import org.elasticsearch.common.unit.TimeValue; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java index 9a4ad942a08d0..07acc1eaba06a 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java @@ -240,7 +240,7 @@ public TransformConfig(final StreamInput in) throws IOException { } else { settings = new SettingsConfig(); } - if (in.getVersion().onOrAfter(Version.V_8_0_0)) { + if (in.getVersion().onOrAfter(Version.V_8_0_0)) { // todo: V_7_12_0 retentionPolicyConfig = in.readOptionalNamedWriteable(RetentionPolicyConfig.class); } else { retentionPolicyConfig = null; @@ -369,7 +369,7 @@ public void writeTo(final StreamOutput out) throws IOException { if (out.getVersion().onOrAfter(Version.V_7_8_0)) { settings.writeTo(out); } - if (out.getVersion().onOrAfter(Version.V_8_0_0)) { + if (out.getVersion().onOrAfter(Version.V_8_0_0)) { // todo: V_7_12_0 out.writeOptionalNamedWriteable(retentionPolicyConfig); } } From 35e7c9a1c1498b9ed7987b5303fd0ebc6aaa9a8e Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Fri, 22 Jan 2021 11:05:40 +0100 Subject: [PATCH 04/39] add missing headers --- .../transform/transforms/TimeRetentionPolicyConfig.java | 6 ++++++ .../transforms/TimeRetentionPolicyConfigTests.java | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TimeRetentionPolicyConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TimeRetentionPolicyConfig.java index fae32524bdb18..1dcabdc7d65ff 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TimeRetentionPolicyConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TimeRetentionPolicyConfig.java @@ -1,3 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + package org.elasticsearch.xpack.core.transform.transforms; import org.elasticsearch.common.Strings; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TimeRetentionPolicyConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TimeRetentionPolicyConfigTests.java index 677fd9da72fb8..28c34f8c3e09b 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TimeRetentionPolicyConfigTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TimeRetentionPolicyConfigTests.java @@ -1,3 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + package org.elasticsearch.xpack.core.transform.transforms; import org.elasticsearch.common.io.stream.Writeable.Reader; From 04bc8c8324ddb0ca30c5f9a87f52dff603d2d998 Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Fri, 22 Jan 2021 11:15:34 +0100 Subject: [PATCH 05/39] checkstyle --- .../org/elasticsearch/xpack/core/XPackClientPlugin.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java index 458e7b24f9787..d8f43e33029bb 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java @@ -507,7 +507,11 @@ public List getNamedWriteables() { new NamedWriteableRegistry.Entry(Task.Status.class, TransformField.TASK_NAME, TransformState::new), new NamedWriteableRegistry.Entry(PersistentTaskState.class, TransformField.TASK_NAME, TransformState::new), new NamedWriteableRegistry.Entry(SyncConfig.class, TransformField.TIME.getPreferredName(), TimeSyncConfig::new), - new NamedWriteableRegistry.Entry(RetentionPolicyConfig.class, TransformField.TIME.getPreferredName(), TimeRetentionPolicyConfig::new), + new NamedWriteableRegistry.Entry( + RetentionPolicyConfig.class, + TransformField.TIME.getPreferredName(), + TimeRetentionPolicyConfig::new + ), // Flattened for backward compatibility with 7.x new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.FLATTENED, FlattenedFeatureSetUsage::new), // Vectors From 59acf310c663b941513a0a7fb3d87e4b2d506f4a Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Fri, 22 Jan 2021 11:49:52 +0100 Subject: [PATCH 06/39] fix compile error --- .../integration/TransformProgressIT.java | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformProgressIT.java b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformProgressIT.java index 506dc1a95e729..0fdfa2754e3c9 100644 --- a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformProgressIT.java +++ b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformProgressIT.java @@ -152,8 +152,21 @@ public void assertGetProgress(int userWithMissingBuckets) throws Exception { aggs.addAggregator(AggregationBuilders.avg("avg_rating").field("stars")); AggregationConfig aggregationConfig = new AggregationConfig(Collections.emptyMap(), aggs); PivotConfig pivotConfig = new PivotConfig(histgramGroupConfig, aggregationConfig, null); - TransformConfig config = - new TransformConfig(transformId, sourceConfig, destConfig, null, null, null, pivotConfig, null, null, null, null, null); + TransformConfig config = new TransformConfig( + transformId, + sourceConfig, + destConfig, + null, + null, + null, + pivotConfig, + null, + null, + null, + null, + null, + null + ); Pivot pivot = new Pivot(pivotConfig, new SettingsConfig(), Version.CURRENT); From 4daf8840a502a1ac7883590a33e9c4741a39fc75 Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Fri, 22 Jan 2021 14:41:16 +0100 Subject: [PATCH 07/39] more compile errors --- .../xpack/transform/transforms/TransformIndexerStateTests.java | 2 ++ .../xpack/transform/transforms/TransformIndexerTests.java | 3 +++ 2 files changed, 5 insertions(+) diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerStateTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerStateTests.java index 922d19b19afa4..0486fe3b479f9 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerStateTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerStateTests.java @@ -254,6 +254,7 @@ public void testStopAtCheckpoint() throws Exception { randomBoolean() ? null : randomAlphaOfLengthBetween(1, 1000), null, null, + null, null ); @@ -460,6 +461,7 @@ public void testStopAtCheckpointForThrottledTransform() throws Exception { randomBoolean() ? null : randomAlphaOfLengthBetween(1, 1000), new SettingsConfig(null, Float.valueOf(1.0f), (Boolean) null), null, + null, null ); AtomicReference state = new AtomicReference<>(IndexerState.STARTED); diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerTests.java index 3f779ff7d9ae2..c0d589b0562b6 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerTests.java @@ -276,6 +276,7 @@ public void testPageSizeAdapt() throws Exception { randomBoolean() ? null : randomAlphaOfLengthBetween(1, 1000), new SettingsConfig(pageSize, null, (Boolean) null), null, + null, null ); AtomicReference state = new AtomicReference<>(IndexerState.STOPPED); @@ -347,6 +348,7 @@ public void testDoProcessAggNullCheck() { randomBoolean() ? null : randomAlphaOfLengthBetween(1, 1000), new SettingsConfig(pageSize, null, (Boolean) null), null, + null, null ); SearchResponse searchResponse = new SearchResponse( @@ -408,6 +410,7 @@ public void testScriptError() throws Exception { randomBoolean() ? null : randomAlphaOfLengthBetween(1, 1000), new SettingsConfig(pageSize, null, (Boolean) null), null, + null, null ); AtomicReference state = new AtomicReference<>(IndexerState.STOPPED); From ea5e63863c9a838a42706c68405ac51dce4e6614 Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Mon, 25 Jan 2021 10:25:01 +0100 Subject: [PATCH 08/39] fix test randomization --- .../action/UpdateTransformsActionResponseTests.java | 10 +++++++++- .../transform/transforms/TransformConfigTests.java | 4 ++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/UpdateTransformsActionResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/UpdateTransformsActionResponseTests.java index db8f8c66394c8..fe9240583b5d3 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/UpdateTransformsActionResponseTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/UpdateTransformsActionResponseTests.java @@ -15,6 +15,7 @@ import org.elasticsearch.xpack.core.transform.action.compat.UpdateTransformActionPre78; import org.elasticsearch.xpack.core.transform.transforms.TransformConfig; import org.elasticsearch.xpack.core.transform.transforms.TransformConfigTests; +import org.elasticsearch.xpack.core.transform.transforms.pivot.PivotConfigTests; import org.elasticsearch.xpack.core.transform.transforms.pivot.SingleGroupSource; import java.io.IOException; @@ -42,7 +43,14 @@ protected Response doParseInstance(XContentParser parser) throws IOException { } public void testBWCPre78() throws IOException { - Response newResponse = new Response(TransformConfigTests.randomTransformConfigWithoutHeaders(randomAlphaOfLengthBetween(1, 10))); + // latest has been added in 7.11, so we only need to test pivot + Response newResponse = new Response( + TransformConfigTests.randomTransformConfigWithoutHeaders( + randomAlphaOfLengthBetween(1, 10), + PivotConfigTests.randomPivotConfig(), + null + ) + ); UpdateTransformActionPre78.Response oldResponse = writeAndReadBWCObject( newResponse, getNamedWriteableRegistry(), diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigTests.java index 212265741bab0..a62897c1ae2f8 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigTests.java @@ -55,6 +55,10 @@ public static TransformConfig randomTransformConfigWithoutHeaders(String id) { latestConfig = LatestConfigTests.randomLatestConfig(); } + return randomTransformConfigWithoutHeaders(id, pivotConfig, latestConfig); + } + + public static TransformConfig randomTransformConfigWithoutHeaders(String id, PivotConfig pivotConfig, LatestConfig latestConfig) { return new TransformConfig( id, randomSourceConfig(), From 4923da43c5f67b7b02e93a04fdff2eb82e5945be Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Tue, 26 Jan 2021 12:52:39 +0100 Subject: [PATCH 09/39] implement delete by query based on retention policy --- .../transforms/ClientTransformIndexer.java | 15 +++++ ...PolicyToDeleteByQueryRequestConverter.java | 62 +++++++++++++++++++ .../transforms/TransformIndexer.java | 62 +++++++++++++++---- .../TransformIndexerStateTests.java | 17 +++++ .../transforms/TransformIndexerTests.java | 17 +++++ 5 files changed, 162 insertions(+), 11 deletions(-) create mode 100644 x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/RetentionPolicyToDeleteByQueryRequestConverter.java diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/ClientTransformIndexer.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/ClientTransformIndexer.java index 73fe07983be4a..d91555e638da5 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/ClientTransformIndexer.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/ClientTransformIndexer.java @@ -24,6 +24,9 @@ import org.elasticsearch.common.logging.LoggerMessageFormat; import org.elasticsearch.index.engine.VersionConflictEngineException; import org.elasticsearch.index.mapper.MapperParsingException; +import org.elasticsearch.index.reindex.BulkByScrollResponse; +import org.elasticsearch.index.reindex.DeleteByQueryAction; +import org.elasticsearch.index.reindex.DeleteByQueryRequest; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.core.ClientHelper; import org.elasticsearch.xpack.core.indexing.IndexerState; @@ -196,6 +199,18 @@ protected void doNextBulk(BulkRequest request, ActionListener next ); } + @Override + protected void doDeleteByQuery(DeleteByQueryRequest deleteByQueryRequest, ActionListener responseListener) { + ClientHelper.executeWithHeadersAsync( + transformConfig.getHeaders(), + ClientHelper.TRANSFORM_ORIGIN, + client, + DeleteByQueryAction.INSTANCE, + deleteByQueryRequest, + responseListener + ); + } + @Override void doGetInitialProgress(SearchRequest request, ActionListener responseListener) { ClientHelper.executeWithHeadersAsync( diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/RetentionPolicyToDeleteByQueryRequestConverter.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/RetentionPolicyToDeleteByQueryRequestConverter.java new file mode 100644 index 0000000000000..c6cdf0de443fe --- /dev/null +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/RetentionPolicyToDeleteByQueryRequestConverter.java @@ -0,0 +1,62 @@ +package org.elasticsearch.xpack.transform.transforms; + +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.index.reindex.AbstractBulkByScrollRequest; +import org.elasticsearch.index.reindex.DeleteByQueryRequest; +import org.elasticsearch.xpack.core.transform.transforms.DestConfig; +import org.elasticsearch.xpack.core.transform.transforms.RetentionPolicyConfig; +import org.elasticsearch.xpack.core.transform.transforms.SettingsConfig; +import org.elasticsearch.xpack.core.transform.transforms.TimeRetentionPolicyConfig; +import org.elasticsearch.xpack.core.transform.transforms.TransformCheckpoint; + +import java.time.Instant; + +public final class RetentionPolicyToDeleteByQueryRequestConverter { + + public static class RetentionPolicyException extends ElasticsearchException { + RetentionPolicyException(String msg, Object... args) { + super(msg, args); + } + } + + private RetentionPolicyToDeleteByQueryRequestConverter() {} + + static DeleteByQueryRequest buildDeleteByQueryRequest( + RetentionPolicyConfig retentionPolicyConfig, + SettingsConfig settingsConfig, + DestConfig destConfig, + TransformCheckpoint checkpoint + ) { + DeleteByQueryRequest request = new DeleteByQueryRequest(); + + if (retentionPolicyConfig instanceof TimeRetentionPolicyConfig) { + request.setQuery(getDeleteQueryFromTimeBasedRetentionPolicy((TimeRetentionPolicyConfig) retentionPolicyConfig, checkpoint)); + } else { + throw new RetentionPolicyException("unsupported retention policy of type [{}]", retentionPolicyConfig.getWriteableName()); + } + + /* other dbq options not set and why: + * - timeout: we do not timeout for search, so we don't timeout for dbq + * - batch size: don't use max page size search, dbq should be simple + */ + request.setSlices(AbstractBulkByScrollRequest.AUTO_SLICES) + .setBatchSize(AbstractBulkByScrollRequest.DEFAULT_SCROLL_SIZE) + // this should not happen, but still go over version conflicts and report later + .setAbortOnVersionConflict(false) + // use the same throttling as for search + .setRequestsPerSecond(settingsConfig.getDocsPerSecond()) + .indices(destConfig.getIndex()); + + return request; + } + + private static QueryBuilder getDeleteQueryFromTimeBasedRetentionPolicy( + TimeRetentionPolicyConfig config, + TransformCheckpoint checkpoint + ) { + Instant cutOffDate = Instant.ofEpochMilli(checkpoint.getTimeUpperBound()).minusMillis(config.getMaxAge().getMillis()); + return QueryBuilders.rangeQuery(config.getField()).lt(cutOffDate.toEpochMilli()).format("epoch_millis"); + } +} diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformIndexer.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformIndexer.java index 05b32c5ff4d24..326ab9a8ff427 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformIndexer.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformIndexer.java @@ -23,6 +23,8 @@ import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.index.reindex.BulkByScrollResponse; +import org.elasticsearch.index.reindex.DeleteByQueryRequest; import org.elasticsearch.script.ScriptException; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.threadpool.ThreadPool; @@ -103,6 +105,7 @@ private enum RunState { // collects changes for continuous mode private ChangeCollector changeCollector; + // position of the change collector, in flux (not yet persisted as we haven't processed changes yet) private Map nextChangeCollectorBucketPosition = null; @@ -158,6 +161,8 @@ public TransformIndexer( abstract void doGetInitialProgress(SearchRequest request, ActionListener responseListener); + abstract void doDeleteByQuery(DeleteByQueryRequest deleteByQueryRequest, ActionListener responseListener); + public int getPageSize() { return pageSize; } @@ -404,17 +409,53 @@ protected boolean initialRun() { @Override protected void onFinish(ActionListener listener) { startIndexerThreadShutdown(); - try { - // This indicates an early exit since no changes were found. - // So, don't treat this like a checkpoint being completed, as no work was done. - if (hasSourceChanged == false) { - if (context.shouldStopAtCheckpoint()) { - stop(); - } - listener.onResponse(null); - return; + // This indicates an early exit since no changes were found. + // So, don't treat this like a checkpoint being completed, as no work was done. + if (hasSourceChanged == false) { + if (context.shouldStopAtCheckpoint()) { + stop(); } + listener.onResponse(null); + return; + } + + // delete data defined by retention policy + if (transformConfig.getRetentionPolicyConfig() != null) { + doDeleteByQuery( + RetentionPolicyToDeleteByQueryRequestConverter.buildDeleteByQueryRequest( + transformConfig.getRetentionPolicyConfig(), + transformConfig.getSettings(), + transformConfig.getDestination(), + lastCheckpoint + ), + ActionListener.wrap(bulkByScrollResponse -> { + if (bulkByScrollResponse.getDeleted() > 0) { + logger.debug( + "[{}] deleted [{}] documents as part of the retention policy.", + getJobId(), + bulkByScrollResponse.getDeleted() + ); + } + + // this should not happen as part of checkpointing + if (bulkByScrollResponse.getVersionConflicts() > 0) { + logger.warn( + "[{}] found [{}] version conflicts when deleting documents as part of the retention policy.", + getJobId(), + bulkByScrollResponse.getDeleted() + ); + } + finalizeCheckpoint(listener); + }, listener::onFailure) + ); + } else { + finalizeCheckpoint(listener); + } + } + + private void finalizeCheckpoint(ActionListener listener) { + try { // reset the page size, so we do not memorize a low page size forever pageSize = function.getInitialPageSize(); // reset the changed bucket to free memory @@ -853,8 +894,7 @@ protected SearchRequest buildSearchRequest() { throw new IllegalStateException("Transform indexer job encountered an illegal state [" + runState + "]"); } - return new SearchRequest(getConfig().getSource().getIndex()) - .allowPartialSearchResults(false) + return new SearchRequest(getConfig().getSource().getIndex()).allowPartialSearchResults(false) .indicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN) .source(sourceBuilder); } diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerStateTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerStateTests.java index 0486fe3b479f9..3324e94392feb 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerStateTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerStateTests.java @@ -19,6 +19,9 @@ import org.elasticsearch.action.search.ShardSearchFailure; import org.elasticsearch.client.Client; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.index.reindex.BulkByScrollResponse; +import org.elasticsearch.index.reindex.BulkByScrollTask; +import org.elasticsearch.index.reindex.DeleteByQueryRequest; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; import org.elasticsearch.search.internal.InternalSearchResponse; @@ -160,6 +163,19 @@ void doGetInitialProgress(SearchRequest request, ActionListener responseListener.onResponse(ONE_HIT_SEARCH_RESPONSE); } + @Override + void doDeleteByQuery(DeleteByQueryRequest deleteByQueryRequest, ActionListener responseListener) { + responseListener.onResponse( + new BulkByScrollResponse( + TimeValue.ZERO, + new BulkByScrollTask.Status(Collections.emptyList(), null), + Collections.emptyList(), + Collections.emptyList(), + false + ) + ); + } + @Override protected void doNextSearch(long waitTimeInNanos, ActionListener nextPhase) { if (searchLatch != null) { @@ -225,6 +241,7 @@ public int getSaveStateListenerCallCount() { public TransformState getPersistedState() { return persistedState; } + } @Before diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerTests.java index c0d589b0562b6..c29a79994d060 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerTests.java @@ -19,6 +19,10 @@ import org.elasticsearch.client.Client; import org.elasticsearch.common.breaker.CircuitBreaker.Durability; import org.elasticsearch.common.breaker.CircuitBreakingException; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.index.reindex.BulkByScrollResponse; +import org.elasticsearch.index.reindex.BulkByScrollTask; +import org.elasticsearch.index.reindex.DeleteByQueryRequest; import org.elasticsearch.script.ScriptException; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; @@ -248,6 +252,19 @@ void doGetInitialProgress(SearchRequest request, ActionListener ); } + @Override + void doDeleteByQuery(DeleteByQueryRequest deleteByQueryRequest, ActionListener responseListener) { + responseListener.onResponse( + new BulkByScrollResponse( + TimeValue.ZERO, + new BulkByScrollTask.Status(Collections.emptyList(), null), + Collections.emptyList(), + Collections.emptyList(), + false + ) + ); + } + } @Before From 156b7af32466767a36b8fcdf996cef82444fe897 Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Tue, 26 Jan 2021 16:28:09 +0100 Subject: [PATCH 10/39] add file header --- .../RetentionPolicyToDeleteByQueryRequestConverter.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/RetentionPolicyToDeleteByQueryRequestConverter.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/RetentionPolicyToDeleteByQueryRequestConverter.java index c6cdf0de443fe..a976dd09630d0 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/RetentionPolicyToDeleteByQueryRequestConverter.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/RetentionPolicyToDeleteByQueryRequestConverter.java @@ -1,3 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + package org.elasticsearch.xpack.transform.transforms; import org.elasticsearch.ElasticsearchException; From de1321200fe3840684257a7cab7cfd04794c9001 Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Thu, 28 Jan 2021 12:18:15 +0100 Subject: [PATCH 11/39] refresh the destination index before executing the retention policy --- .../transforms/TimeRetentionPolicyConfig.java | 4 +- .../indices/refresh/RefreshResponse.java | 2 +- .../transforms/ClientTransformIndexer.java | 17 ++++ ...PolicyToDeleteByQueryRequestConverter.java | 13 ++- .../transforms/TransformIndexer.java | 88 +++++++++++++------ .../TransformIndexerStateTests.java | 8 +- .../transforms/TransformIndexerTests.java | 6 ++ 7 files changed, 101 insertions(+), 37 deletions(-) diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/TimeRetentionPolicyConfig.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/TimeRetentionPolicyConfig.java index f26d7d547ba22..eaf2838a2dd08 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/TimeRetentionPolicyConfig.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/TimeRetentionPolicyConfig.java @@ -70,9 +70,7 @@ public TimeRetentionPolicyConfig(String field, TimeValue maxAge) { public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); builder.field(FIELD.getPreferredName(), field); - if (maxAge.duration() > 0) { - builder.field(MAX_AGE.getPreferredName(), maxAge.getStringRep()); - } + builder.field(MAX_AGE.getPreferredName(), maxAge.getStringRep()); builder.endObject(); return builder; } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/RefreshResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/RefreshResponse.java index e85f08136587d..c61e8c2a76b3b 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/RefreshResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/RefreshResponse.java @@ -38,7 +38,7 @@ public class RefreshResponse extends BroadcastResponse { super(in); } - RefreshResponse(int totalShards, int successfulShards, int failedShards, List shardFailures) { + public RefreshResponse(int totalShards, int successfulShards, int failedShards, List shardFailures) { super(totalShards, successfulShards, failedShards, shardFailures); } diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/ClientTransformIndexer.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/ClientTransformIndexer.java index d91555e638da5..a081bf47630b3 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/ClientTransformIndexer.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/ClientTransformIndexer.java @@ -12,6 +12,9 @@ import org.apache.logging.log4j.message.ParameterizedMessage; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.indices.refresh.RefreshAction; +import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; +import org.elasticsearch.action.admin.indices.refresh.RefreshResponse; import org.elasticsearch.action.bulk.BulkAction; import org.elasticsearch.action.bulk.BulkItemResponse; import org.elasticsearch.action.bulk.BulkRequest; @@ -211,6 +214,20 @@ protected void doDeleteByQuery(DeleteByQueryRequest deleteByQueryRequest, Action ); } + @Override + protected void refreshDestinationIndex(RefreshRequest refreshRequest, ActionListener responseListener) { + // note: this gets executed _without_ the headers of the user as the user might not have the rights to call + // _refresh for performance reasons. However this refresh is an internal detail of transform and this is only + // called for the transform destination index + ClientHelper.executeAsyncWithOrigin( + client, + ClientHelper.TRANSFORM_ORIGIN, + RefreshAction.INSTANCE, + refreshRequest, + responseListener + ); + } + @Override void doGetInitialProgress(SearchRequest request, ActionListener responseListener) { ClientHelper.executeWithHeadersAsync( diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/RetentionPolicyToDeleteByQueryRequestConverter.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/RetentionPolicyToDeleteByQueryRequestConverter.java index a976dd09630d0..924287fbb43ef 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/RetentionPolicyToDeleteByQueryRequestConverter.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/RetentionPolicyToDeleteByQueryRequestConverter.java @@ -51,10 +51,17 @@ static DeleteByQueryRequest buildDeleteByQueryRequest( .setBatchSize(AbstractBulkByScrollRequest.DEFAULT_SCROLL_SIZE) // this should not happen, but still go over version conflicts and report later .setAbortOnVersionConflict(false) - // use the same throttling as for search - .setRequestsPerSecond(settingsConfig.getDocsPerSecond()) + // refresh the index, so docs are really gone + .setRefresh(true) + // use transforms retry mechanics instead + .setMaxRetries(0) .indices(destConfig.getIndex()); + // use the same throttling as for search + if (settingsConfig.getDocsPerSecond() != null) { + request.setRequestsPerSecond(settingsConfig.getDocsPerSecond()); + } + return request; } @@ -62,7 +69,7 @@ private static QueryBuilder getDeleteQueryFromTimeBasedRetentionPolicy( TimeRetentionPolicyConfig config, TransformCheckpoint checkpoint ) { - Instant cutOffDate = Instant.ofEpochMilli(checkpoint.getTimeUpperBound()).minusMillis(config.getMaxAge().getMillis()); + Instant cutOffDate = Instant.ofEpochMilli(checkpoint.getTimestamp()).minusMillis(config.getMaxAge().getMillis()); return QueryBuilders.rangeQuery(config.getField()).lt(cutOffDate.toEpochMilli()).format("epoch_millis"); } } diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformIndexer.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformIndexer.java index 326ab9a8ff427..edc4341bcebaa 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformIndexer.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformIndexer.java @@ -13,6 +13,8 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; +import org.elasticsearch.action.admin.indices.refresh.RefreshResponse; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; @@ -163,6 +165,8 @@ public TransformIndexer( abstract void doDeleteByQuery(DeleteByQueryRequest deleteByQueryRequest, ActionListener responseListener); + abstract void refreshDestinationIndex(RefreshRequest refreshRequest, ActionListener responseListener); + public int getPageSize() { return pageSize; } @@ -409,6 +413,7 @@ protected boolean initialRun() { @Override protected void onFinish(ActionListener listener) { startIndexerThreadShutdown(); + // This indicates an early exit since no changes were found. // So, don't treat this like a checkpoint being completed, as no work was done. if (hasSourceChanged == false) { @@ -419,39 +424,64 @@ protected void onFinish(ActionListener listener) { return; } - // delete data defined by retention policy - if (transformConfig.getRetentionPolicyConfig() != null) { - doDeleteByQuery( - RetentionPolicyToDeleteByQueryRequestConverter.buildDeleteByQueryRequest( - transformConfig.getRetentionPolicyConfig(), - transformConfig.getSettings(), - transformConfig.getDestination(), - lastCheckpoint - ), - ActionListener.wrap(bulkByScrollResponse -> { - if (bulkByScrollResponse.getDeleted() > 0) { - logger.debug( - "[{}] deleted [{}] documents as part of the retention policy.", - getJobId(), - bulkByScrollResponse.getDeleted() - ); - } - - // this should not happen as part of checkpointing - if (bulkByScrollResponse.getVersionConflicts() > 0) { + refreshDestinationIndex( + new RefreshRequest(transformConfig.getDestination().getIndex()), + ActionListener.wrap( + response -> { + if (response.getFailedShards() > 0) { logger.warn( - "[{}] found [{}] version conflicts when deleting documents as part of the retention policy.", - getJobId(), - bulkByScrollResponse.getDeleted() + "[{}] failed to refresh transform destination index, not all data might be available after checkpoint.", + getJobId() ); } + // delete data defined by retention policy + if (transformConfig.getRetentionPolicyConfig() != null) { + executeRetentionPolicy(listener); + } else { + finalizeCheckpoint(listener); + } + }, + listener::onFailure + ) + ); + } - finalizeCheckpoint(listener); - }, listener::onFailure) - ); - } else { - finalizeCheckpoint(listener); - } + private void executeRetentionPolicy(ActionListener listener) { + DeleteByQueryRequest deleteByQuery = RetentionPolicyToDeleteByQueryRequestConverter.buildDeleteByQueryRequest( + transformConfig.getRetentionPolicyConfig(), + transformConfig.getSettings(), + transformConfig.getDestination(), + nextCheckpoint + ); + logger.debug( + "[{}] Run delete based on retention policy using dbq [{}] with query: [{}]", + getJobId(), + deleteByQuery, + deleteByQuery.getSearchRequest() + ); + + doDeleteByQuery(deleteByQuery, ActionListener.wrap(bulkByScrollResponse -> { + logger.trace("[{}] dbq response: [{}]", getJobId(), bulkByScrollResponse); + if (bulkByScrollResponse.getDeleted() > 0) { + logger.debug( + "[{}] deleted [{}] documents as part of the retention policy.", + getJobId(), + bulkByScrollResponse.getDeleted() + ); + } + + // this should not happen as part of checkpointing + if (bulkByScrollResponse.getVersionConflicts() > 0) { + logger.warn( + "[{}] found [{}] version conflicts when deleting documents as part of the retention policy.", + getJobId(), + bulkByScrollResponse.getDeleted() + ); + } + + finalizeCheckpoint(listener); + }, listener::onFailure) + ); } private void finalizeCheckpoint(ActionListener listener) { diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerStateTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerStateTests.java index 3324e94392feb..d1613ceb316f1 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerStateTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerStateTests.java @@ -10,6 +10,8 @@ import org.apache.lucene.search.TotalHits; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.LatchedActionListener; +import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; +import org.elasticsearch.action.admin.indices.refresh.RefreshResponse; import org.elasticsearch.action.bulk.BulkItemResponse; import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.bulk.BulkResponse; @@ -176,6 +178,11 @@ void doDeleteByQuery(DeleteByQueryRequest deleteByQueryRequest, ActionListener responseListener) { + responseListener.onResponse(new RefreshResponse(1,1,0,Collections.emptyList())); + } + @Override protected void doNextSearch(long waitTimeInNanos, ActionListener nextPhase) { if (searchLatch != null) { @@ -241,7 +248,6 @@ public int getSaveStateListenerCallCount() { public TransformState getPersistedState() { return persistedState; } - } @Before diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerTests.java index c29a79994d060..eb94d179fc9fb 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerTests.java @@ -9,6 +9,8 @@ import org.apache.lucene.search.TotalHits; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; +import org.elasticsearch.action.admin.indices.refresh.RefreshResponse; import org.elasticsearch.action.bulk.BulkItemResponse; import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.bulk.BulkResponse; @@ -265,6 +267,10 @@ void doDeleteByQuery(DeleteByQueryRequest deleteByQueryRequest, ActionListener responseListener) { + responseListener.onResponse(new RefreshResponse(1,1,0,Collections.emptyList())); + } } @Before From 3b5ded7eb206ca75690a9a1aea3181da1d282f04 Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Thu, 28 Jan 2021 12:29:41 +0100 Subject: [PATCH 12/39] add validation hook --- .../transforms/TimeRetentionPolicyConfig.java | 6 +++ .../transforms/ClientTransformIndexer.java | 2 +- .../transforms/TransformIndexer.java | 47 +++++++------------ .../TransformIndexerStateTests.java | 2 +- .../transforms/TransformIndexerTests.java | 2 +- 5 files changed, 27 insertions(+), 32 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TimeRetentionPolicyConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TimeRetentionPolicyConfig.java index 1dcabdc7d65ff..29c6265658af8 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TimeRetentionPolicyConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TimeRetentionPolicyConfig.java @@ -6,6 +6,7 @@ package org.elasticsearch.xpack.core.transform.transforms; +import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -66,6 +67,11 @@ public TimeValue getMaxAge() { return maxAge; } + public ActionRequestValidationException validate(ActionRequestValidationException validationException) { + + return validationException; + } + @Override public void writeTo(final StreamOutput out) throws IOException { out.writeString(field); diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/ClientTransformIndexer.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/ClientTransformIndexer.java index a081bf47630b3..df08506aa8194 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/ClientTransformIndexer.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/ClientTransformIndexer.java @@ -382,7 +382,7 @@ private void doSaveState(TransformState state, ActionListener listener) { // this should never happen, but indicates a race condition in state persistence: // - there should be only 1 save persistence at a time // - this is not a catastrophic failure, if 2 state persistence calls run at the same time, 1 should succeed and update - // seqNoPrimaryTermAndIndex + // seqNoPrimaryTermAndIndex // - for tests fail(assert), so we can debug the problem logger.error( new ParameterizedMessage( diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformIndexer.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformIndexer.java index edc4341bcebaa..99d1926f0b945 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformIndexer.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformIndexer.java @@ -424,26 +424,20 @@ protected void onFinish(ActionListener listener) { return; } - refreshDestinationIndex( - new RefreshRequest(transformConfig.getDestination().getIndex()), - ActionListener.wrap( - response -> { - if (response.getFailedShards() > 0) { - logger.warn( - "[{}] failed to refresh transform destination index, not all data might be available after checkpoint.", - getJobId() - ); - } - // delete data defined by retention policy - if (transformConfig.getRetentionPolicyConfig() != null) { - executeRetentionPolicy(listener); - } else { - finalizeCheckpoint(listener); - } - }, - listener::onFailure - ) - ); + refreshDestinationIndex(new RefreshRequest(transformConfig.getDestination().getIndex()), ActionListener.wrap(response -> { + if (response.getFailedShards() > 0) { + logger.warn( + "[{}] failed to refresh transform destination index, not all data might be available after checkpoint.", + getJobId() + ); + } + // delete data defined by retention policy + if (transformConfig.getRetentionPolicyConfig() != null) { + executeRetentionPolicy(listener); + } else { + finalizeCheckpoint(listener); + } + }, listener::onFailure)); } private void executeRetentionPolicy(ActionListener listener) { @@ -463,11 +457,7 @@ private void executeRetentionPolicy(ActionListener listener) { doDeleteByQuery(deleteByQuery, ActionListener.wrap(bulkByScrollResponse -> { logger.trace("[{}] dbq response: [{}]", getJobId(), bulkByScrollResponse); if (bulkByScrollResponse.getDeleted() > 0) { - logger.debug( - "[{}] deleted [{}] documents as part of the retention policy.", - getJobId(), - bulkByScrollResponse.getDeleted() - ); + logger.debug("[{}] deleted [{}] documents as part of the retention policy.", getJobId(), bulkByScrollResponse.getDeleted()); } // this should not happen as part of checkpointing @@ -477,11 +467,10 @@ private void executeRetentionPolicy(ActionListener listener) { getJobId(), bulkByScrollResponse.getDeleted() ); - } + } - finalizeCheckpoint(listener); - }, listener::onFailure) - ); + finalizeCheckpoint(listener); + }, listener::onFailure)); } private void finalizeCheckpoint(ActionListener listener) { diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerStateTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerStateTests.java index d1613ceb316f1..131e89e246056 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerStateTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerStateTests.java @@ -180,7 +180,7 @@ void doDeleteByQuery(DeleteByQueryRequest deleteByQueryRequest, ActionListener responseListener) { - responseListener.onResponse(new RefreshResponse(1,1,0,Collections.emptyList())); + responseListener.onResponse(new RefreshResponse(1, 1, 0, Collections.emptyList())); } @Override diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerTests.java index eb94d179fc9fb..a70f7a7417eac 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerTests.java @@ -269,7 +269,7 @@ void doDeleteByQuery(DeleteByQueryRequest deleteByQueryRequest, ActionListener responseListener) { - responseListener.onResponse(new RefreshResponse(1,1,0,Collections.emptyList())); + responseListener.onResponse(new RefreshResponse(1, 1, 0, Collections.emptyList())); } } From c1aa7e76d235d294e1772b6fde3dda09139495cf Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Thu, 28 Jan 2021 12:44:53 +0100 Subject: [PATCH 13/39] checkstyle --- .../action/admin/indices/refresh/RefreshResponse.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/RefreshResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/RefreshResponse.java index c61e8c2a76b3b..4c162fa14ef24 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/RefreshResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/RefreshResponse.java @@ -38,7 +38,12 @@ public class RefreshResponse extends BroadcastResponse { super(in); } - public RefreshResponse(int totalShards, int successfulShards, int failedShards, List shardFailures) { + public RefreshResponse( + int totalShards, + int successfulShards, + int failedShards, + List shardFailures + ) { super(totalShards, successfulShards, failedShards, shardFailures); } From f2407b812555426fb2799a588f6072ccfd39b770 Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Fri, 29 Jan 2021 11:41:12 +0100 Subject: [PATCH 14/39] add some basic validation --- .../transform/transforms/RetentionPolicyConfig.java | 3 ++- .../transforms/TimeRetentionPolicyConfig.java | 10 +++++++++- .../core/transform/transforms/TransformConfig.java | 5 +++++ 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/RetentionPolicyConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/RetentionPolicyConfig.java index 08a4f853976f9..1e3b56cba8cf6 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/RetentionPolicyConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/RetentionPolicyConfig.java @@ -6,9 +6,10 @@ package org.elasticsearch.xpack.core.transform.transforms; +import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.common.io.stream.NamedWriteable; import org.elasticsearch.common.xcontent.ToXContentObject; public interface RetentionPolicyConfig extends ToXContentObject, NamedWriteable { - + public ActionRequestValidationException validate(ActionRequestValidationException validationException); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TimeRetentionPolicyConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TimeRetentionPolicyConfig.java index 29c6265658af8..238cbe7f60061 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TimeRetentionPolicyConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TimeRetentionPolicyConfig.java @@ -21,11 +21,13 @@ import java.io.IOException; import java.util.Objects; +import static org.elasticsearch.action.ValidateActions.addValidationError; import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; public class TimeRetentionPolicyConfig implements RetentionPolicyConfig { private static final String NAME = "transform_retention_policy_time"; + private static long MIN_AGE_SECONDS = 60; private final String field; private final TimeValue maxAge; @@ -67,8 +69,14 @@ public TimeValue getMaxAge() { return maxAge; } + @Override public ActionRequestValidationException validate(ActionRequestValidationException validationException) { - + if (maxAge.getSeconds() < MIN_AGE_SECONDS) { + validationException = addValidationError( + "retention_policy.time.max_age must be more than " + MIN_AGE_SECONDS + "s, found [" + maxAge + "]", + validationException + ); + } return validationException; } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java index 07acc1eaba06a..cb6902e765781 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java @@ -325,6 +325,11 @@ public ActionRequestValidationException validate(ActionRequestValidationExceptio validationException = latestConfig.validate(validationException); } validationException = settings.validate(validationException); + + if (retentionPolicyConfig != null) { + validationException = retentionPolicyConfig.validate(validationException); + } + return validationException; } From 68d84f3ce5a960c93100738625baad4d72297c14 Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Mon, 1 Feb 2021 10:52:15 +0100 Subject: [PATCH 15/39] checkstyle --- .../xpack/core/transform/transforms/RetentionPolicyConfig.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/RetentionPolicyConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/RetentionPolicyConfig.java index 1e3b56cba8cf6..a1e047ef1f5f6 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/RetentionPolicyConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/RetentionPolicyConfig.java @@ -11,5 +11,5 @@ import org.elasticsearch.common.xcontent.ToXContentObject; public interface RetentionPolicyConfig extends ToXContentObject, NamedWriteable { - public ActionRequestValidationException validate(ActionRequestValidationException validationException); + ActionRequestValidationException validate(ActionRequestValidationException validationException); } From 0c6732ecc2158404eb6b9d3ec5b301a9cb5dcfe6 Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Mon, 1 Feb 2021 11:24:53 +0100 Subject: [PATCH 16/39] remove transform dependency from async search tests --- .../xpack/search/AsyncSearchResponseTests.java | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchResponseTests.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchResponseTests.java index 21ee11a653913..8e875c7aa4d39 100644 --- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchResponseTests.java +++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchResponseTests.java @@ -14,7 +14,6 @@ import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentElasticsearchExtension; @@ -24,10 +23,6 @@ import org.elasticsearch.search.internal.InternalSearchResponse; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.core.search.action.AsyncSearchResponse; -import org.elasticsearch.xpack.core.transform.TransformField; -import org.elasticsearch.xpack.core.transform.TransformNamedXContentProvider; -import org.elasticsearch.xpack.core.transform.transforms.SyncConfig; -import org.elasticsearch.xpack.core.transform.transforms.TimeSyncConfig; import org.junit.Before; import java.io.IOException; @@ -48,12 +43,6 @@ public void registerNamedObjects() { SearchModule searchModule = new SearchModule(Settings.EMPTY, emptyList()); List namedWriteables = searchModule.getNamedWriteables(); - namedWriteables.add(new NamedWriteableRegistry.Entry(SyncConfig.class, TransformField.TIME.getPreferredName(), - TimeSyncConfig::new)); - - List namedXContents = searchModule.getNamedXContents(); - namedXContents.addAll(new TransformNamedXContentProvider().getNamedXContentParsers()); - namedWriteableRegistry = new NamedWriteableRegistry(namedWriteables); } From eb37fbeded77d062d8180c996a8127e36dcccc6b Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Mon, 1 Feb 2021 11:51:00 +0100 Subject: [PATCH 17/39] validate retention config --- .../xpack/core/transform/transforms/TransformConfig.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java index cb6902e765781..b208e97cb602c 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java @@ -334,6 +334,7 @@ public ActionRequestValidationException validate(ActionRequestValidationExceptio } public boolean isValid() { + // todo: base this on validate if (pivotConfig != null && pivotConfig.isValid() == false) { return false; } @@ -346,6 +347,10 @@ public boolean isValid() { return false; } + if (retentionPolicyConfig != null && retentionPolicyConfig.validate(null) != null) { + return false; + } + return settings.isValid() && source.isValid() && dest.isValid(); } From 7101dadb92e21725b4b7a20b02e92f291e68503d Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Mon, 1 Feb 2021 11:53:19 +0100 Subject: [PATCH 18/39] delete accidentaly added file --- .../transforms/TransformConfig.java.orig | 662 ------------------ 1 file changed, 662 deletions(-) delete mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java.orig diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java.orig b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java.orig deleted file mode 100644 index 105314bd43225..0000000000000 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java.orig +++ /dev/null @@ -1,662 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -package org.elasticsearch.xpack.core.transform.transforms; - -import org.elasticsearch.Version; -import org.elasticsearch.action.ActionRequestValidationException; -import org.elasticsearch.cluster.AbstractDiffable; -import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.ParseField; -import org.elasticsearch.common.Strings; -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.common.xcontent.ConstructingObjectParser; -import org.elasticsearch.common.xcontent.ObjectParser; -import org.elasticsearch.common.xcontent.ToXContentObject; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.common.xcontent.XContentParserUtils; -import org.elasticsearch.xpack.core.common.time.TimeUtils; -import org.elasticsearch.xpack.core.transform.TransformField; -import org.elasticsearch.xpack.core.transform.TransformMessages; -import org.elasticsearch.xpack.core.transform.transforms.map.MapConfig; -import org.elasticsearch.xpack.core.transform.transforms.pivot.PivotConfig; -import org.elasticsearch.xpack.core.transform.utils.ExceptionsHelper; - -import java.io.IOException; -import java.time.Instant; -import java.util.Collections; -import java.util.Map; -import java.util.Objects; - -import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; -import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg; - -/** - * This class holds the configuration details of a data frame transform - */ -public class TransformConfig extends AbstractDiffable implements Writeable, ToXContentObject { - - public static final String NAME = "data_frame_transform_config"; - public static final ParseField HEADERS = new ParseField("headers"); - - // types of transforms - public static final ParseField PIVOT_TRANSFORM = new ParseField("pivot"); - public static final ParseField MAP_TRANSFORM = new ParseField("map"); - - private static final ConstructingObjectParser STRICT_PARSER = createParser(false); - private static final ConstructingObjectParser LENIENT_PARSER = createParser(true); - static final int MAX_DESCRIPTION_LENGTH = 1_000; - - private final String id; - private final SourceConfig source; - private final DestConfig dest; - private final TimeValue frequency; - private final SyncConfig syncConfig; - private final SettingsConfig settings; - private final String description; - // headers store the user context from the creating user, which allows us to run the transform as this user - // the header only contains name, groups and other context but no authorization keys - private Map headers; - private Version transformVersion; - private Instant createTime; - - private final PivotConfig pivotConfig; - private final MapConfig mapConfig; - - private static void validateStrictParsingParams(Object arg, String parameterName) { - if (arg != null) { - throw new IllegalArgumentException("Found [" + parameterName + "], not allowed for strict parsing"); - } - } - - private static ConstructingObjectParser createParser(boolean lenient) { - ConstructingObjectParser parser = new ConstructingObjectParser<>(NAME, lenient, (args, optionalId) -> { - String id = (String) args[0]; - - // if the id has been specified in the body and the path, they must match - if (id == null) { - id = optionalId; - } else if (optionalId != null && id.equals(optionalId) == false) { - throw new IllegalArgumentException( - TransformMessages.getMessage(TransformMessages.REST_PUT_TRANSFORM_INCONSISTENT_ID, id, optionalId) - ); - } - - SourceConfig source = (SourceConfig) args[1]; - DestConfig dest = (DestConfig) args[2]; - - TimeValue frequency = args[3] == null - ? null - : TimeValue.parseTimeValue((String) args[3], TransformField.FREQUENCY.getPreferredName()); - - SyncConfig syncConfig = (SyncConfig) args[4]; - // ignored, only for internal storage: String docType = (String) args[5]; - - // on strict parsing do not allow injection of headers, transform version, or create time - if (lenient == false) { - validateStrictParsingParams(args[6], HEADERS.getPreferredName()); - validateStrictParsingParams(args[11], TransformField.CREATE_TIME.getPreferredName()); - validateStrictParsingParams(args[12], TransformField.VERSION.getPreferredName()); - } - - @SuppressWarnings("unchecked") - Map headers = (Map) args[6]; - - PivotConfig pivotConfig = (PivotConfig) args[7]; - MapConfig mapConfig = (MapConfig) args[8]; - - String description = (String) args[9]; - SettingsConfig settings = (SettingsConfig) args[10]; - return new TransformConfig( - id, - source, - dest, - frequency, - syncConfig, - headers, - pivotConfig, - mapConfig, - description, - settings, - (Instant) args[11], - (String) args[12] - ); - }); - - parser.declareString(optionalConstructorArg(), TransformField.ID); - parser.declareObject(constructorArg(), (p, c) -> SourceConfig.fromXContent(p, lenient), TransformField.SOURCE); - parser.declareObject(constructorArg(), (p, c) -> DestConfig.fromXContent(p, lenient), TransformField.DESTINATION); - parser.declareString(optionalConstructorArg(), TransformField.FREQUENCY); - parser.declareObject(optionalConstructorArg(), (p, c) -> parseSyncConfig(p, lenient), TransformField.SYNC); - parser.declareString(optionalConstructorArg(), TransformField.INDEX_DOC_TYPE); - parser.declareObject(optionalConstructorArg(), (p, c) -> p.mapStrings(), HEADERS); - parser.declareObject(optionalConstructorArg(), (p, c) -> PivotConfig.fromXContent(p, lenient), PIVOT_TRANSFORM); - parser.declareObject(optionalConstructorArg(), (p, c) -> MapConfig.fromXContent(p, lenient), MAP_TRANSFORM); - parser.declareString(optionalConstructorArg(), TransformField.DESCRIPTION); - parser.declareObject(optionalConstructorArg(), (p, c) -> SettingsConfig.fromXContent(p, lenient), TransformField.SETTINGS); - parser.declareField( - optionalConstructorArg(), - p -> TimeUtils.parseTimeFieldToInstant(p, TransformField.CREATE_TIME.getPreferredName()), - TransformField.CREATE_TIME, - ObjectParser.ValueType.VALUE - ); - parser.declareString(optionalConstructorArg(), TransformField.VERSION); - return parser; - } - - private static SyncConfig parseSyncConfig(XContentParser parser, boolean ignoreUnknownFields) throws IOException { - XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser); - XContentParserUtils.ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.nextToken(), parser); - SyncConfig syncConfig = parser.namedObject(SyncConfig.class, parser.currentName(), ignoreUnknownFields); - XContentParserUtils.ensureExpectedToken(XContentParser.Token.END_OBJECT, parser.nextToken(), parser); - return syncConfig; - } - - public static String documentId(String transformId) { - return NAME + "-" + transformId; - } - - TransformConfig( - final String id, - final SourceConfig source, - final DestConfig dest, - final TimeValue frequency, - final SyncConfig syncConfig, - final Map headers, - final PivotConfig pivotConfig, - final MapConfig mapConfig, - final String description, - final SettingsConfig settings, - final Instant createTime, - final String version - ) { - this.id = ExceptionsHelper.requireNonNull(id, TransformField.ID.getPreferredName()); - this.source = ExceptionsHelper.requireNonNull(source, TransformField.SOURCE.getPreferredName()); - this.dest = ExceptionsHelper.requireNonNull(dest, TransformField.DESTINATION.getPreferredName()); - this.frequency = frequency; - this.syncConfig = syncConfig; - this.setHeaders(headers == null ? Collections.emptyMap() : headers); - this.pivotConfig = pivotConfig; - this.mapConfig = mapConfig; - this.description = description; - this.settings = settings == null ? new SettingsConfig() : settings; - - // at least one function must be defined - if (this.pivotConfig == null && this.mapConfig == null) { - throw new IllegalArgumentException(TransformMessages.TRANSFORM_CONFIGURATION_NO_TRANSFORM); - } - // check that not both are given - if (this.pivotConfig != null && this.mapConfig != null) { - throw new IllegalArgumentException("only 1 function allowed"); - } - - if (this.description != null && this.description.length() > MAX_DESCRIPTION_LENGTH) { - throw new IllegalArgumentException("[description] must be less than 1000 characters in length."); - } - this.createTime = createTime == null ? null : Instant.ofEpochMilli(createTime.toEpochMilli()); - this.transformVersion = version == null ? null : Version.fromString(version); - } - - public TransformConfig( - final String id, - final SourceConfig source, - final DestConfig dest, - final TimeValue frequency, - final SyncConfig syncConfig, - final Map headers, - final PivotConfig pivotConfig, - final String description, - final SettingsConfig settings - ) { - this(id, source, dest, frequency, syncConfig, headers, pivotConfig, null, description, settings, null, null); - } - - public TransformConfig(final StreamInput in) throws IOException { - id = in.readString(); - source = new SourceConfig(in); - dest = new DestConfig(in); - if (in.getVersion().onOrAfter(Version.V_7_3_0)) { - frequency = in.readOptionalTimeValue(); - } else { - frequency = null; - } - setHeaders(in.readMap(StreamInput::readString, StreamInput::readString)); - pivotConfig = in.readOptionalWriteable(PivotConfig::new); - description = in.readOptionalString(); - if (in.getVersion().onOrAfter(Version.V_7_3_0)) { - syncConfig = in.readOptionalNamedWriteable(SyncConfig.class); - createTime = in.readOptionalInstant(); - transformVersion = in.readBoolean() ? Version.readVersion(in) : null; - } else { - syncConfig = null; - createTime = null; - transformVersion = null; - } - if (in.getVersion().onOrAfter(Version.V_7_8_0)) { - settings = new SettingsConfig(in); - } else { - settings = new SettingsConfig(); - } - if (in.getVersion().onOrAfter(Version.V_8_0_0)) { // todo 7.9.0 - mapConfig = in.readOptionalWriteable(MapConfig::new); - } else { - mapConfig = null; - } - } - - public String getId() { - return id; - } - - public SourceConfig getSource() { - return source; - } - - public DestConfig getDestination() { - return dest; - } - - public TimeValue getFrequency() { - return frequency; - } - - public SyncConfig getSyncConfig() { - return syncConfig; - } - - public Map getHeaders() { - return headers; - } - - public TransformConfig setHeaders(Map headers) { - this.headers = headers; - return this; - } - - public Version getVersion() { - return transformVersion; - } - - public TransformConfig setVersion(Version transformVersion) { - this.transformVersion = transformVersion; - return this; - } - - public Instant getCreateTime() { - return createTime; - } - - public TransformConfig setCreateTime(Instant createTime) { - ExceptionsHelper.requireNonNull(createTime, TransformField.CREATE_TIME.getPreferredName()); - this.createTime = Instant.ofEpochMilli(createTime.toEpochMilli()); - return this; - } - - public PivotConfig getPivotConfig() { - return pivotConfig; - } - - public MapConfig getMapConfig() { - return mapConfig; - } - - @Nullable - public String getDescription() { - return description; - } - - public SettingsConfig getSettings() { - return settings; - } - - public ActionRequestValidationException validate(ActionRequestValidationException validationException) { - if (pivotConfig != null) { - validationException = pivotConfig.validate(validationException); - } - validationException = settings.validate(validationException); - - return validationException; - } - - public boolean isValid() { - if (pivotConfig != null && pivotConfig.isValid() == false) { - return false; - } - - if (syncConfig != null && syncConfig.isValid() == false) { - return false; - } - - return settings.isValid() && source.isValid() && dest.isValid(); - } - - @Override - public void writeTo(final StreamOutput out) throws IOException { - out.writeString(id); - source.writeTo(out); - dest.writeTo(out); - if (out.getVersion().onOrAfter(Version.V_7_3_0)) { - out.writeOptionalTimeValue(frequency); - } - out.writeMap(headers, StreamOutput::writeString, StreamOutput::writeString); - out.writeOptionalWriteable(pivotConfig); - out.writeOptionalString(description); - if (out.getVersion().onOrAfter(Version.V_7_3_0)) { - out.writeOptionalNamedWriteable(syncConfig); - out.writeOptionalInstant(createTime); - if (transformVersion != null) { - out.writeBoolean(true); - Version.writeVersion(transformVersion, out); - } else { - out.writeBoolean(false); - } - } - if (out.getVersion().onOrAfter(Version.V_7_8_0)) { - settings.writeTo(out); - } - if (out.getVersion().onOrAfter(Version.V_8_0_0)) { // todo 7.9.0 - out.writeOptionalWriteable(mapConfig); - } - } - - @Override - public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException { - final boolean excludeGenerated = params.paramAsBoolean(TransformField.EXCLUDE_GENERATED, false); - final boolean forInternalStorage = params.paramAsBoolean(TransformField.FOR_INTERNAL_STORAGE, false); - assert (forInternalStorage && excludeGenerated) == false: - "unsupported behavior, exclude_generated is true and for_internal_storage is true"; - builder.startObject(); - builder.field(TransformField.ID.getPreferredName(), id); - if (excludeGenerated == false) { - if (headers.isEmpty() == false && forInternalStorage) { - builder.field(HEADERS.getPreferredName(), headers); - } - if (transformVersion != null) { - builder.field(TransformField.VERSION.getPreferredName(), transformVersion); - } - if (createTime != null) { - builder.timeField( - TransformField.CREATE_TIME.getPreferredName(), - TransformField.CREATE_TIME.getPreferredName() + "_string", - createTime.toEpochMilli() - ); - } - if (forInternalStorage) { - builder.field(TransformField.INDEX_DOC_TYPE.getPreferredName(), NAME); - } - } - builder.field(TransformField.SOURCE.getPreferredName(), source, params); - builder.field(TransformField.DESTINATION.getPreferredName(), dest); - if (frequency != null) { - builder.field(TransformField.FREQUENCY.getPreferredName(), frequency.getStringRep()); - } - if (syncConfig != null) { - builder.startObject(TransformField.SYNC.getPreferredName()); - builder.field(syncConfig.getWriteableName(), syncConfig); - builder.endObject(); - } - if (pivotConfig != null) { - builder.field(PIVOT_TRANSFORM.getPreferredName(), pivotConfig); - } -<<<<<<< HEAD -======= - if (mapConfig != null) { - builder.field(MAP_TRANSFORM.getPreferredName(), mapConfig); - } - if (params.paramAsBoolean(TransformField.FOR_INTERNAL_STORAGE, false)) { - builder.field(TransformField.INDEX_DOC_TYPE.getPreferredName(), NAME); - } - if (headers.isEmpty() == false && params.paramAsBoolean(TransformField.FOR_INTERNAL_STORAGE, false)) { - builder.field(HEADERS.getPreferredName(), headers); - } ->>>>>>> implement map as function - if (description != null) { - builder.field(TransformField.DESCRIPTION.getPreferredName(), description); - } - builder.field(TransformField.SETTINGS.getPreferredName(), settings); - builder.endObject(); - return builder; - } - - @Override - public boolean equals(Object other) { - if (this == other) { - return true; - } - - if (other == null || getClass() != other.getClass()) { - return false; - } - - final TransformConfig that = (TransformConfig) other; - - return Objects.equals(this.id, that.id) - && Objects.equals(this.source, that.source) - && Objects.equals(this.dest, that.dest) - && Objects.equals(this.frequency, that.frequency) - && Objects.equals(this.syncConfig, that.syncConfig) - && Objects.equals(this.headers, that.headers) - && Objects.equals(this.pivotConfig, that.pivotConfig) - && Objects.equals(this.mapConfig, that.mapConfig) - && Objects.equals(this.description, that.description) - && Objects.equals(this.settings, that.settings) - && Objects.equals(this.createTime, that.createTime) - && Objects.equals(this.transformVersion, that.transformVersion); - } - - @Override - public int hashCode() { - return Objects.hash( - id, - source, - dest, - frequency, - syncConfig, - headers, - pivotConfig, - mapConfig, - description, - settings, - createTime, - transformVersion - ); - } - - @Override - public String toString() { - return Strings.toString(this, true, true); - } - - public static TransformConfig fromXContent(final XContentParser parser, @Nullable final String optionalTransformId, boolean lenient) { - - return lenient ? LENIENT_PARSER.apply(parser, optionalTransformId) : STRICT_PARSER.apply(parser, optionalTransformId); - } - - /** - * Rewrites the transform config according to the latest format, for example moving deprecated - * settings to its new place. - * - * @param transformConfig original config - * @return a rewritten transform config if a rewrite was necessary, otherwise the given transformConfig - */ - public static TransformConfig rewriteForUpdate(final TransformConfig transformConfig) { - - // quick checks for deprecated features, if none found just return the original - if (transformConfig.getPivotConfig() == null || transformConfig.getPivotConfig().getMaxPageSearchSize() == null) { - return transformConfig; - } - - Builder builder = new Builder(transformConfig); - - if (transformConfig.getPivotConfig() != null && transformConfig.getPivotConfig().getMaxPageSearchSize() != null) { - // create a new pivot config but set maxPageSearchSize to null - PivotConfig newPivotConfig = new PivotConfig( - transformConfig.getPivotConfig().getGroupConfig(), - transformConfig.getPivotConfig().getAggregationConfig(), - null - ); - builder.setPivotConfig(newPivotConfig); - - Integer maxPageSearchSizeDeprecated = transformConfig.getPivotConfig().getMaxPageSearchSize(); - Integer maxPageSearchSize = transformConfig.getSettings().getMaxPageSearchSize() != null - ? transformConfig.getSettings().getMaxPageSearchSize() - : maxPageSearchSizeDeprecated; - - builder.setSettings(new SettingsConfig(maxPageSearchSize, transformConfig.getSettings().getDocsPerSecond())); - } - return builder.setVersion(Version.CURRENT).build(); - } - - public static class Builder { - private String id; - private SourceConfig source; - private DestConfig dest; - private TimeValue frequency; - private SyncConfig syncConfig; - private String description; - private Map headers; - private Version transformVersion; - private Instant createTime; - private PivotConfig pivotConfig; - private MapConfig mapConfig; - private SettingsConfig settings; - - public Builder() {} - - public Builder(TransformConfig config) { - this.id = config.id; - this.source = config.source; - this.dest = config.dest; - this.frequency = config.frequency; - this.syncConfig = config.syncConfig; - this.description = config.description; - this.transformVersion = config.transformVersion; - this.createTime = config.createTime; - this.pivotConfig = config.pivotConfig; - this.settings = config.settings; - } - - public Builder setId(String id) { - this.id = id; - return this; - } - - public Builder setSource(SourceConfig source) { - this.source = source; - return this; - } - - public Builder setDest(DestConfig dest) { - this.dest = dest; - return this; - } - - public Builder setFrequency(TimeValue frequency) { - this.frequency = frequency; - return this; - } - - public Builder setSyncConfig(SyncConfig syncConfig) { - this.syncConfig = syncConfig; - return this; - } - - public Builder setDescription(String description) { - this.description = description; - return this; - } - - public Builder setSettings(SettingsConfig settings) { - this.settings = settings; - return this; - } - - public Builder setHeaders(Map headers) { - this.headers = headers; - return this; - } - - public Builder setPivotConfig(PivotConfig pivotConfig) { - this.pivotConfig = pivotConfig; - return this; - } - - public Builder setMapConfig(MapConfig mapConfig) { - this.mapConfig = mapConfig; - return this; - } - - Builder setVersion(Version version) { - this.transformVersion = version; - return this; - } - - public TransformConfig build() { - return new TransformConfig( - id, - source, - dest, - frequency, - syncConfig, - headers, - pivotConfig, - mapConfig, - description, - settings, - createTime, - transformVersion == null ? null : transformVersion.toString() - ); - } - - @Override - public boolean equals(Object other) { - if (this == other) { - return true; - } - - if (other == null || getClass() != other.getClass()) { - return false; - } - - final TransformConfig.Builder that = (TransformConfig.Builder) other; - - return Objects.equals(this.id, that.id) - && Objects.equals(this.source, that.source) - && Objects.equals(this.dest, that.dest) - && Objects.equals(this.frequency, that.frequency) - && Objects.equals(this.syncConfig, that.syncConfig) - && Objects.equals(this.headers, that.headers) - && Objects.equals(this.pivotConfig, that.pivotConfig) - && Objects.equals(this.mapConfig, that.mapConfig) - && Objects.equals(this.description, that.description) - && Objects.equals(this.settings, that.settings) - && Objects.equals(this.createTime, that.createTime) - && Objects.equals(this.transformVersion, that.transformVersion); - } - - @Override - public int hashCode() { - return Objects.hash( - id, - source, - dest, - frequency, - syncConfig, - headers, - pivotConfig, - mapConfig, - description, - settings, - createTime, - transformVersion - ); - } - } -} From 4ed9bb5e3bfdba65969a84daa95f787cef921697 Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Tue, 2 Feb 2021 10:04:21 +0100 Subject: [PATCH 19/39] add counters for delete to stats --- .../transforms/TransformIndexerStats.java | 38 ++++++++++++++-- .../TransformIndexerStatsTests.java | 6 +++ .../hlrc/TransformIndexerStatsTests.java | 2 + .../transforms/hlrc/TransformStatsTests.java | 2 + .../transforms/TransformIndexerStats.java | 43 +++++++++++++++++-- .../TimeRetentionPolicyConfigTests.java | 2 +- .../TransformIndexerStatsTests.java | 4 ++ .../TransformInfoTransportAction.java | 28 ++++++------ .../TransformInfoTransportActionTests.java | 26 +++++------ 9 files changed, 120 insertions(+), 31 deletions(-) diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/TransformIndexerStats.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/TransformIndexerStats.java index 8706659538d80..553515e635353 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/TransformIndexerStats.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/TransformIndexerStats.java @@ -26,10 +26,12 @@ public class TransformIndexerStats { static ParseField PAGES_PROCESSED = new ParseField("pages_processed"); static ParseField DOCUMENTS_PROCESSED = new ParseField("documents_processed"); static ParseField DOCUMENTS_INDEXED = new ParseField("documents_indexed"); + static ParseField DOCUMENTS_DELETED = new ParseField("documents_deleted"); static ParseField TRIGGER_COUNT = new ParseField("trigger_count"); static ParseField INDEX_TIME_IN_MS = new ParseField("index_time_in_ms"); static ParseField SEARCH_TIME_IN_MS = new ParseField("search_time_in_ms"); static ParseField PROCESSING_TIME_IN_MS = new ParseField("processing_time_in_ms"); + static ParseField DELETE_TIME_IN_MS = new ParseField("delete_time_in_ms"); static ParseField INDEX_TOTAL = new ParseField("index_total"); static ParseField SEARCH_TOTAL = new ParseField("search_total"); static ParseField PROCESSING_TOTAL = new ParseField("processing_total"); @@ -52,9 +54,11 @@ public class TransformIndexerStats { unboxSafe(args[9], 0L), unboxSafe(args[10], 0L), unboxSafe(args[11], 0L), - unboxSafe(args[12], 0.0), - unboxSafe(args[13], 0.0), - unboxSafe(args[14], 0.0) + unboxSafe(args[12], 0L), + unboxSafe(args[13], 0L), + unboxSafe(args[14], 0.0), + unboxSafe(args[15], 0.0), + unboxSafe(args[16], 0.0) ) ); @@ -62,10 +66,12 @@ public class TransformIndexerStats { LENIENT_PARSER.declareLong(optionalConstructorArg(), PAGES_PROCESSED); LENIENT_PARSER.declareLong(optionalConstructorArg(), DOCUMENTS_PROCESSED); LENIENT_PARSER.declareLong(optionalConstructorArg(), DOCUMENTS_INDEXED); + LENIENT_PARSER.declareLong(optionalConstructorArg(), DOCUMENTS_DELETED); LENIENT_PARSER.declareLong(optionalConstructorArg(), TRIGGER_COUNT); LENIENT_PARSER.declareLong(optionalConstructorArg(), INDEX_TIME_IN_MS); LENIENT_PARSER.declareLong(optionalConstructorArg(), SEARCH_TIME_IN_MS); LENIENT_PARSER.declareLong(optionalConstructorArg(), PROCESSING_TIME_IN_MS); + LENIENT_PARSER.declareLong(optionalConstructorArg(), DELETE_TIME_IN_MS); LENIENT_PARSER.declareLong(optionalConstructorArg(), INDEX_TOTAL); LENIENT_PARSER.declareLong(optionalConstructorArg(), SEARCH_TOTAL); LENIENT_PARSER.declareLong(optionalConstructorArg(), PROCESSING_TOTAL); @@ -86,12 +92,14 @@ public static TransformIndexerStats fromXContent(XContentParser parser) throws I private final long pagesProcessed; private final long documentsProcessed; private final long documentsIndexed; + private final long documentsDeleted; private final long triggerCount; private final long indexTime; private final long indexTotal; private final long searchTime; private final long searchTotal; private final long processingTime; + private final long deleteTime; private final long processingTotal; private final long indexFailures; private final long searchFailures; @@ -100,10 +108,12 @@ public TransformIndexerStats( long pagesProcessed, long documentsProcessed, long documentsIndexed, + long documentsDeleted, long triggerCount, long indexTime, long searchTime, long processingTime, + long deleteTime, long indexTotal, long searchTotal, long processingTotal, @@ -116,10 +126,12 @@ public TransformIndexerStats( this.pagesProcessed = pagesProcessed; this.documentsProcessed = documentsProcessed; this.documentsIndexed = documentsIndexed; + this.documentsDeleted = documentsDeleted; this.triggerCount = triggerCount; this.indexTime = indexTime; this.indexTotal = indexTotal; this.searchTime = searchTime; + this.deleteTime = deleteTime; this.searchTotal = searchTotal; this.processingTime = processingTime; this.processingTotal = processingTotal; @@ -206,6 +218,14 @@ public long getOutputDocuments() { return getDocumentsIndexed(); } + /** + * Number of documents deleted + */ + + public long getDocumentsDeleted() { + return documentsDeleted; + } + /** * Number of index failures that have occurred */ @@ -241,6 +261,14 @@ public long getProcessingTime() { return processingTime; } + /** + * Returns the time spent deleting (cumulative) in milliseconds + */ + public long getDeleteTime() { + return deleteTime; + } + + /** * Returns the total number of indexing requests that have been processed * (Note: this is not the number of _documents_ that have been indexed) @@ -278,10 +306,12 @@ public boolean equals(Object other) { return Objects.equals(this.pagesProcessed, that.pagesProcessed) && Objects.equals(this.documentsProcessed, that.documentsProcessed) && Objects.equals(this.documentsIndexed, that.documentsIndexed) + && Objects.equals(this.documentsDeleted, that.documentsDeleted) && Objects.equals(this.triggerCount, that.triggerCount) && Objects.equals(this.indexTime, that.indexTime) && Objects.equals(this.searchTime, that.searchTime) && Objects.equals(this.processingTime, that.processingTime) + && Objects.equals(this.deleteTime, that.deleteTime) && Objects.equals(this.indexFailures, that.indexFailures) && Objects.equals(this.searchFailures, that.searchFailures) && Objects.equals(this.indexTotal, that.indexTotal) @@ -298,10 +328,12 @@ public int hashCode() { pagesProcessed, documentsProcessed, documentsIndexed, + documentsDeleted, triggerCount, indexTime, searchTime, processingTime, + deleteTime, indexFailures, searchFailures, indexTotal, diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/TransformIndexerStatsTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/TransformIndexerStatsTests.java index 971dbb82622c1..beee811328923 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/TransformIndexerStatsTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/TransformIndexerStatsTests.java @@ -40,6 +40,8 @@ public static TransformIndexerStats randomStats() { randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), + randomNonNegativeLong(), + randomNonNegativeLong(), randomDouble(), randomDouble(), randomDouble() @@ -52,6 +54,7 @@ public static void toXContent(TransformIndexerStats stats, XContentBuilder build builder.field(TransformIndexerStats.PAGES_PROCESSED.getPreferredName(), stats.getPagesProcessed()); builder.field(TransformIndexerStats.DOCUMENTS_PROCESSED.getPreferredName(), stats.getDocumentsProcessed()); builder.field(TransformIndexerStats.DOCUMENTS_INDEXED.getPreferredName(), stats.getDocumentsIndexed()); + builder.field(TransformIndexerStats.DOCUMENTS_DELETED.getPreferredName(), stats.getDocumentsDeleted()); builder.field(TransformIndexerStats.TRIGGER_COUNT.getPreferredName(), stats.getTriggerCount()); builder.field(TransformIndexerStats.INDEX_TIME_IN_MS.getPreferredName(), stats.getIndexTime()); builder.field(TransformIndexerStats.INDEX_TOTAL.getPreferredName(), stats.getIndexTotal()); @@ -60,6 +63,7 @@ public static void toXContent(TransformIndexerStats stats, XContentBuilder build builder.field(TransformIndexerStats.SEARCH_TOTAL.getPreferredName(), stats.getSearchTotal()); builder.field(TransformIndexerStats.PROCESSING_TIME_IN_MS.getPreferredName(), stats.getProcessingTime()); builder.field(TransformIndexerStats.PROCESSING_TOTAL.getPreferredName(), stats.getProcessingTotal()); + builder.field(TransformIndexerStats.DELETE_TIME_IN_MS.getPreferredName(), stats.getDeleteTime()); builder.field(TransformIndexerStats.SEARCH_FAILURES.getPreferredName(), stats.getSearchFailures()); builder.field( TransformIndexerStats.EXPONENTIAL_AVG_CHECKPOINT_DURATION_MS.getPreferredName(), @@ -75,6 +79,7 @@ public static void toXContent(TransformIndexerStats stats, XContentBuilder build xContentFieldIfNotZero(builder, TransformIndexerStats.PAGES_PROCESSED.getPreferredName(), stats.getPagesProcessed()); xContentFieldIfNotZero(builder, TransformIndexerStats.DOCUMENTS_PROCESSED.getPreferredName(), stats.getDocumentsProcessed()); xContentFieldIfNotZero(builder, TransformIndexerStats.DOCUMENTS_INDEXED.getPreferredName(), stats.getDocumentsIndexed()); + xContentFieldIfNotZero(builder, TransformIndexerStats.DOCUMENTS_DELETED.getPreferredName(), stats.getDocumentsDeleted()); xContentFieldIfNotZero(builder, TransformIndexerStats.TRIGGER_COUNT.getPreferredName(), stats.getTriggerCount()); xContentFieldIfNotZero(builder, TransformIndexerStats.INDEX_TIME_IN_MS.getPreferredName(), stats.getIndexTime()); xContentFieldIfNotZero(builder, TransformIndexerStats.INDEX_TOTAL.getPreferredName(), stats.getIndexTotal()); @@ -83,6 +88,7 @@ public static void toXContent(TransformIndexerStats stats, XContentBuilder build xContentFieldIfNotZero(builder, TransformIndexerStats.SEARCH_TOTAL.getPreferredName(), stats.getSearchTotal()); xContentFieldIfNotZero(builder, TransformIndexerStats.PROCESSING_TIME_IN_MS.getPreferredName(), stats.getProcessingTime()); xContentFieldIfNotZero(builder, TransformIndexerStats.PROCESSING_TOTAL.getPreferredName(), stats.getProcessingTotal()); + xContentFieldIfNotZero(builder, TransformIndexerStats.DELETE_TIME_IN_MS.getPreferredName(), stats.getDeleteTime()); xContentFieldIfNotZero(builder, TransformIndexerStats.SEARCH_FAILURES.getPreferredName(), stats.getSearchFailures()); xContentFieldIfNotZero( builder, diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/hlrc/TransformIndexerStatsTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/hlrc/TransformIndexerStatsTests.java index 6faaa3d2a3510..92a828c63d114 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/hlrc/TransformIndexerStatsTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/hlrc/TransformIndexerStatsTests.java @@ -35,6 +35,8 @@ public static org.elasticsearch.xpack.core.transform.transforms.TransformIndexer randomLongBetween(0L, 10000L), randomLongBetween(0L, 10000L), randomLongBetween(0L, 10000L), + randomLongBetween(0L, 10000L), + randomLongBetween(0L, 10000L), randomDouble(), randomDouble(), randomDouble() diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/hlrc/TransformStatsTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/hlrc/TransformStatsTests.java index 103dd6790d6dd..b989b68aba13b 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/hlrc/TransformStatsTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/hlrc/TransformStatsTests.java @@ -156,8 +156,10 @@ public static void assertTransformIndexerStats( assertThat(serverTestInstance.getNumDocuments(), equalTo(clientInstance.getDocumentsProcessed())); assertThat(serverTestInstance.getNumInvocations(), equalTo(clientInstance.getTriggerCount())); assertThat(serverTestInstance.getOutputDocuments(), equalTo(clientInstance.getDocumentsIndexed())); + assertThat(serverTestInstance.getNumDeletedDocuments(), equalTo(clientInstance.getDocumentsDeleted())); assertThat(serverTestInstance.getSearchFailures(), equalTo(clientInstance.getSearchFailures())); assertThat(serverTestInstance.getSearchTime(), equalTo(clientInstance.getSearchTime())); assertThat(serverTestInstance.getSearchTotal(), equalTo(clientInstance.getSearchTotal())); + assertThat(serverTestInstance.getDeleteTime(), equalTo(clientInstance.getDeleteTime())); } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformIndexerStats.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformIndexerStats.java index 1f1d34a72dc91..eb0cc2cdef7e1 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformIndexerStats.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformIndexerStats.java @@ -30,10 +30,12 @@ public class TransformIndexerStats extends IndexerJobStats { public static ParseField NUM_PAGES = new ParseField("pages_processed"); public static ParseField NUM_INPUT_DOCUMENTS = new ParseField("documents_processed"); public static ParseField NUM_OUTPUT_DOCUMENTS = new ParseField("documents_indexed"); + public static ParseField NUM_DELETED_DOCUMENTS = new ParseField("documents_deleted"); public static ParseField NUM_INVOCATIONS = new ParseField("trigger_count"); public static ParseField INDEX_TIME_IN_MS = new ParseField("index_time_in_ms"); public static ParseField SEARCH_TIME_IN_MS = new ParseField("search_time_in_ms"); public static ParseField PROCESSING_TIME_IN_MS = new ParseField("processing_time_in_ms"); + public static ParseField DELETE_TIME_IN_MS = new ParseField("delete_time_in_ms"); public static ParseField INDEX_TOTAL = new ParseField("index_total"); public static ParseField SEARCH_TOTAL = new ParseField("search_total"); public static ParseField PROCESSING_TOTAL = new ParseField("processing_total"); @@ -64,9 +66,11 @@ public class TransformIndexerStats extends IndexerJobStats { unboxSafe(args[9], 0L), unboxSafe(args[10], 0L), unboxSafe(args[11], 0L), - unboxSafe(args[12], 0.0), - unboxSafe(args[13], 0.0), - unboxSafe(args[14], 0.0) + unboxSafe(args[12], 0L), + unboxSafe(args[13], 0L), + unboxSafe(args[14], 0.0), + unboxSafe(args[15], 0.0), + unboxSafe(args[16], 0.0) ) ); @@ -74,10 +78,12 @@ public class TransformIndexerStats extends IndexerJobStats { LENIENT_PARSER.declareLong(optionalConstructorArg(), NUM_PAGES); LENIENT_PARSER.declareLong(optionalConstructorArg(), NUM_INPUT_DOCUMENTS); LENIENT_PARSER.declareLong(optionalConstructorArg(), NUM_OUTPUT_DOCUMENTS); + LENIENT_PARSER.declareLong(optionalConstructorArg(), NUM_DELETED_DOCUMENTS); LENIENT_PARSER.declareLong(optionalConstructorArg(), NUM_INVOCATIONS); LENIENT_PARSER.declareLong(optionalConstructorArg(), INDEX_TIME_IN_MS); LENIENT_PARSER.declareLong(optionalConstructorArg(), SEARCH_TIME_IN_MS); LENIENT_PARSER.declareLong(optionalConstructorArg(), PROCESSING_TIME_IN_MS); + LENIENT_PARSER.declareLong(optionalConstructorArg(), DELETE_TIME_IN_MS); LENIENT_PARSER.declareLong(optionalConstructorArg(), INDEX_TOTAL); LENIENT_PARSER.declareLong(optionalConstructorArg(), SEARCH_TOTAL); LENIENT_PARSER.declareLong(optionalConstructorArg(), PROCESSING_TOTAL); @@ -88,6 +94,9 @@ public class TransformIndexerStats extends IndexerJobStats { LENIENT_PARSER.declareDouble(optionalConstructorArg(), EXPONENTIAL_AVG_DOCUMENTS_PROCESSED); } + private long deleteTime; + private long numDeletedDocuments; + private double expAvgCheckpointDurationMs; private double expAvgDocumentsIndexed; private double expAvgDocumentsProcessed; @@ -103,10 +112,12 @@ public TransformIndexerStats( long numPages, long numInputDocuments, long numOutputDocuments, + long numDeletedDocuments, long numInvocations, long indexTime, long searchTime, long processingTime, + long deleteTime, long indexTotal, long searchTotal, long processingTotal, @@ -130,6 +141,8 @@ public TransformIndexerStats( indexFailures, searchFailures ); + this.numDeletedDocuments = numDeletedDocuments; + this.deleteTime = deleteTime; this.expAvgCheckpointDurationMs = expAvgCheckpointDurationMs; this.expAvgDocumentsIndexed = expAvgDocumentsIndexed; this.expAvgDocumentsProcessed = expAvgDocumentsProcessed; @@ -140,10 +153,12 @@ public TransformIndexerStats(TransformIndexerStats other) { other.numPages, other.numInputDocuments, other.numOuputDocuments, + other.numDeletedDocuments, other.numInvocations, other.indexTime, other.searchTime, other.processingTime, + other.deleteTime, other.indexTotal, other.searchTotal, other.processingTotal, @@ -165,6 +180,10 @@ public TransformIndexerStats(StreamInput in) throws IOException { this.expAvgDocumentsIndexed = in.readDouble(); this.expAvgDocumentsProcessed = in.readDouble(); } + if (in.getVersion().onOrAfter(Version.V_8_0_0)) { // todo: V_7_12_0 + this.numDeletedDocuments = in.readVLong(); + this.deleteTime = in.readVLong(); + } } @Override @@ -178,6 +197,10 @@ public void writeTo(StreamOutput out) throws IOException { out.writeDouble(this.expAvgDocumentsIndexed); out.writeDouble(this.expAvgDocumentsProcessed); } + if (out.getVersion().onOrAfter(Version.V_8_0_0)) { // todo: V_7_12_0 + out.writeVLong(numDeletedDocuments); + out.writeVLong(deleteTime); + } } @Override @@ -186,6 +209,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(NUM_PAGES.getPreferredName(), numPages); builder.field(NUM_INPUT_DOCUMENTS.getPreferredName(), numInputDocuments); builder.field(NUM_OUTPUT_DOCUMENTS.getPreferredName(), numOuputDocuments); + builder.field(NUM_DELETED_DOCUMENTS.getPreferredName(), numDeletedDocuments); builder.field(NUM_INVOCATIONS.getPreferredName(), numInvocations); builder.field(INDEX_TIME_IN_MS.getPreferredName(), indexTime); builder.field(INDEX_TOTAL.getPreferredName(), indexTotal); @@ -195,6 +219,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(SEARCH_FAILURES.getPreferredName(), searchFailures); builder.field(PROCESSING_TIME_IN_MS.getPreferredName(), processingTime); builder.field(PROCESSING_TOTAL.getPreferredName(), processingTotal); + builder.field(DELETE_TIME_IN_MS.getPreferredName(), deleteTime); builder.field(EXPONENTIAL_AVG_CHECKPOINT_DURATION_MS.getPreferredName(), this.expAvgCheckpointDurationMs); builder.field(EXPONENTIAL_AVG_DOCUMENTS_INDEXED.getPreferredName(), this.expAvgDocumentsIndexed); builder.field(EXPONENTIAL_AVG_DOCUMENTS_PROCESSED.getPreferredName(), this.expAvgDocumentsProcessed); @@ -202,6 +227,14 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws return builder; } + public long getDeleteTime() { + return deleteTime; + } + + public long getNumDeletedDocuments() { + return numDeletedDocuments; + } + public double getExpAvgCheckpointDurationMs() { return expAvgCheckpointDurationMs; } @@ -249,10 +282,12 @@ public boolean equals(Object other) { return Objects.equals(this.numPages, that.numPages) && Objects.equals(this.numInputDocuments, that.numInputDocuments) && Objects.equals(this.numOuputDocuments, that.numOuputDocuments) + && Objects.equals(this.numDeletedDocuments, that.numDeletedDocuments) && Objects.equals(this.numInvocations, that.numInvocations) && Objects.equals(this.indexTime, that.indexTime) && Objects.equals(this.searchTime, that.searchTime) && Objects.equals(this.processingTime, that.processingTime) + && Objects.equals(this.deleteTime, that.deleteTime) && Objects.equals(this.indexFailures, that.indexFailures) && Objects.equals(this.searchFailures, that.searchFailures) && Objects.equals(this.indexTotal, that.indexTotal) @@ -269,10 +304,12 @@ public int hashCode() { numPages, numInputDocuments, numOuputDocuments, + numDeletedDocuments, numInvocations, indexTime, searchTime, processingTime, + deleteTime, indexFailures, searchFailures, indexTotal, diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TimeRetentionPolicyConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TimeRetentionPolicyConfigTests.java index 28c34f8c3e09b..2de0abd49f851 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TimeRetentionPolicyConfigTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TimeRetentionPolicyConfigTests.java @@ -16,7 +16,7 @@ public class TimeRetentionPolicyConfigTests extends AbstractSerializingTestCase { public static TimeRetentionPolicyConfig randomTimeRetentionPolicyConfig() { - return new TimeRetentionPolicyConfig(randomAlphaOfLengthBetween(1, 10), new TimeValue(randomNonNegativeLong())); + return new TimeRetentionPolicyConfig(randomAlphaOfLengthBetween(1, 10), new TimeValue(randomLongBetween(60000, Long.MAX_VALUE))); } @Override diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformIndexerStatsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformIndexerStatsTests.java index 86cb2bc28783e..3b43d95f0ca55 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformIndexerStatsTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformIndexerStatsTests.java @@ -34,6 +34,8 @@ public static TransformIndexerStats randomStats() { randomLongBetween(0L, 10000L), randomLongBetween(0L, 10000L), randomLongBetween(0L, 10000L), + randomLongBetween(0L, 10000L), + randomLongBetween(0L, 10000L), randomDouble(), randomDouble(), randomDouble() @@ -99,6 +101,7 @@ public static void toXContentIfNotZero(TransformIndexerStats stats, XContentBuil xContentFieldIfNotZero(builder, TransformIndexerStats.NUM_PAGES.getPreferredName(), stats.getNumPages()); xContentFieldIfNotZero(builder, TransformIndexerStats.NUM_INPUT_DOCUMENTS.getPreferredName(), stats.getNumDocuments()); xContentFieldIfNotZero(builder, TransformIndexerStats.NUM_OUTPUT_DOCUMENTS.getPreferredName(), stats.getOutputDocuments()); + xContentFieldIfNotZero(builder, TransformIndexerStats.NUM_DELETED_DOCUMENTS.getPreferredName(), stats.getNumDeletedDocuments()); xContentFieldIfNotZero(builder, TransformIndexerStats.NUM_INVOCATIONS.getPreferredName(), stats.getNumInvocations()); xContentFieldIfNotZero(builder, TransformIndexerStats.INDEX_TIME_IN_MS.getPreferredName(), stats.getIndexTime()); xContentFieldIfNotZero(builder, TransformIndexerStats.INDEX_TOTAL.getPreferredName(), stats.getIndexTotal()); @@ -108,6 +111,7 @@ public static void toXContentIfNotZero(TransformIndexerStats stats, XContentBuil xContentFieldIfNotZero(builder, TransformIndexerStats.PROCESSING_TIME_IN_MS.getPreferredName(), stats.getProcessingTime()); xContentFieldIfNotZero(builder, TransformIndexerStats.PROCESSING_TOTAL.getPreferredName(), stats.getProcessingTotal()); xContentFieldIfNotZero(builder, TransformIndexerStats.SEARCH_FAILURES.getPreferredName(), stats.getSearchFailures()); + xContentFieldIfNotZero(builder, TransformIndexerStats.DELETE_TIME_IN_MS.getPreferredName(), stats.getDeleteTime()); xContentFieldIfNotZero( builder, TransformIndexerStats.EXPONENTIAL_AVG_CHECKPOINT_DURATION_MS.getPreferredName(), diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/TransformInfoTransportAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/TransformInfoTransportAction.java index 9056d063f282e..48ca9148497a4 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/TransformInfoTransportAction.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/TransformInfoTransportAction.java @@ -43,10 +43,12 @@ public class TransformInfoTransportAction extends XPackInfoFeatureTransportActio TransformIndexerStats.NUM_PAGES.getPreferredName(), TransformIndexerStats.NUM_INPUT_DOCUMENTS.getPreferredName(), TransformIndexerStats.NUM_OUTPUT_DOCUMENTS.getPreferredName(), + TransformIndexerStats.NUM_DELETED_DOCUMENTS.getPreferredName(), TransformIndexerStats.NUM_INVOCATIONS.getPreferredName(), TransformIndexerStats.INDEX_TIME_IN_MS.getPreferredName(), TransformIndexerStats.SEARCH_TIME_IN_MS.getPreferredName(), TransformIndexerStats.PROCESSING_TIME_IN_MS.getPreferredName(), + TransformIndexerStats.DELETE_TIME_IN_MS.getPreferredName(), TransformIndexerStats.INDEX_TOTAL.getPreferredName(), TransformIndexerStats.SEARCH_TOTAL.getPreferredName(), TransformIndexerStats.PROCESSING_TOTAL.getPreferredName(), @@ -95,18 +97,20 @@ static TransformIndexerStats parseSearchAggs(SearchResponse searchResponse) { statisticsList.get(0).longValue(), // numPages statisticsList.get(1).longValue(), // numInputDocuments statisticsList.get(2).longValue(), // numOutputDocuments - statisticsList.get(3).longValue(), // numInvocations - statisticsList.get(4).longValue(), // indexTime - statisticsList.get(5).longValue(), // searchTime - statisticsList.get(6).longValue(), // processingTime - statisticsList.get(7).longValue(), // indexTotal - statisticsList.get(8).longValue(), // searchTotal - statisticsList.get(9).longValue(), // processingTotal - statisticsList.get(10).longValue(), // indexFailures - statisticsList.get(11).longValue(), // searchFailures - statisticsList.get(12), // exponential_avg_checkpoint_duration_ms - statisticsList.get(13), // exponential_avg_documents_indexed - statisticsList.get(14) // exponential_avg_documents_processed + statisticsList.get(3).longValue(), // numDeletedDocuments + statisticsList.get(4).longValue(), // numInvocations + statisticsList.get(5).longValue(), // indexTime + statisticsList.get(6).longValue(), // searchTime + statisticsList.get(7).longValue(), // processingTime + statisticsList.get(8).longValue(), // deleteTime + statisticsList.get(9).longValue(), // indexTotal + statisticsList.get(10).longValue(), // searchTotal + statisticsList.get(11).longValue(), // processingTotal + statisticsList.get(12).longValue(), // indexFailures + statisticsList.get(13).longValue(), // searchFailures + statisticsList.get(14), // exponential_avg_checkpoint_duration_ms + statisticsList.get(15), // exponential_avg_documents_indexed + statisticsList.get(16) // exponential_avg_documents_processed ); } diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/TransformInfoTransportActionTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/TransformInfoTransportActionTests.java index d1b71174337b4..fddb7c5a8c568 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/TransformInfoTransportActionTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/TransformInfoTransportActionTests.java @@ -55,18 +55,20 @@ public void testParseSearchAggs() { 1, // numPages 2, // numInputDocuments 3, // numOutputDocuments - 4, // numInvocations - 5, // indexTime - 6, // searchTime - 7, // processingTime - 8, // indexTotal - 9, // searchTotal - 10, // processingTotal - 11, // indexFailures - 12, // searchFailures - 13.0, // exponential_avg_checkpoint_duration_ms - 14.0, // exponential_avg_documents_indexed - 15.0 // exponential_avg_documents_processed + 4, // numDeletedDocuments + 5, // numInvocations + 6, // indexTime + 7, // searchTime + 8, // processingTime + 9, // deleteTime + 10, // indexTotal + 11, // searchTotal + 12, // processingTotal + 13, // indexFailures + 14, // searchFailures + 15.0, // exponential_avg_checkpoint_duration_ms + 16.0, // exponential_avg_documents_indexed + 17.0 // exponential_avg_documents_processed ); int currentStat = 1; From e7ea5e7b2f2825f5f85c3f129a1e688afe5e9f6b Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Tue, 2 Feb 2021 10:19:06 +0100 Subject: [PATCH 20/39] fix test compile errors --- .../src/test/java/org/elasticsearch/client/TransformIT.java | 2 ++ .../xpack/core/transform/transforms/TransformStatsTests.java | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/TransformIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/TransformIT.java index b8584900a469f..855130ae63980 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/TransformIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/TransformIT.java @@ -432,6 +432,8 @@ public void testGetStats() throws Exception { 0L, 0L, 0L, + 0L, + 0L, 0.0, 0.0, 0.0); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformStatsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformStatsTests.java index 6e06a1514678c..bee240f9f3798 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformStatsTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformStatsTests.java @@ -70,7 +70,7 @@ public void testBwcWith73() throws IOException { STARTED, randomBoolean() ? null : randomAlphaOfLength(100), randomBoolean() ? null : NodeAttributeTests.randomNodeAttributes(), - new TransformIndexerStats(1, 2, 3, 4, 5, 6, 0, 8, 9, 0, 11, 12, 0.0, 0.0, 0.0), + new TransformIndexerStats(1, 2, 3, 0, 5, 6, 7, 0, 0, 10, 11, 0, 13, 14, 0.0, 0.0, 0.0), new TransformCheckpointingInfo( new TransformCheckpointStats(0, null, null, 10, 100), new TransformCheckpointStats(0, null, null, 100, 1000), @@ -99,7 +99,7 @@ public void testBwcWith76() throws IOException { STARTED, randomBoolean() ? null : randomAlphaOfLength(100), randomBoolean() ? null : NodeAttributeTests.randomNodeAttributes(), - new TransformIndexerStats(1, 2, 3, 4, 5, 6, 0, 8, 9, 0, 11, 12, 13.0, 14.0, 15.0), + new TransformIndexerStats(1, 2, 3, 0, 5, 6, 7, 0, 0, 10, 11, 0, 13, 14, 15.0, 16.0, 17.0), new TransformCheckpointingInfo( new TransformCheckpointStats(0, null, null, 10, 100), new TransformCheckpointStats(0, null, null, 100, 1000), From f2d0db854b19f3412639f9d2ba9df0c9f0611a5b Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Wed, 3 Feb 2021 10:22:41 +0100 Subject: [PATCH 21/39] rename indexer tests --- ...ndexerTests.java => TransformIndexerFailureHandlingTests.java} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/{TransformIndexerTests.java => TransformIndexerFailureHandlingTests.java} (100%) diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerFailureHandlingTests.java similarity index 100% rename from x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerTests.java rename to x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerFailureHandlingTests.java From 91fc37f8864e65407ffd52c570a1ca93c1773554 Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Wed, 3 Feb 2021 10:24:01 +0100 Subject: [PATCH 22/39] add test to ensure retention policy gets executed --- .../MockTimebasedCheckpointProvider.java | 5 +- .../TransformIndexerFailureHandlingTests.java | 2 +- .../transforms/TransformIndexerTests.java | 346 ++++++++++++++++++ 3 files changed, 350 insertions(+), 3 deletions(-) create mode 100644 x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerTests.java diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/checkpoint/MockTimebasedCheckpointProvider.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/checkpoint/MockTimebasedCheckpointProvider.java index 4b51b40a9f673..b874cbf9140b5 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/checkpoint/MockTimebasedCheckpointProvider.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/checkpoint/MockTimebasedCheckpointProvider.java @@ -37,8 +37,9 @@ public void createNextCheckpoint(TransformCheckpoint lastCheckpoint, ActionListe final long timestamp = System.currentTimeMillis(); long timeUpperBound = timestamp - timeSyncConfig.getDelay().millis(); - if (lastCheckpoint == null) { - listener.onResponse(new TransformCheckpoint(transformConfig.getId(), timestamp, 0, Collections.emptyMap(), timeUpperBound)); + if (TransformCheckpoint.isNullOrEmpty(lastCheckpoint)) { + listener.onResponse(new TransformCheckpoint(transformConfig.getId(), timestamp, 1, Collections.emptyMap(), timeUpperBound)); + return; } listener.onResponse( diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerFailureHandlingTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerFailureHandlingTests.java index a70f7a7417eac..bfb9ffbe84b01 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerFailureHandlingTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerFailureHandlingTests.java @@ -88,7 +88,7 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; -public class TransformIndexerTests extends ESTestCase { +public class TransformIndexerFailureHandlingTests extends ESTestCase { private Client client; private ThreadPool threadPool; diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerTests.java new file mode 100644 index 0000000000000..bcd65f3d309fb --- /dev/null +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerTests.java @@ -0,0 +1,346 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.transform.transforms; + +import org.apache.lucene.search.TotalHits; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; +import org.elasticsearch.action.admin.indices.refresh.RefreshResponse; +import org.elasticsearch.action.bulk.BulkItemResponse; +import org.elasticsearch.action.bulk.BulkRequest; +import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.search.ShardSearchFailure; +import org.elasticsearch.client.Client; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.index.reindex.BulkByScrollResponse; +import org.elasticsearch.index.reindex.BulkByScrollTask; +import org.elasticsearch.index.reindex.DeleteByQueryRequest; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.SearchHits; +import org.elasticsearch.search.internal.InternalSearchResponse; +import org.elasticsearch.search.profile.SearchProfileShardResults; +import org.elasticsearch.search.suggest.Suggest; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.client.NoOpClient; +import org.elasticsearch.threadpool.TestThreadPool; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xpack.core.indexing.IndexerState; +import org.elasticsearch.xpack.core.indexing.IterationResult; +import org.elasticsearch.xpack.core.transform.transforms.TimeRetentionPolicyConfigTests; +import org.elasticsearch.xpack.core.transform.transforms.TimeSyncConfig; +import org.elasticsearch.xpack.core.transform.transforms.TransformCheckpoint; +import org.elasticsearch.xpack.core.transform.transforms.TransformConfig; +import org.elasticsearch.xpack.core.transform.transforms.TransformIndexerPosition; +import org.elasticsearch.xpack.core.transform.transforms.TransformIndexerStats; +import org.elasticsearch.xpack.core.transform.transforms.TransformTaskState; +import org.elasticsearch.xpack.transform.checkpoint.CheckpointProvider; +import org.elasticsearch.xpack.transform.checkpoint.MockTimebasedCheckpointProvider; +import org.elasticsearch.xpack.transform.notifications.MockTransformAuditor; +import org.elasticsearch.xpack.transform.notifications.TransformAuditor; +import org.elasticsearch.xpack.transform.persistence.InMemoryTransformConfigManager; +import org.elasticsearch.xpack.transform.persistence.TransformConfigManager; +import org.junit.After; +import org.junit.Before; + +import java.util.Collections; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; + +import static org.elasticsearch.xpack.core.transform.transforms.DestConfigTests.randomDestConfig; +import static org.elasticsearch.xpack.core.transform.transforms.SourceConfigTests.randomSourceConfig; +import static org.elasticsearch.xpack.core.transform.transforms.pivot.PivotConfigTests.randomPivotConfig; +import static org.mockito.Mockito.mock; + +public class TransformIndexerTests extends ESTestCase { + + private static final SearchResponse ONE_HIT_SEARCH_RESPONSE = new SearchResponse( + new InternalSearchResponse( + new SearchHits(new SearchHit[] { new SearchHit(1) }, new TotalHits(1L, TotalHits.Relation.EQUAL_TO), 1.0f), + // Simulate completely null aggs + null, + new Suggest(Collections.emptyList()), + new SearchProfileShardResults(Collections.emptyMap()), + false, + false, + 1 + ), + "", + 1, + 1, + 0, + 0, + ShardSearchFailure.EMPTY_ARRAY, + SearchResponse.Clusters.EMPTY + ); + + private Client client; + private ThreadPool threadPool; + private TransformAuditor auditor; + private TransformConfigManager transformConfigManager; + + class MockedTransformIndexer extends TransformIndexer { + + private final ThreadPool threadPool; + + private int deleteByQueryCallCount = 0; + // used for synchronizing with the test + private CountDownLatch searchLatch; + private CountDownLatch doProcessLatch; + + // how many loops to execute until reporting done + private int numberOfLoops; + + MockedTransformIndexer( + int numberOfLoops, + ThreadPool threadPool, + TransformConfigManager transformsConfigManager, + CheckpointProvider checkpointProvider, + TransformAuditor auditor, + TransformConfig transformConfig, + Map fieldMappings, + AtomicReference initialState, + TransformIndexerPosition initialPosition, + TransformIndexerStats jobStats, + TransformContext context + ) { + super( + threadPool, + transformsConfigManager, + checkpointProvider, + auditor, + transformConfig, + fieldMappings, + initialState, + initialPosition, + jobStats, + /* TransformProgress */ null, + TransformCheckpoint.EMPTY, + TransformCheckpoint.EMPTY, + context + ); + this.threadPool = threadPool; + this.numberOfLoops = numberOfLoops; + } + + public void initialize() { + this.initializeFunction(); + } + + public CountDownLatch createAwaitForSearchLatch(int count) { + return searchLatch = new CountDownLatch(count); + } + + public CountDownLatch createCountDownOnResponseLatch(int count) { + return doProcessLatch = new CountDownLatch(count); + } + + @Override + void doGetInitialProgress(SearchRequest request, ActionListener responseListener) { + responseListener.onResponse(ONE_HIT_SEARCH_RESPONSE); + } + + @Override + void doDeleteByQuery(DeleteByQueryRequest deleteByQueryRequest, ActionListener responseListener) { + deleteByQueryCallCount++; + responseListener.onResponse( + new BulkByScrollResponse( + TimeValue.ZERO, + new BulkByScrollTask.Status(Collections.emptyList(), null), + Collections.emptyList(), + Collections.emptyList(), + false + ) + ); + } + + @Override + void refreshDestinationIndex(RefreshRequest refreshRequest, ActionListener responseListener) { + responseListener.onResponse(new RefreshResponse(1, 1, 0, Collections.emptyList())); + } + + @Override + protected void doNextSearch(long waitTimeInNanos, ActionListener nextPhase) { + if (searchLatch != null) { + try { + searchLatch.await(); + } catch (InterruptedException e) { + throw new IllegalStateException(e); + } + } + threadPool.executor(ThreadPool.Names.GENERIC).execute(() -> nextPhase.onResponse(ONE_HIT_SEARCH_RESPONSE)); + } + + @Override + protected void doNextBulk(BulkRequest request, ActionListener nextPhase) { + if (doProcessLatch != null) { + doProcessLatch.countDown(); + } + threadPool.executor(ThreadPool.Names.GENERIC) + .execute(() -> nextPhase.onResponse(new BulkResponse(new BulkItemResponse[0], 100))); + } + + @Override + protected void doSaveState(IndexerState state, TransformIndexerPosition position, Runnable next) { + assert state == IndexerState.STARTED || state == IndexerState.INDEXING || state == IndexerState.STOPPED; + next.run(); + } + + @Override + protected IterationResult doProcess(SearchResponse searchResponse) { + assert numberOfLoops > 0; + --numberOfLoops; + // pretend that we processed 10k documents for each call + getStats().incrementNumDocuments(10_000); + return new IterationResult<>( + Collections.singletonList(new IndexRequest()), + new TransformIndexerPosition(null, null), + numberOfLoops == 0 + ); + } + + public boolean waitingForNextSearch() { + return super.getScheduledNextSearch() != null; + } + + public int getDeleteByQueryCallCount() { + return deleteByQueryCallCount; + } + } + + @Before + public void setUpMocks() { + auditor = MockTransformAuditor.createMockAuditor(); + transformConfigManager = new InMemoryTransformConfigManager(); + client = new NoOpClient(getTestName()); + threadPool = new TestThreadPool(ThreadPool.Names.GENERIC); + } + + @After + public void tearDownClient() { + client.close(); + ThreadPool.terminate(threadPool, 30, TimeUnit.SECONDS); + } + + public void testRetentionPolicyExecution() throws Exception { + TransformConfig config = new TransformConfig( + randomAlphaOfLength(10), + randomSourceConfig(), + randomDestConfig(), + null, + new TimeSyncConfig("timestamp", TimeValue.timeValueSeconds(1)), + null, + randomPivotConfig(), + null, + randomBoolean() ? null : randomAlphaOfLengthBetween(1, 1000), + null, + TimeRetentionPolicyConfigTests.randomTimeRetentionPolicyConfig(), + null, + null + ); + AtomicReference state = new AtomicReference<>(IndexerState.STARTED); + { + TransformContext context = new TransformContext(TransformTaskState.STARTED, "", 0, mock(TransformContext.Listener.class)); + final MockedTransformIndexer indexer = createMockIndexer( + 10, + config, + state, + null, + threadPool, + auditor, + new TransformIndexerStats(), + context + ); + + indexer.start(); + assertTrue(indexer.maybeTriggerAsyncJob(System.currentTimeMillis())); + assertEquals(indexer.getState(), IndexerState.INDEXING); + + assertBusy(() -> assertEquals(1L, indexer.getLastCheckpoint().getCheckpoint()), 5, TimeUnit.HOURS); + + // delete by query has been executed + assertEquals(1, indexer.getDeleteByQueryCallCount()); + } + + // test without retention + config = new TransformConfig( + randomAlphaOfLength(10), + randomSourceConfig(), + randomDestConfig(), + null, + new TimeSyncConfig("timestamp", TimeValue.timeValueSeconds(1)), + null, + randomPivotConfig(), + null, + randomBoolean() ? null : randomAlphaOfLengthBetween(1, 1000), + null, + null, + null, + null + ); + + state = new AtomicReference<>(IndexerState.STARTED); + { + TransformContext context = new TransformContext(TransformTaskState.STARTED, "", 0, mock(TransformContext.Listener.class)); + final MockedTransformIndexer indexer = createMockIndexer( + 10, + config, + state, + null, + threadPool, + auditor, + new TransformIndexerStats(), + context + ); + + indexer.start(); + assertTrue(indexer.maybeTriggerAsyncJob(System.currentTimeMillis())); + assertEquals(indexer.getState(), IndexerState.INDEXING); + + assertBusy(() -> assertEquals(1L, indexer.getLastCheckpoint().getCheckpoint()), 5, TimeUnit.SECONDS); + + // delete by query has _not_ been executed + assertEquals(0, indexer.getDeleteByQueryCallCount()); + } + } + + private MockedTransformIndexer createMockIndexer( + int numberOfLoops, + TransformConfig config, + AtomicReference state, + Consumer failureConsumer, + ThreadPool threadPool, + TransformAuditor auditor, + TransformIndexerStats jobStats, + TransformContext context + ) { + CheckpointProvider checkpointProvider = new MockTimebasedCheckpointProvider(config); + transformConfigManager.putTransformConfiguration(config, ActionListener.wrap(r -> {}, e -> {})); + + MockedTransformIndexer indexer = new MockedTransformIndexer( + numberOfLoops, + threadPool, + transformConfigManager, + checkpointProvider, + auditor, + config, + Collections.emptyMap(), + state, + null, + jobStats, + context + ); + + indexer.initialize(); + return indexer; + } +} From ad233b5ca37c952774203043588646da6a901c1f Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Wed, 3 Feb 2021 10:24:43 +0100 Subject: [PATCH 23/39] add tests for retention policy to DBQ conversion --- ...PolicyToDeleteByQueryRequestConverter.java | 21 ++++ .../transforms/TransformContext.java | 2 +- .../transforms/TransformIndexer.java | 6 ++ ...ntionPolicyConfigToDeleteByQueryTests.java | 98 +++++++++++++++++++ 4 files changed, 126 insertions(+), 1 deletion(-) create mode 100644 x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/RetentionPolicyConfigToDeleteByQueryTests.java diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/RetentionPolicyToDeleteByQueryRequestConverter.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/RetentionPolicyToDeleteByQueryRequestConverter.java index 924287fbb43ef..299c07b2c36ec 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/RetentionPolicyToDeleteByQueryRequestConverter.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/RetentionPolicyToDeleteByQueryRequestConverter.java @@ -19,6 +19,11 @@ import java.time.Instant; +/** + * Implementation of `retention_policy` configuration parameter. + * + * All implementations of `rentention_policy` are converted to a {@link DeleteByQueryRequest}, which is than executed by the indexer. + */ public final class RetentionPolicyToDeleteByQueryRequestConverter { public static class RetentionPolicyException extends ElasticsearchException { @@ -29,12 +34,28 @@ public static class RetentionPolicyException extends ElasticsearchException { private RetentionPolicyToDeleteByQueryRequestConverter() {} + /** + * Build a {@link DeleteByQueryRequest} from a retention policy. The DBQ should run _after_ a new checkpoint has finished. + * The given checkpoint should be the one that just finished indexing, however the DBQ executes before this checkpoint + * gets exposed. + * + * @param retentionPolicyConfig The retention policy configuration + * @param settingsConfig settings to set certain parameters + * @param destConfig the destination config + * @param checkpoint The checkpoint that just finished + * + * @return a delete by query request according to the given configurations or null if no delete by query should be executed + */ static DeleteByQueryRequest buildDeleteByQueryRequest( RetentionPolicyConfig retentionPolicyConfig, SettingsConfig settingsConfig, DestConfig destConfig, TransformCheckpoint checkpoint ) { + if (checkpoint == null || checkpoint.isEmpty()) { + return null; + } + DeleteByQueryRequest request = new DeleteByQueryRequest(); if (retentionPolicyConfig instanceof TimeRetentionPolicyConfig) { diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformContext.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformContext.java index c13dae70db742..5dc8ce143e9e1 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformContext.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformContext.java @@ -43,7 +43,7 @@ public interface Listener { this.currentCheckpoint = new AtomicLong(currentCheckpoint); this.taskListener = taskListener; this.failureCount = new AtomicInteger(0); - this.shouldStopAtCheckpoint = shouldStopAtCheckpoint; + this.shouldStopAtCheckpoint = false; } TransformTaskState getTaskState() { diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformIndexer.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformIndexer.java index 99d1926f0b945..85d908d42a069 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformIndexer.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformIndexer.java @@ -447,6 +447,12 @@ private void executeRetentionPolicy(ActionListener listener) { transformConfig.getDestination(), nextCheckpoint ); + + if (deleteByQuery == null) { + finalizeCheckpoint(listener); + return; + } + logger.debug( "[{}] Run delete based on retention policy using dbq [{}] with query: [{}]", getJobId(), diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/RetentionPolicyConfigToDeleteByQueryTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/RetentionPolicyConfigToDeleteByQueryTests.java new file mode 100644 index 0000000000000..8c2b3fdc290f0 --- /dev/null +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/RetentionPolicyConfigToDeleteByQueryTests.java @@ -0,0 +1,98 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.transform.transforms; + +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.RangeQueryBuilder; +import org.elasticsearch.index.reindex.DeleteByQueryRequest; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.core.transform.transforms.DestConfig; +import org.elasticsearch.xpack.core.transform.transforms.DestConfigTests; +import org.elasticsearch.xpack.core.transform.transforms.RetentionPolicyConfig; +import org.elasticsearch.xpack.core.transform.transforms.SettingsConfig; +import org.elasticsearch.xpack.core.transform.transforms.SettingsConfigTests; +import org.elasticsearch.xpack.core.transform.transforms.TimeRetentionPolicyConfig; +import org.elasticsearch.xpack.core.transform.transforms.TimeRetentionPolicyConfigTests; +import org.elasticsearch.xpack.core.transform.transforms.TransformCheckpoint; + +import java.util.Collections; + +import static org.hamcrest.CoreMatchers.instanceOf; +import static org.hamcrest.Matchers.equalTo; + +public class RetentionPolicyConfigToDeleteByQueryTests extends ESTestCase { + + public void testRetentionPolicyConfigBasics() { + // null checkpoint + assertNull( + RetentionPolicyToDeleteByQueryRequestConverter.buildDeleteByQueryRequest( + randomRetentionPolicyConfig(), + SettingsConfigTests.randomSettingsConfig(), + DestConfigTests.randomDestConfig(), + null + ) + ); + + // empty checkpoint + assertNull( + RetentionPolicyToDeleteByQueryRequestConverter.buildDeleteByQueryRequest( + randomRetentionPolicyConfig(), + SettingsConfigTests.randomSettingsConfig(), + DestConfigTests.randomDestConfig(), + TransformCheckpoint.EMPTY + ) + ); + } + + public void testTimeBasedRetentionPolicyConfig() { + SettingsConfig settingsConfig = SettingsConfigTests.randomSettingsConfig(); + DestConfig destConfig = DestConfigTests.randomDestConfig(); + + RetentionPolicyConfig retentionPolicyConfig = new TimeRetentionPolicyConfig( + randomAlphaOfLengthBetween(3, 5), + new TimeValue(60_000L) + ); + TransformCheckpoint checkpoint = new TransformCheckpoint( + randomAlphaOfLengthBetween(3, 5), + 10_000_000L, + 1, + Collections.emptyMap(), + 8_000_000L + ); + DeleteByQueryRequest deleteByQueryRequest = RetentionPolicyToDeleteByQueryRequestConverter.buildDeleteByQueryRequest( + retentionPolicyConfig, + settingsConfig, + destConfig, + checkpoint + ); + + assertNotNull(deleteByQueryRequest); + assertNotNull(deleteByQueryRequest.getSearchRequest().source()); + assertNotNull(deleteByQueryRequest.getSearchRequest().source().query()); + + assertTrue(deleteByQueryRequest.isRefresh()); + assertEquals(0, deleteByQueryRequest.getMaxRetries()); + assertEquals(1, deleteByQueryRequest.getSearchRequest().indices().length); + assertEquals(destConfig.getIndex(), deleteByQueryRequest.getSearchRequest().indices()[0]); + + if (settingsConfig.getDocsPerSecond() != null) { + assertEquals(settingsConfig.getDocsPerSecond().floatValue(), deleteByQueryRequest.getRequestsPerSecond(), 1E-15); + } + + QueryBuilder query = deleteByQueryRequest.getSearchRequest().source().query(); + assertThat(query, instanceOf(RangeQueryBuilder.class)); + RangeQueryBuilder rangeQuery = (RangeQueryBuilder) query; + + assertThat(rangeQuery.to(), equalTo(9_940_000L)); + assertTrue(rangeQuery.includeLower()); + } + + private static RetentionPolicyConfig randomRetentionPolicyConfig() { + return TimeRetentionPolicyConfigTests.randomTimeRetentionPolicyConfig(); + } +} From 3880d88f266f57723217e8d39b4a93ffac746961 Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Wed, 3 Feb 2021 12:35:46 +0100 Subject: [PATCH 24/39] update license headers --- .../transforms/RetentionPolicyConfig.java | 21 +++++-------------- .../transforms/TimeRetentionPolicyConfig.java | 21 +++++-------------- .../TimeRetentionPolicyConfigTests.java | 21 +++++-------------- .../hlrc/TimeRetentionPolicyConfigTests.java | 21 +++++-------------- .../transforms/RetentionPolicyConfig.java | 5 +++-- .../transforms/TimeRetentionPolicyConfig.java | 5 +++-- .../TimeRetentionPolicyConfigTests.java | 5 +++-- ...PolicyToDeleteByQueryRequestConverter.java | 5 +++-- ...ntionPolicyConfigToDeleteByQueryTests.java | 5 +++-- .../transforms/TransformIndexerTests.java | 5 +++-- 10 files changed, 38 insertions(+), 76 deletions(-) diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/RetentionPolicyConfig.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/RetentionPolicyConfig.java index 639ca82b877f3..9ad72c8db3f6f 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/RetentionPolicyConfig.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/RetentionPolicyConfig.java @@ -1,20 +1,9 @@ /* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. */ package org.elasticsearch.client.transform.transforms; diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/TimeRetentionPolicyConfig.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/TimeRetentionPolicyConfig.java index eaf2838a2dd08..cbff1ce71e18c 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/TimeRetentionPolicyConfig.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/TimeRetentionPolicyConfig.java @@ -1,20 +1,9 @@ /* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. */ package org.elasticsearch.client.transform.transforms; diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/TimeRetentionPolicyConfigTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/TimeRetentionPolicyConfigTests.java index b2e23cf5fe6b3..33be3e535d5e6 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/TimeRetentionPolicyConfigTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/TimeRetentionPolicyConfigTests.java @@ -1,20 +1,9 @@ /* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. */ package org.elasticsearch.client.transform.transforms; diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/hlrc/TimeRetentionPolicyConfigTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/hlrc/TimeRetentionPolicyConfigTests.java index 913de99053ee0..6b12921881d71 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/hlrc/TimeRetentionPolicyConfigTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/hlrc/TimeRetentionPolicyConfigTests.java @@ -1,20 +1,9 @@ /* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. */ package org.elasticsearch.client.transform.transforms.hlrc; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/RetentionPolicyConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/RetentionPolicyConfig.java index a1e047ef1f5f6..2fb1e340a8041 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/RetentionPolicyConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/RetentionPolicyConfig.java @@ -1,7 +1,8 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ package org.elasticsearch.xpack.core.transform.transforms; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TimeRetentionPolicyConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TimeRetentionPolicyConfig.java index 238cbe7f60061..f70881895b02c 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TimeRetentionPolicyConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TimeRetentionPolicyConfig.java @@ -1,7 +1,8 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ package org.elasticsearch.xpack.core.transform.transforms; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TimeRetentionPolicyConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TimeRetentionPolicyConfigTests.java index 2de0abd49f851..15c3070be8caa 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TimeRetentionPolicyConfigTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TimeRetentionPolicyConfigTests.java @@ -1,7 +1,8 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ package org.elasticsearch.xpack.core.transform.transforms; diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/RetentionPolicyToDeleteByQueryRequestConverter.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/RetentionPolicyToDeleteByQueryRequestConverter.java index 299c07b2c36ec..5d262af0a0d65 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/RetentionPolicyToDeleteByQueryRequestConverter.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/RetentionPolicyToDeleteByQueryRequestConverter.java @@ -1,7 +1,8 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ package org.elasticsearch.xpack.transform.transforms; diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/RetentionPolicyConfigToDeleteByQueryTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/RetentionPolicyConfigToDeleteByQueryTests.java index 8c2b3fdc290f0..8938467c0548c 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/RetentionPolicyConfigToDeleteByQueryTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/RetentionPolicyConfigToDeleteByQueryTests.java @@ -1,7 +1,8 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ package org.elasticsearch.xpack.transform.transforms; diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerTests.java index bcd65f3d309fb..31b269ff2f942 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerTests.java @@ -1,7 +1,8 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ package org.elasticsearch.xpack.transform.transforms; From 7e63ad33cd2ab58c0537b29b74d1a18601c2a756 Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Wed, 3 Feb 2021 14:42:00 +0100 Subject: [PATCH 25/39] make retention policy updateable --- .../transform/transforms/TransformConfig.java | 4 +- .../transforms/TransformConfigUpdate.java | 39 ++++++++++++-- .../TransformConfigUpdateTests.java | 51 ++++++++++++++----- .../TransformIndexerFailureHandlingTests.java | 36 ++++++++++--- 4 files changed, 106 insertions(+), 24 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java index b208e97cb602c..fb999be9a2553 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java @@ -149,7 +149,7 @@ private static ConstructingObjectParser createParser(bo parser.declareObject(optionalConstructorArg(), (p, c) -> LatestConfig.fromXContent(p, lenient), LATEST_TRANSFORM); parser.declareString(optionalConstructorArg(), TransformField.DESCRIPTION); parser.declareObject(optionalConstructorArg(), (p, c) -> SettingsConfig.fromXContent(p, lenient), TransformField.SETTINGS); - parser.declareObject(optionalConstructorArg(), (p, c) -> parseRentionPolicyConfig(p), TransformField.RETENTION_POLICY); + parser.declareObject(optionalConstructorArg(), (p, c) -> parseRetentionPolicyConfig(p), TransformField.RETENTION_POLICY); parser.declareField( optionalConstructorArg(), p -> TimeUtils.parseTimeFieldToInstant(p, TransformField.CREATE_TIME.getPreferredName()), @@ -168,7 +168,7 @@ private static SyncConfig parseSyncConfig(XContentParser parser, boolean ignoreU return syncConfig; } - private static RetentionPolicyConfig parseRentionPolicyConfig(XContentParser parser) throws IOException { + private static RetentionPolicyConfig parseRetentionPolicyConfig(XContentParser parser) throws IOException { XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser); XContentParserUtils.ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.nextToken(), parser); RetentionPolicyConfig retentionPolicyConfig = parser.namedObject(RetentionPolicyConfig.class, parser.currentName(), true); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigUpdate.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigUpdate.java index acd293c52ee8b..0ab6e9cd1f2f9 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigUpdate.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigUpdate.java @@ -47,7 +47,8 @@ public class TransformConfigUpdate implements Writeable { SyncConfig syncConfig = (SyncConfig) args[3]; String description = (String) args[4]; SettingsConfig settings = (SettingsConfig) args[5]; - return new TransformConfigUpdate(source, dest, frequency, syncConfig, description, settings); + RetentionPolicyConfig retentionPolicyConfig = (RetentionPolicyConfig) args[6]; + return new TransformConfigUpdate(source, dest, frequency, syncConfig, description, settings, retentionPolicyConfig); } ); @@ -58,6 +59,7 @@ public class TransformConfigUpdate implements Writeable { PARSER.declareObject(optionalConstructorArg(), (p, c) -> parseSyncConfig(p), TransformField.SYNC); PARSER.declareString(optionalConstructorArg(), TransformField.DESCRIPTION); PARSER.declareObject(optionalConstructorArg(), (p, c) -> SettingsConfig.fromXContent(p, false), TransformField.SETTINGS); + PARSER.declareObject(optionalConstructorArg(), (p, c) -> parseRetentionPolicyConfig(p), TransformField.RETENTION_POLICY); } private static SyncConfig parseSyncConfig(XContentParser parser) throws IOException { @@ -68,12 +70,21 @@ private static SyncConfig parseSyncConfig(XContentParser parser) throws IOExcept return syncConfig; } + private static RetentionPolicyConfig parseRetentionPolicyConfig(XContentParser parser) throws IOException { + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser); + XContentParserUtils.ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.nextToken(), parser); + RetentionPolicyConfig retentionPolicyConfig = parser.namedObject(RetentionPolicyConfig.class, parser.currentName(), true); + XContentParserUtils.ensureExpectedToken(XContentParser.Token.END_OBJECT, parser.nextToken(), parser); + return retentionPolicyConfig; + } + private final SourceConfig source; private final DestConfig dest; private final TimeValue frequency; private final SyncConfig syncConfig; private final String description; private final SettingsConfig settings; + private final RetentionPolicyConfig retentionPolicyConfig; private Map headers; public TransformConfigUpdate( @@ -82,7 +93,8 @@ public TransformConfigUpdate( final TimeValue frequency, final SyncConfig syncConfig, final String description, - final SettingsConfig settings + final SettingsConfig settings, + final RetentionPolicyConfig retentionPolicyConfig ) { this.source = source; this.dest = dest; @@ -93,6 +105,7 @@ public TransformConfigUpdate( throw new IllegalArgumentException("[description] must be less than 1000 characters in length."); } this.settings = settings; + this.retentionPolicyConfig = retentionPolicyConfig; } public TransformConfigUpdate(final StreamInput in) throws IOException { @@ -109,7 +122,11 @@ public TransformConfigUpdate(final StreamInput in) throws IOException { } else { settings = null; } - + if (in.getVersion().onOrAfter(Version.V_8_0_0)) { // todo: V_7_12_0 + retentionPolicyConfig = in.readOptionalNamedWriteable(RetentionPolicyConfig.class); + } else { + retentionPolicyConfig = null; + } } public SourceConfig getSource() { @@ -138,6 +155,11 @@ public SettingsConfig getSettings() { return settings; } + @Nullable + public RetentionPolicyConfig getRetentionPolicyConfig() { + return retentionPolicyConfig; + } + public Map getHeaders() { return headers; } @@ -162,6 +184,9 @@ public void writeTo(final StreamOutput out) throws IOException { if (out.getVersion().onOrAfter(Version.V_7_8_0)) { out.writeOptionalWriteable(settings); } + if (out.getVersion().onOrAfter(Version.V_8_0_0)) { // todo: V_7_12_0 + out.writeOptionalNamedWriteable(retentionPolicyConfig); + } } @Override @@ -182,12 +207,13 @@ public boolean equals(Object other) { && Objects.equals(this.syncConfig, that.syncConfig) && Objects.equals(this.description, that.description) && Objects.equals(this.settings, that.settings) + && Objects.equals(this.retentionPolicyConfig, that.retentionPolicyConfig) && Objects.equals(this.headers, that.headers); } @Override public int hashCode() { - return Objects.hash(source, dest, frequency, syncConfig, description, settings, headers); + return Objects.hash(source, dest, frequency, syncConfig, description, settings, retentionPolicyConfig, headers); } public static TransformConfigUpdate fromXContent(final XContentParser parser) { @@ -201,6 +227,7 @@ && isNullOrEqual(frequency, config.getFrequency()) && isNullOrEqual(syncConfig, config.getSyncConfig()) && isNullOrEqual(description, config.getDescription()) && isNullOrEqual(settings, config.getSettings()) + && isNullOrEqual(retentionPolicyConfig, config.getRetentionPolicyConfig()) && isNullOrEqual(headers, config.getHeaders()); } @@ -253,6 +280,10 @@ public TransformConfig apply(TransformConfig config) { settingsBuilder.update(settings); builder.setSettings(settingsBuilder.build()); } + if (retentionPolicyConfig != null) { + builder.setRetentionPolicyConfig(retentionPolicyConfig); + } + builder.setVersion(Version.CURRENT); return builder.build(); } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigUpdateTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigUpdateTests.java index bb0705355d96b..ef44035a4e423 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigUpdateTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigUpdateTests.java @@ -38,7 +38,8 @@ public static TransformConfigUpdate randomTransformConfigUpdate() { randomBoolean() ? null : TimeValue.timeValueMillis(randomIntBetween(1_000, 3_600_000)), randomBoolean() ? null : randomSyncConfig(), randomBoolean() ? null : randomAlphaOfLengthBetween(1, 1000), - randomBoolean() ? null : SettingsConfigTests.randomSettingsConfig() + randomBoolean() ? null : SettingsConfigTests.randomSettingsConfig(), + randomBoolean() ? null : randomRetentionPolicyConfig() ); } @@ -55,7 +56,7 @@ protected Reader instanceReader() { public void testIsNoop() { for (int i = 0; i < NUMBER_OF_TEST_RUNS; i++) { TransformConfig config = randomTransformConfig(); - TransformConfigUpdate update = new TransformConfigUpdate(null, null, null, null, null, null); + TransformConfigUpdate update = new TransformConfigUpdate(null, null, null, null, null, null, null); assertTrue("null update is not noop", update.isNoop(config)); update = new TransformConfigUpdate( config.getSource(), @@ -63,7 +64,8 @@ public void testIsNoop() { config.getFrequency(), config.getSyncConfig(), config.getDescription(), - config.getSettings() + config.getSettings(), + config.getRetentionPolicyConfig() ); assertTrue("equal update is not noop", update.isNoop(config)); @@ -73,7 +75,8 @@ public void testIsNoop() { config.getFrequency(), config.getSyncConfig(), "this is a new description", - config.getSettings() + config.getSettings(), + config.getRetentionPolicyConfig() ); assertFalse("true update is noop", update.isNoop(config)); } @@ -95,7 +98,7 @@ public void testApply() { randomBoolean() ? null : Instant.now(), randomBoolean() ? null : Version.V_7_2_0.toString() ); - TransformConfigUpdate update = new TransformConfigUpdate(null, null, null, null, null, null); + TransformConfigUpdate update = new TransformConfigUpdate(null, null, null, null, null, null, null); assertThat(config, equalTo(update.apply(config))); SourceConfig sourceConfig = new SourceConfig("the_new_index"); @@ -104,7 +107,16 @@ public void testApply() { SyncConfig syncConfig = new TimeSyncConfig("time_field", TimeValue.timeValueSeconds(30)); String newDescription = "new description"; SettingsConfig settings = new SettingsConfig(4_000, 4_000.400F, true); - update = new TransformConfigUpdate(sourceConfig, destConfig, frequency, syncConfig, newDescription, settings); + RetentionPolicyConfig retentionPolicyConfig = new TimeRetentionPolicyConfig("time_field", new TimeValue(60_000)); + update = new TransformConfigUpdate( + sourceConfig, + destConfig, + frequency, + syncConfig, + newDescription, + settings, + retentionPolicyConfig + ); Map headers = Collections.singletonMap("foo", "bar"); update.setHeaders(headers); @@ -116,6 +128,7 @@ public void testApply() { assertThat(updatedConfig.getSyncConfig(), equalTo(syncConfig)); assertThat(updatedConfig.getDescription(), equalTo(newDescription)); assertThat(updatedConfig.getSettings(), equalTo(settings)); + assertThat(updatedConfig.getRetentionPolicyConfig(), equalTo(retentionPolicyConfig)); assertThat(updatedConfig.getHeaders(), equalTo(headers)); assertThat(updatedConfig.getVersion(), equalTo(Version.CURRENT)); } @@ -143,7 +156,8 @@ public void testApplySettings() { null, null, null, - new SettingsConfig(4_000, null, (Boolean) null) + new SettingsConfig(4_000, null, (Boolean) null), + null ); TransformConfig updatedConfig = update.apply(config); @@ -152,18 +166,18 @@ public void testApplySettings() { assertThat(updatedConfig.getSettings().getMaxPageSearchSize(), equalTo(4_000)); assertThat(updatedConfig.getSettings().getDocsPerSecond(), equalTo(config.getSettings().getDocsPerSecond())); - update = new TransformConfigUpdate(null, null, null, null, null, new SettingsConfig(null, 43.244F, (Boolean) null)); + update = new TransformConfigUpdate(null, null, null, null, null, new SettingsConfig(null, 43.244F, (Boolean) null), null); updatedConfig = update.apply(updatedConfig); assertThat(updatedConfig.getSettings().getMaxPageSearchSize(), equalTo(4_000)); assertThat(updatedConfig.getSettings().getDocsPerSecond(), equalTo(43.244F)); // now reset to default using the magic -1 - update = new TransformConfigUpdate(null, null, null, null, null, new SettingsConfig(-1, null, (Boolean) null)); + update = new TransformConfigUpdate(null, null, null, null, null, new SettingsConfig(-1, null, (Boolean) null), null); updatedConfig = update.apply(updatedConfig); assertNull(updatedConfig.getSettings().getMaxPageSearchSize()); assertThat(updatedConfig.getSettings().getDocsPerSecond(), equalTo(43.244F)); - update = new TransformConfigUpdate(null, null, null, null, null, new SettingsConfig(-1, -1F, (Boolean) null)); + update = new TransformConfigUpdate(null, null, null, null, null, new SettingsConfig(-1, -1F, (Boolean) null), null); updatedConfig = update.apply(updatedConfig); assertNull(updatedConfig.getSettings().getMaxPageSearchSize()); assertNull(updatedConfig.getSettings().getDocsPerSecond()); @@ -186,7 +200,15 @@ public void testApplyWithSyncChange() { randomBoolean() ? null : Version.CURRENT.toString() ); - TransformConfigUpdate update = new TransformConfigUpdate(null, null, null, TimeSyncConfigTests.randomTimeSyncConfig(), null, null); + TransformConfigUpdate update = new TransformConfigUpdate( + null, + null, + null, + TimeSyncConfigTests.randomTimeSyncConfig(), + null, + null, + null + ); ElasticsearchStatusException ex = expectThrows(ElasticsearchStatusException.class, () -> update.apply(batchConfig)); assertThat( @@ -210,7 +232,7 @@ public void testApplyWithSyncChange() { randomBoolean() ? null : Version.CURRENT.toString() ); - TransformConfigUpdate fooSyncUpdate = new TransformConfigUpdate(null, null, null, new FooSync(), null, null); + TransformConfigUpdate fooSyncUpdate = new TransformConfigUpdate(null, null, null, new FooSync(), null, null, null); ex = expectThrows(ElasticsearchStatusException.class, () -> fooSyncUpdate.apply(timeSyncedConfig)); assertThat( ex.getMessage(), @@ -250,6 +272,11 @@ private void toXContent(TransformConfigUpdate update, XContentBuilder builder) t if (update.getSettings() != null) { builder.field(TransformField.SETTINGS.getPreferredName(), update.getSettings()); } + if (update.getRetentionPolicyConfig() != null) { + builder.startObject(TransformField.RETENTION_POLICY.getPreferredName()); + builder.field(update.getRetentionPolicyConfig().getWriteableName(), update.getRetentionPolicyConfig()); + builder.endObject(); + } builder.endObject(); } diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerFailureHandlingTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerFailureHandlingTests.java index bfb9ffbe84b01..b74dd7d7b7f8b 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerFailureHandlingTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerFailureHandlingTests.java @@ -514,9 +514,21 @@ public void testInitializeFunction_WithNoWarnings() { }}); SyncConfig syncConfig = new TimeSyncConfig("field", null); LatestConfig latestConfig = new LatestConfig(Arrays.asList("field-A", "field-B"), "sort"); - TransformConfig config = - new TransformConfig( - transformId, sourceConfig, randomDestConfig(), null, syncConfig, null, null, latestConfig, null, null, null, null); + TransformConfig config = new TransformConfig( + transformId, + sourceConfig, + randomDestConfig(), + null, + syncConfig, + null, + null, + latestConfig, + null, + null, + null, + null, + null + ); MockTransformAuditor auditor = MockTransformAuditor.createMockAuditor(); auditor.addExpectation( @@ -545,9 +557,21 @@ public void testInitializeFunction_WithWarnings() { }}); SyncConfig syncConfig = new TimeSyncConfig("field-t", null); LatestConfig latestConfig = new LatestConfig(Arrays.asList("field-A", "field-B"), "sort"); - TransformConfig config = - new TransformConfig( - transformId, sourceConfig, randomDestConfig(), null, syncConfig, null, null, latestConfig, null, null, null, null); + TransformConfig config = new TransformConfig( + transformId, + sourceConfig, + randomDestConfig(), + null, + syncConfig, + null, + null, + latestConfig, + null, + null, + null, + null, + null + ); MockTransformAuditor auditor = MockTransformAuditor.createMockAuditor(); auditor.addExpectation( From 32af5fe22a77da9afa9dca58800e4877ca20f005 Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Wed, 3 Feb 2021 17:46:46 +0100 Subject: [PATCH 26/39] connect stats to measure deletes --- .../transforms/TransformIndexerStats.java | 15 ++++++++++ .../TransformInternalIndexConstants.java | 5 ++-- .../persistence/TransformInternalIndex.java | 7 +++++ .../transforms/TransformIndexer.java | 3 ++ .../transforms/TransformIndexerTests.java | 28 ++++++++++++++++++- .../80_transform_jobs_crud.yml | 8 +++--- 6 files changed, 59 insertions(+), 7 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformIndexerStats.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformIndexerStats.java index eb0cc2cdef7e1..bdc8ab34daa85 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformIndexerStats.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformIndexerStats.java @@ -97,6 +97,8 @@ public class TransformIndexerStats extends IndexerJobStats { private long deleteTime; private long numDeletedDocuments; + private long startDeleteTime; + private double expAvgCheckpointDurationMs; private double expAvgDocumentsIndexed; private double expAvgDocumentsProcessed; @@ -247,6 +249,19 @@ public double getExpAvgDocumentsProcessed() { return expAvgDocumentsProcessed; } + public void incrementNumDeletedDocuments(long n) { + assert(n >= 0); + numDeletedDocuments += n; + } + + public void markStartDelete() { + startDeleteTime = System.nanoTime(); + } + + public void markEndDelete() { + deleteTime += ((System.nanoTime() - startDeleteTime) / 1000000); + } + public void incrementCheckpointExponentialAverages(long checkpointDurationMs, long docsIndexed, long docsProcessed) { assert checkpointDurationMs >= 0; assert docsIndexed >= 0; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/persistence/TransformInternalIndexConstants.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/persistence/TransformInternalIndexConstants.java index 062e648d189b7..1f8472d159e56 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/persistence/TransformInternalIndexConstants.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/persistence/TransformInternalIndexConstants.java @@ -21,6 +21,7 @@ public final class TransformInternalIndexConstants { * - XPackRestTestConstants * - yaml tests under x-pack/qa/ * - upgrade tests under x-pack/qa/rolling-upgrade + * - TransformSurvivesUpgradeIT * * (pro-tip: grep for the constant) */ @@ -28,8 +29,8 @@ public final class TransformInternalIndexConstants { // internal index // version is not a rollover pattern, however padded because sort is string based - public static final Version INDEX_VERSION_LAST_CHANGED = Version.V_7_7_0; - public static final String INDEX_VERSION = "005"; + public static final Version INDEX_VERSION_LAST_CHANGED = Version.V_7_12_0; + public static final String INDEX_VERSION = "006"; public static final String INDEX_PATTERN = ".transform-internal-"; public static final String LATEST_INDEX_VERSIONED_NAME = INDEX_PATTERN + INDEX_VERSION; public static final String LATEST_INDEX_NAME = LATEST_INDEX_VERSIONED_NAME; diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/TransformInternalIndex.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/TransformInternalIndex.java index 1f322d5f630db..d0875d65faeb3 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/TransformInternalIndex.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/TransformInternalIndex.java @@ -64,6 +64,7 @@ public final class TransformInternalIndex { * version 4 (7.6): state::should_stop_at_checkpoint * checkpoint::checkpoint * version 5 (7.7): stats::processing_time_in_ms, stats::processing_total + * version 6 (7.12):stats::delete_time_in_ms, stats::documents_deleted */ // constants for mappings @@ -241,6 +242,9 @@ private static XContentBuilder addTransformStoredDocMappings(XContentBuilder bui .endObject() .startObject(TransformIndexerStats.NUM_OUTPUT_DOCUMENTS.getPreferredName()) .field(TYPE, LONG) + .endObject() + .startObject(TransformIndexerStats.NUM_DELETED_DOCUMENTS.getPreferredName()) + .field(TYPE, LONG) .endObject() .startObject(TransformIndexerStats.NUM_INVOCATIONS.getPreferredName()) .field(TYPE, LONG) @@ -254,6 +258,9 @@ private static XContentBuilder addTransformStoredDocMappings(XContentBuilder bui .startObject(TransformIndexerStats.PROCESSING_TIME_IN_MS.getPreferredName()) .field(TYPE, LONG) .endObject() + .startObject(TransformIndexerStats.DELETE_TIME_IN_MS.getPreferredName()) + .field(TYPE, LONG) + .endObject() .startObject(TransformIndexerStats.INDEX_TOTAL.getPreferredName()) .field(TYPE, LONG) .endObject() diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformIndexer.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformIndexer.java index 85d908d42a069..d6b10f75772fc 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformIndexer.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformIndexer.java @@ -459,10 +459,13 @@ private void executeRetentionPolicy(ActionListener listener) { deleteByQuery, deleteByQuery.getSearchRequest() ); + getStats().markStartDelete(); doDeleteByQuery(deleteByQuery, ActionListener.wrap(bulkByScrollResponse -> { logger.trace("[{}] dbq response: [{}]", getJobId(), bulkByScrollResponse); + getStats().markEndDelete(); if (bulkByScrollResponse.getDeleted() > 0) { + getStats().incrementNumDeletedDocuments(bulkByScrollResponse.getDeleted()); logger.debug("[{}] deleted [{}] documents as part of the retention policy.", getJobId(), bulkByScrollResponse.getDeleted()); } diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerTests.java index 31b269ff2f942..6f026d6fcdf23 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerTests.java @@ -60,6 +60,7 @@ import static org.elasticsearch.xpack.core.transform.transforms.DestConfigTests.randomDestConfig; import static org.elasticsearch.xpack.core.transform.transforms.SourceConfigTests.randomSourceConfig; import static org.elasticsearch.xpack.core.transform.transforms.pivot.PivotConfigTests.randomPivotConfig; +import static org.hamcrest.Matchers.greaterThan; import static org.mockito.Mockito.mock; public class TransformIndexerTests extends ESTestCase { @@ -153,10 +154,31 @@ void doGetInitialProgress(SearchRequest request, ActionListener @Override void doDeleteByQuery(DeleteByQueryRequest deleteByQueryRequest, ActionListener responseListener) { deleteByQueryCallCount++; + try { + // yes, I know, a sleep, how dare you, this is to test stats collection and this requires a resolution of a millisecond + Thread.sleep(1); + } catch (InterruptedException e) { + fail("unexpected exception during sleep: " + e); + } responseListener.onResponse( new BulkByScrollResponse( TimeValue.ZERO, - new BulkByScrollTask.Status(Collections.emptyList(), null), + new BulkByScrollTask.Status( + 0, + 0L, + 0L, + 0L, + /*deleted*/ 42L, + 0, + 0L, + 0L, + 0L, + 0L, + TimeValue.ZERO, + 0.0f, + null, + TimeValue.ZERO + ), Collections.emptyList(), Collections.emptyList(), false @@ -270,6 +292,8 @@ public void testRetentionPolicyExecution() throws Exception { // delete by query has been executed assertEquals(1, indexer.getDeleteByQueryCallCount()); + assertEquals(42L, indexer.getStats().getNumDeletedDocuments()); + assertThat(indexer.getStats().getDeleteTime(), greaterThan(0L)); } // test without retention @@ -311,6 +335,8 @@ public void testRetentionPolicyExecution() throws Exception { // delete by query has _not_ been executed assertEquals(0, indexer.getDeleteByQueryCallCount()); + assertEquals(0L, indexer.getStats().getNumDeletedDocuments()); + assertEquals(0L, indexer.getStats().getDeleteTime()); } } diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/80_transform_jobs_crud.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/80_transform_jobs_crud.yml index 9d2f27565d2ad..0993d3c9f3bb9 100644 --- a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/80_transform_jobs_crud.yml +++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/80_transform_jobs_crud.yml @@ -298,11 +298,11 @@ setup: - do: warnings: - - "this request accesses system indices: [.transform-internal-005], but in a future major version, direct access to system indices will be prevented by default" + - "this request accesses system indices: [.transform-internal-006], but in a future major version, direct access to system indices will be prevented by default" indices.get_mapping: - index: .transform-internal-005 - - match: { \.transform-internal-005.mappings.dynamic: "false" } - - match: { \.transform-internal-005.mappings.properties.id.type: "keyword" } + index: .transform-internal-006 + - match: { \.transform-internal-006.mappings.dynamic: "false" } + - match: { \.transform-internal-006.mappings.properties.id.type: "keyword" } - do: indices.get_mapping: index: .transform-notifications-000002 From 01b7769eecb2a979739b1f5bcf4b539355204fae Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Wed, 3 Feb 2021 18:15:43 +0100 Subject: [PATCH 27/39] spotless --- .../transforms/TransformIndexerStats.java | 1 - .../org/elasticsearch/client/TransformIT.java | 159 ++++++++++-------- .../transforms/TransformIndexerStats.java | 2 +- .../transforms/SourceConfigTests.java | 34 ++-- .../TransformInfoTransportAction.java | 5 +- .../TransformIndexerFailureHandlingTests.java | 40 +++-- 6 files changed, 131 insertions(+), 110 deletions(-) diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/TransformIndexerStats.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/TransformIndexerStats.java index 553515e635353..b9477c5945dc1 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/TransformIndexerStats.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/TransformIndexerStats.java @@ -268,7 +268,6 @@ public long getDeleteTime() { return deleteTime; } - /** * Returns the total number of indexing requests that have been processed * (Note: this is not the number of _documents_ that have been indexed) diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/TransformIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/TransformIT.java index 855130ae63980..37316a543cd17 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/TransformIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/TransformIT.java @@ -82,18 +82,18 @@ private void createIndex(String indexName) throws IOException { XContentBuilder builder = jsonBuilder(); builder.startObject() - .startObject("properties") - .startObject("timestamp") - .field("type", "date") - .endObject() - .startObject("user_id") - .field("type", "keyword") - .endObject() - .startObject("stars") - .field("type", "integer") - .endObject() - .endObject() - .endObject(); + .startObject("properties") + .startObject("timestamp") + .field("type", "date") + .endObject() + .startObject("user_id") + .field("type", "keyword") + .endObject() + .startObject("stars") + .field("type", "integer") + .endObject() + .endObject() + .endObject(); CreateIndexRequest request = new CreateIndexRequest(indexName); request.mapping(builder); @@ -180,18 +180,17 @@ public void testCreateDelete() throws IOException { TransformConfig transform = validDataFrameTransformConfig(id, sourceIndex, "pivot-dest"); TransformClient client = highLevelClient().transform(); - AcknowledgedResponse ack = execute(new PutTransformRequest(transform), client::putTransform, - client::putTransformAsync); + AcknowledgedResponse ack = execute(new PutTransformRequest(transform), client::putTransform, client::putTransformAsync); assertTrue(ack.isAcknowledged()); - ack = execute(new DeleteTransformRequest(transform.getId()), client::deleteTransform, - client::deleteTransformAsync); + ack = execute(new DeleteTransformRequest(transform.getId()), client::deleteTransform, client::deleteTransformAsync); assertTrue(ack.isAcknowledged()); // The second delete should fail - ElasticsearchStatusException deleteError = expectThrows(ElasticsearchStatusException.class, - () -> execute(new DeleteTransformRequest(transform.getId()), client::deleteTransform, - client::deleteTransformAsync)); + ElasticsearchStatusException deleteError = expectThrows( + ElasticsearchStatusException.class, + () -> execute(new DeleteTransformRequest(transform.getId()), client::deleteTransform, client::deleteTransformAsync) + ); assertThat(deleteError.getMessage(), containsString("Transform with id [test-crud] could not be found")); } @@ -200,25 +199,27 @@ public void testUpdate() throws IOException { createIndex(sourceIndex); String id = "test-update"; - TransformConfig transform = validDataFrameTransformConfigBuilder(id, sourceIndex, "pivot-dest") - .setSyncConfig(new TimeSyncConfig("timefield", TimeValue.timeValueSeconds(60))) - .build(); + TransformConfig transform = validDataFrameTransformConfigBuilder(id, sourceIndex, "pivot-dest").setSyncConfig( + new TimeSyncConfig("timefield", TimeValue.timeValueSeconds(60)) + ).build(); TransformClient client = highLevelClient().transform(); - AcknowledgedResponse ack = execute(new PutTransformRequest(transform), client::putTransform, - client::putTransformAsync); + AcknowledgedResponse ack = execute(new PutTransformRequest(transform), client::putTransform, client::putTransformAsync); assertTrue(ack.isAcknowledged()); String updatedDescription = "my new description"; TransformConfigUpdate update = TransformConfigUpdate.builder().setDescription(updatedDescription).build(); UpdateTransformResponse response = execute( - new UpdateTransformRequest(update, id), client::updateTransform, - client::updateTransformAsync); + new UpdateTransformRequest(update, id), + client::updateTransform, + client::updateTransformAsync + ); assertThat(response.getTransformConfiguration().getDescription(), equalTo(updatedDescription)); - ElasticsearchStatusException updateError = expectThrows(ElasticsearchStatusException.class, - () -> execute(new UpdateTransformRequest(update, "missing-transform"), client::updateTransform, - client::updateTransformAsync)); + ElasticsearchStatusException updateError = expectThrows( + ElasticsearchStatusException.class, + () -> execute(new UpdateTransformRequest(update, "missing-transform"), client::updateTransform, client::updateTransformAsync) + ); assertThat(updateError.getMessage(), containsString("Transform with id [missing-transform] could not be found")); } @@ -233,8 +234,7 @@ public void testCreateDeleteWithDefer() throws IOException { AcknowledgedResponse ack = execute(request, client::putTransform, client::putTransformAsync); assertTrue(ack.isAcknowledged()); - ack = execute(new DeleteTransformRequest(transform.getId()), client::deleteTransform, - client::deleteTransformAsync); + ack = execute(new DeleteTransformRequest(transform.getId()), client::deleteTransform, client::deleteTransformAsync); assertTrue(ack.isAcknowledged()); } @@ -249,8 +249,7 @@ public void testGetTransform() throws IOException { putTransform(transform); GetTransformRequest getRequest = new GetTransformRequest(id); - GetTransformResponse getResponse = execute(getRequest, client::getTransform, - client::getTransformAsync); + GetTransformResponse getResponse = execute(getRequest, client::getTransform, client::getTransformAsync); assertNull(getResponse.getInvalidTransforms()); assertThat(getResponse.getTransformConfigurations(), hasSize(1)); assertEquals(transform.getId(), getResponse.getTransformConfigurations().get(0).getId()); @@ -269,21 +268,18 @@ public void testGetAllAndPageTransforms() throws IOException { putTransform(transform); GetTransformRequest getRequest = new GetTransformRequest("_all"); - GetTransformResponse getResponse = execute(getRequest, client::getTransform, - client::getTransformAsync); + GetTransformResponse getResponse = execute(getRequest, client::getTransform, client::getTransformAsync); assertNull(getResponse.getInvalidTransforms()); assertThat(getResponse.getTransformConfigurations(), hasSize(2)); assertEquals(transform.getId(), getResponse.getTransformConfigurations().get(1).getId()); - getRequest.setPageParams(new PageParams(0,1)); - getResponse = execute(getRequest, client::getTransform, - client::getTransformAsync); + getRequest.setPageParams(new PageParams(0, 1)); + getResponse = execute(getRequest, client::getTransform, client::getTransformAsync); assertNull(getResponse.getInvalidTransforms()); assertThat(getResponse.getTransformConfigurations(), hasSize(1)); GetTransformRequest getMulitple = new GetTransformRequest("test-get-all-1", "test-get-all-2"); - getResponse = execute(getMulitple, client::getTransform, - client::getTransformAsync); + getResponse = execute(getMulitple, client::getTransform, client::getTransformAsync); assertNull(getResponse.getInvalidTransforms()); assertThat(getResponse.getTransformConfigurations(), hasSize(2)); } @@ -291,9 +287,10 @@ public void testGetAllAndPageTransforms() throws IOException { public void testGetMissingTransform() { TransformClient client = highLevelClient().transform(); - ElasticsearchStatusException missingError = expectThrows(ElasticsearchStatusException.class, - () -> execute(new GetTransformRequest("unknown"), client::getTransform, - client::getTransformAsync)); + ElasticsearchStatusException missingError = expectThrows( + ElasticsearchStatusException.class, + () -> execute(new GetTransformRequest("unknown"), client::getTransform, client::getTransformAsync) + ); assertThat(missingError.status(), equalTo(RestStatus.NOT_FOUND)); } @@ -308,31 +305,33 @@ public void testStartStop() throws IOException { putTransform(transform); StartTransformRequest startRequest = new StartTransformRequest(id); - StartTransformResponse startResponse = - execute(startRequest, client::startTransform, client::startTransformAsync); + StartTransformResponse startResponse = execute(startRequest, client::startTransform, client::startTransformAsync); assertTrue(startResponse.isAcknowledged()); assertThat(startResponse.getNodeFailures(), empty()); assertThat(startResponse.getTaskFailures(), empty()); - GetTransformStatsResponse statsResponse = execute(new GetTransformStatsRequest(id), - client::getTransformStats, client::getTransformStatsAsync); + GetTransformStatsResponse statsResponse = execute( + new GetTransformStatsRequest(id), + client::getTransformStats, + client::getTransformStatsAsync + ); assertThat(statsResponse.getTransformsStats(), hasSize(1)); TransformStats.State taskState = statsResponse.getTransformsStats().get(0).getState(); // Since we are non-continuous, the transform could auto-stop between being started earlier and us gathering the statistics - assertThat(taskState, oneOf(TransformStats.State.STARTED, TransformStats.State.INDEXING, - TransformStats.State.STOPPING, TransformStats.State.STOPPED)); + assertThat( + taskState, + oneOf(TransformStats.State.STARTED, TransformStats.State.INDEXING, TransformStats.State.STOPPING, TransformStats.State.STOPPED) + ); StopTransformRequest stopRequest = new StopTransformRequest(id, Boolean.TRUE, null, false); - StopTransformResponse stopResponse = - execute(stopRequest, client::stopTransform, client::stopTransformAsync); + StopTransformResponse stopResponse = execute(stopRequest, client::stopTransform, client::stopTransformAsync); assertTrue(stopResponse.isAcknowledged()); assertThat(stopResponse.getNodeFailures(), empty()); assertThat(stopResponse.getTaskFailures(), empty()); // Calling stop with wait_for_completion assures that we will be in the `STOPPED` state for the transform task - statsResponse = execute(new GetTransformStatsRequest(id), - client::getTransformStats, client::getTransformStatsAsync); + statsResponse = execute(new GetTransformStatsRequest(id), client::getTransformStats, client::getTransformStatsAsync); taskState = statsResponse.getTransformsStats().get(0).getState(); assertThat(taskState, is(TransformStats.State.STOPPED)); } @@ -346,9 +345,11 @@ public void testPreview() throws IOException { TransformConfig transform = validDataFrameTransformConfig("test-preview", sourceIndex, null); TransformClient client = highLevelClient().transform(); - PreviewTransformResponse preview = execute(new PreviewTransformRequest(transform), - client::previewTransform, - client::previewTransformAsync); + PreviewTransformResponse preview = execute( + new PreviewTransformRequest(transform), + client::previewTransform, + client::previewTransformAsync + ); List> docs = preview.getDocs(); assertThat(docs, hasSize(2)); @@ -362,7 +363,7 @@ public void testPreview() throws IOException { Map mappings = preview.getMappings(); assertThat(mappings, hasKey("properties")); - Map fields = (Map)mappings.get("properties"); + Map fields = (Map) mappings.get("properties"); assertThat(fields.get("reviewer"), equalTo(Map.of("type", "keyword"))); assertThat(fields.get("avg_rating"), equalTo(Map.of("type", "double"))); } @@ -372,8 +373,7 @@ private TransformConfig validDataFrameTransformConfig(String id, String source, } private TransformConfig.Builder validDataFrameTransformConfigBuilder(String id, String source, String destination) { - GroupConfig groupConfig = GroupConfig.builder().groupBy("reviewer", - TermsGroupSource.builder().setField("user_id").build()).build(); + GroupConfig groupConfig = GroupConfig.builder().groupBy("reviewer", TermsGroupSource.builder().setField("user_id").build()).build(); AggregatorFactories.Builder aggBuilder = new AggregatorFactories.Builder(); aggBuilder.addAggregator(AggregationBuilders.avg("avg_rating").field("stars")); PivotConfig pivotConfig = PivotConfig.builder().setGroups(groupConfig).setAggregations(aggBuilder).build(); @@ -394,8 +394,7 @@ public void testGetStats() throws Exception { createIndex(sourceIndex); indexData(sourceIndex); - GroupConfig groupConfig = GroupConfig.builder().groupBy("reviewer", - TermsGroupSource.builder().setField("user_id").build()).build(); + GroupConfig groupConfig = GroupConfig.builder().groupBy("reviewer", TermsGroupSource.builder().setField("user_id").build()).build(); AggregatorFactories.Builder aggBuilder = new AggregatorFactories.Builder(); aggBuilder.addAggregator(AggregationBuilders.avg("avg_rating").field("stars")); PivotConfig pivotConfig = PivotConfig.builder().setGroups(groupConfig).setAggregations(aggBuilder).build(); @@ -412,8 +411,11 @@ public void testGetStats() throws Exception { TransformClient client = highLevelClient().transform(); putTransform(transform); - GetTransformStatsResponse statsResponse = execute(new GetTransformStatsRequest(id), - client::getTransformStats, client::getTransformStatsAsync); + GetTransformStatsResponse statsResponse = execute( + new GetTransformStatsRequest(id), + client::getTransformStats, + client::getTransformStatsAsync + ); assertEquals(1, statsResponse.getTransformsStats().size()); TransformStats stats = statsResponse.getTransformsStats().get(0); @@ -436,31 +438,42 @@ public void testGetStats() throws Exception { 0L, 0.0, 0.0, - 0.0); + 0.0 + ); assertEquals(zeroIndexerStats, stats.getIndexerStats()); // start the transform - StartTransformResponse startTransformResponse = execute(new StartTransformRequest(id), + StartTransformResponse startTransformResponse = execute( + new StartTransformRequest(id), client::startTransform, - client::startTransformAsync); + client::startTransformAsync + ); assertThat(startTransformResponse.isAcknowledged(), is(true)); assertBusy(() -> { - GetTransformStatsResponse response = execute(new GetTransformStatsRequest(id), - client::getTransformStats, client::getTransformStatsAsync); + GetTransformStatsResponse response = execute( + new GetTransformStatsRequest(id), + client::getTransformStats, + client::getTransformStatsAsync + ); TransformStats stateAndStats = response.getTransformsStats().get(0); assertNotEquals(zeroIndexerStats, stateAndStats.getIndexerStats()); - assertThat(stateAndStats.getState(), oneOf(TransformStats.State.STARTED, TransformStats.State.INDEXING, - TransformStats.State.STOPPING, TransformStats.State.STOPPED)); + assertThat( + stateAndStats.getState(), + oneOf( + TransformStats.State.STARTED, + TransformStats.State.INDEXING, + TransformStats.State.STOPPING, + TransformStats.State.STOPPED + ) + ); assertThat(stateAndStats.getReason(), is(nullValue())); }); } void putTransform(TransformConfig config) throws IOException { TransformClient client = highLevelClient().transform(); - AcknowledgedResponse ack = execute(new PutTransformRequest(config), client::putTransform, - client::putTransformAsync); + AcknowledgedResponse ack = execute(new PutTransformRequest(config), client::putTransform, client::putTransformAsync); assertTrue(ack.isAcknowledged()); transformsToClean.add(config.getId()); } } - diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformIndexerStats.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformIndexerStats.java index bdc8ab34daa85..2151ebb682aff 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformIndexerStats.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformIndexerStats.java @@ -250,7 +250,7 @@ public double getExpAvgDocumentsProcessed() { } public void incrementNumDeletedDocuments(long n) { - assert(n >= 0); + assert (n >= 0); numDeletedDocuments += n; } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/SourceConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/SourceConfigTests.java index cca201acfb2dc..9cecfbf789ab2 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/SourceConfigTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/SourceConfigTests.java @@ -83,32 +83,34 @@ protected Reader instanceReader() { } public void testGetRuntimeMappings_EmptyRuntimeMappings() { - SourceConfig sourceConfig = - new SourceConfig( - generateRandomStringArray(10, 10, false, false), - QueryConfigTests.randomQueryConfig(), - emptyMap()); + SourceConfig sourceConfig = new SourceConfig( + generateRandomStringArray(10, 10, false, false), + QueryConfigTests.randomQueryConfig(), + emptyMap() + ); assertThat(sourceConfig.getRuntimeMappings(), is(anEmptyMap())); assertThat(sourceConfig.getScriptBasedRuntimeMappings(), is(anEmptyMap())); } public void testGetRuntimeMappings_NonEmptyRuntimeMappings() { - Map runtimeMappings = - new HashMap<>() {{ + Map runtimeMappings = new HashMap<>() { + { put("field-A", singletonMap("type", "keyword")); put("field-B", singletonMap("script", "some script")); put("field-C", singletonMap("script", "some other script")); - }}; - Map scriptBasedRuntimeMappings = - new HashMap<>() {{ + } + }; + Map scriptBasedRuntimeMappings = new HashMap<>() { + { put("field-B", singletonMap("script", "some script")); put("field-C", singletonMap("script", "some other script")); - }}; - SourceConfig sourceConfig = - new SourceConfig( - generateRandomStringArray(10, 10, false, false), - QueryConfigTests.randomQueryConfig(), - runtimeMappings); + } + }; + SourceConfig sourceConfig = new SourceConfig( + generateRandomStringArray(10, 10, false, false), + QueryConfigTests.randomQueryConfig(), + runtimeMappings + ); assertThat(sourceConfig.getRuntimeMappings(), is(equalTo(runtimeMappings))); assertThat(sourceConfig.getScriptBasedRuntimeMappings(), is(equalTo(scriptBasedRuntimeMappings))); } diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/TransformInfoTransportAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/TransformInfoTransportAction.java index 48ca9148497a4..706f9256a8e3f 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/TransformInfoTransportAction.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/TransformInfoTransportAction.java @@ -59,10 +59,7 @@ public class TransformInfoTransportAction extends XPackInfoFeatureTransportActio TransformIndexerStats.EXPONENTIAL_AVG_DOCUMENTS_PROCESSED.getPreferredName(), }; @Inject - public TransformInfoTransportAction( - TransportService transportService, - ActionFilters actionFilters - ) { + public TransformInfoTransportAction(TransportService transportService, ActionFilters actionFilters) { super(XPackInfoFeatureAction.TRANSFORM.name(), transportService, actionFilters); } diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerFailureHandlingTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerFailureHandlingTests.java index b74dd7d7b7f8b..c5f5fcf0b9778 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerFailureHandlingTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerFailureHandlingTests.java @@ -503,15 +503,17 @@ public void testScriptError() throws Exception { public void testInitializeFunction_WithNoWarnings() { String transformId = randomAlphaOfLength(10); - SourceConfig sourceConfig = - new SourceConfig( - generateRandomStringArray(10, 10, false, false), - QueryConfigTests.randomQueryConfig(), - new HashMap<>() {{ + SourceConfig sourceConfig = new SourceConfig( + generateRandomStringArray(10, 10, false, false), + QueryConfigTests.randomQueryConfig(), + new HashMap<>() { + { put("field-A", singletonMap("script", "some script")); put("field-B", emptyMap()); put("field-C", singletonMap("script", "some script")); - }}); + } + } + ); SyncConfig syncConfig = new TimeSyncConfig("field", null); LatestConfig latestConfig = new LatestConfig(Arrays.asList("field-A", "field-B"), "sort"); TransformConfig config = new TransformConfig( @@ -536,7 +538,9 @@ public void testInitializeFunction_WithNoWarnings() { "warn when all the group-by fields are script-based runtime fields", Level.WARNING, transformId, - "all the group-by fields are script-based runtime fields")); + "all the group-by fields are script-based runtime fields" + ) + ); TransformContext.Listener contextListener = mock(TransformContext.Listener.class); TransformContext context = new TransformContext(TransformTaskState.STARTED, "", 0, contextListener); createMockIndexer(config, null, null, null, null, threadPool, ThreadPool.Names.GENERIC, auditor, context); @@ -545,16 +549,18 @@ public void testInitializeFunction_WithNoWarnings() { public void testInitializeFunction_WithWarnings() { String transformId = randomAlphaOfLength(10); - SourceConfig sourceConfig = - new SourceConfig( - generateRandomStringArray(10, 10, false, false), - QueryConfigTests.randomQueryConfig(), - new HashMap<>() {{ + SourceConfig sourceConfig = new SourceConfig( + generateRandomStringArray(10, 10, false, false), + QueryConfigTests.randomQueryConfig(), + new HashMap<>() { + { put("field-A", singletonMap("script", "some script")); put("field-B", singletonMap("script", "some script")); put("field-C", singletonMap("script", "some script")); put("field-t", singletonMap("script", "some script")); - }}); + } + } + ); SyncConfig syncConfig = new TimeSyncConfig("field-t", null); LatestConfig latestConfig = new LatestConfig(Arrays.asList("field-A", "field-B"), "sort"); TransformConfig config = new TransformConfig( @@ -579,13 +585,17 @@ public void testInitializeFunction_WithWarnings() { "warn when all the group-by fields are script-based runtime fields", Level.WARNING, transformId, - "all the group-by fields are script-based runtime fields")); + "all the group-by fields are script-based runtime fields" + ) + ); auditor.addExpectation( new MockTransformAuditor.SeenAuditExpectation( "warn when the sync time field is a script-based runtime field", Level.WARNING, transformId, - "sync time field is a script-based runtime field")); + "sync time field is a script-based runtime field" + ) + ); TransformContext.Listener contextListener = mock(TransformContext.Listener.class); TransformContext context = new TransformContext(TransformTaskState.STARTED, "", 0, contextListener); createMockIndexer(config, null, null, null, null, threadPool, ThreadPool.Names.GENERIC, auditor, context); From ed3f242b984b2473d4f76184281b3434465afb58 Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Thu, 4 Feb 2021 15:34:05 +0100 Subject: [PATCH 28/39] remove unnecessary new line --- .../client/transform/transforms/TransformIndexerStats.java | 1 - 1 file changed, 1 deletion(-) diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/TransformIndexerStats.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/TransformIndexerStats.java index b9477c5945dc1..f4952a69fafe9 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/TransformIndexerStats.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/TransformIndexerStats.java @@ -221,7 +221,6 @@ public long getOutputDocuments() { /** * Number of documents deleted */ - public long getDocumentsDeleted() { return documentsDeleted; } From c653b76658350582c403c56a3c5005c727ec0540 Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Fri, 5 Feb 2021 09:56:32 +0100 Subject: [PATCH 29/39] use declareNamedObject --- .../transform/transforms/TransformConfig.java | 26 +++++-------------- .../transform/transforms/TransformConfig.java | 25 +++++------------- 2 files changed, 12 insertions(+), 39 deletions(-) diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/TransformConfig.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/TransformConfig.java index 44ce44696705a..8e468c07e2b27 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/TransformConfig.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/TransformConfig.java @@ -21,7 +21,6 @@ import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.common.xcontent.XContentParserUtils; import java.io.IOException; import java.time.Instant; @@ -102,12 +101,16 @@ public class TransformConfig implements ToXContentObject { FREQUENCY, ObjectParser.ValueType.STRING ); - PARSER.declareObject(optionalConstructorArg(), (p, c) -> parseSyncConfig(p), SYNC); + PARSER.declareNamedObject(optionalConstructorArg(), (p, c, n) -> p.namedObject(SyncConfig.class, n, c), SYNC); PARSER.declareObject(optionalConstructorArg(), (p, c) -> PivotConfig.fromXContent(p), PIVOT_TRANSFORM); PARSER.declareObject(optionalConstructorArg(), (p, c) -> LatestConfig.fromXContent(p), LATEST_TRANSFORM); PARSER.declareString(optionalConstructorArg(), DESCRIPTION); PARSER.declareObject(optionalConstructorArg(), (p, c) -> SettingsConfig.fromXContent(p), SETTINGS); - PARSER.declareObject(optionalConstructorArg(), (p, c) -> parseRentionPolicyConfig(p), RETENTION_POLICY); + PARSER.declareNamedObject( + optionalConstructorArg(), + (p, c, n) -> p.namedObject(RetentionPolicyConfig.class, n, c), + RETENTION_POLICY + ); PARSER.declareField( optionalConstructorArg(), p -> TimeUtil.parseTimeFieldToInstant(p, CREATE_TIME.getPreferredName()), @@ -115,23 +118,6 @@ public class TransformConfig implements ToXContentObject { ObjectParser.ValueType.VALUE ); PARSER.declareString(optionalConstructorArg(), VERSION); - PARSER.declareObject(optionalConstructorArg(), (p, c) -> parseRentionPolicyConfig(p), RETENTION_POLICY); - } - - private static SyncConfig parseSyncConfig(XContentParser parser) throws IOException { - XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser); - XContentParserUtils.ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.nextToken(), parser); - SyncConfig syncConfig = parser.namedObject(SyncConfig.class, parser.currentName(), true); - XContentParserUtils.ensureExpectedToken(XContentParser.Token.END_OBJECT, parser.nextToken(), parser); - return syncConfig; - } - - private static RetentionPolicyConfig parseRentionPolicyConfig(XContentParser parser) throws IOException { - XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser); - XContentParserUtils.ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.nextToken(), parser); - RetentionPolicyConfig retentionPolicyConfig = parser.namedObject(RetentionPolicyConfig.class, parser.currentName(), true); - XContentParserUtils.ensureExpectedToken(XContentParser.Token.END_OBJECT, parser.nextToken(), parser); - return retentionPolicyConfig; } public static TransformConfig fromXContent(final XContentParser parser) { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java index fb999be9a2553..2187d2537864a 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java @@ -22,7 +22,6 @@ import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.common.xcontent.XContentParserUtils; import org.elasticsearch.xpack.core.common.time.TimeUtils; import org.elasticsearch.xpack.core.transform.TransformField; import org.elasticsearch.xpack.core.transform.TransformMessages; @@ -142,14 +141,18 @@ private static ConstructingObjectParser createParser(bo parser.declareObject(constructorArg(), (p, c) -> SourceConfig.fromXContent(p, lenient), TransformField.SOURCE); parser.declareObject(constructorArg(), (p, c) -> DestConfig.fromXContent(p, lenient), TransformField.DESTINATION); parser.declareString(optionalConstructorArg(), TransformField.FREQUENCY); - parser.declareObject(optionalConstructorArg(), (p, c) -> parseSyncConfig(p, lenient), TransformField.SYNC); + parser.declareNamedObject(optionalConstructorArg(), (p, c, n) -> p.namedObject(SyncConfig.class, n, c), TransformField.SYNC); parser.declareString(optionalConstructorArg(), TransformField.INDEX_DOC_TYPE); parser.declareObject(optionalConstructorArg(), (p, c) -> p.mapStrings(), HEADERS); parser.declareObject(optionalConstructorArg(), (p, c) -> PivotConfig.fromXContent(p, lenient), PIVOT_TRANSFORM); parser.declareObject(optionalConstructorArg(), (p, c) -> LatestConfig.fromXContent(p, lenient), LATEST_TRANSFORM); parser.declareString(optionalConstructorArg(), TransformField.DESCRIPTION); parser.declareObject(optionalConstructorArg(), (p, c) -> SettingsConfig.fromXContent(p, lenient), TransformField.SETTINGS); - parser.declareObject(optionalConstructorArg(), (p, c) -> parseRetentionPolicyConfig(p), TransformField.RETENTION_POLICY); + parser.declareNamedObject( + optionalConstructorArg(), + (p, c, n) -> p.namedObject(RetentionPolicyConfig.class, n, c), + TransformField.RETENTION_POLICY + ); parser.declareField( optionalConstructorArg(), p -> TimeUtils.parseTimeFieldToInstant(p, TransformField.CREATE_TIME.getPreferredName()), @@ -160,22 +163,6 @@ private static ConstructingObjectParser createParser(bo return parser; } - private static SyncConfig parseSyncConfig(XContentParser parser, boolean ignoreUnknownFields) throws IOException { - XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser); - XContentParserUtils.ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.nextToken(), parser); - SyncConfig syncConfig = parser.namedObject(SyncConfig.class, parser.currentName(), ignoreUnknownFields); - XContentParserUtils.ensureExpectedToken(XContentParser.Token.END_OBJECT, parser.nextToken(), parser); - return syncConfig; - } - - private static RetentionPolicyConfig parseRetentionPolicyConfig(XContentParser parser) throws IOException { - XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser); - XContentParserUtils.ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.nextToken(), parser); - RetentionPolicyConfig retentionPolicyConfig = parser.namedObject(RetentionPolicyConfig.class, parser.currentName(), true); - XContentParserUtils.ensureExpectedToken(XContentParser.Token.END_OBJECT, parser.nextToken(), parser); - return retentionPolicyConfig; - } - public static String documentId(String transformId) { return NAME + "-" + transformId; } From 9d6389a9a20c0b997f122f6f875d0be0c09025c2 Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Fri, 5 Feb 2021 10:41:55 +0100 Subject: [PATCH 30/39] add delete by query paranoia --- .../transforms/TransformIndexer.java | 23 +++++++++++++++---- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformIndexer.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformIndexer.java index d6b10f75772fc..35925ede48881 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformIndexer.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformIndexer.java @@ -454,10 +454,12 @@ private void executeRetentionPolicy(ActionListener listener) { } logger.debug( - "[{}] Run delete based on retention policy using dbq [{}] with query: [{}]", - getJobId(), - deleteByQuery, - deleteByQuery.getSearchRequest() + new ParameterizedMessage( + "[{}] Run delete based on retention policy using dbq [{}] with query: [{}]", + getJobId(), + deleteByQuery, + deleteByQuery.getSearchRequest() + ) ); getStats().markStartDelete(); @@ -472,11 +474,22 @@ private void executeRetentionPolicy(ActionListener listener) { // this should not happen as part of checkpointing if (bulkByScrollResponse.getVersionConflicts() > 0) { logger.warn( - "[{}] found [{}] version conflicts when deleting documents as part of the retention policy.", + "[{}] found [{}] version conflicts when deleting documents as part of the retention policy. Retry at the next checkpoint.", getJobId(), bulkByScrollResponse.getDeleted() ); } + // paranoia: we are not expecting dbq to fail for other reasons + if (bulkByScrollResponse.getBulkFailures().size() > 0 || bulkByScrollResponse.getSearchFailures().size() > 0) { + assert false : "delete by query failed unexpectedly" + bulkByScrollResponse; + logger.warn( + new ParameterizedMessage( + "[{}] found failures when deleting documents as part of the retention policy. Retry at the next checkpoint.", + getJobId(), + bulkByScrollResponse + ) + ); + } finalizeCheckpoint(listener); }, listener::onFailure)); From 94878ab25b89749fabcde22137747316783b39ac Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Fri, 5 Feb 2021 10:43:37 +0100 Subject: [PATCH 31/39] wrap logger.trace as parameterized message --- .../xpack/transform/transforms/TransformIndexer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformIndexer.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformIndexer.java index 35925ede48881..7518765aeeaa6 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformIndexer.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformIndexer.java @@ -464,7 +464,7 @@ private void executeRetentionPolicy(ActionListener listener) { getStats().markStartDelete(); doDeleteByQuery(deleteByQuery, ActionListener.wrap(bulkByScrollResponse -> { - logger.trace("[{}] dbq response: [{}]", getJobId(), bulkByScrollResponse); + logger.trace(new ParameterizedMessage("[{}] dbq response: [{}]", getJobId(), bulkByScrollResponse)); getStats().markEndDelete(); if (bulkByScrollResponse.getDeleted() > 0) { getStats().incrementNumDeletedDocuments(bulkByScrollResponse.getDeleted()); From 9a4bdf63c537a04ef732304c7b99bb229ee34483 Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Fri, 5 Feb 2021 17:44:23 +0100 Subject: [PATCH 32/39] apply review comments --- .../transforms/TransformConfigTests.java | 11 +++++++++-- .../transforms/latest/LatestConfigTests.java | 7 ++++++- ...ionPolicyToDeleteByQueryRequestConverter.java | 16 ++++++++-------- .../transform/transforms/TransformContext.java | 3 +-- .../transform/transforms/TransformIndexer.java | 12 ++++++++---- 5 files changed, 32 insertions(+), 17 deletions(-) diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigTests.java index a62897c1ae2f8..2caf21fc53a8c 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigTests.java @@ -231,8 +231,15 @@ public void testDefaultMatchAll() throws IOException { } public void testConstructor_NoFunctionProvided() throws IOException { - String json = "{" + " \"source\": {\"index\": \"src\"}," + " \"dest\": {\"index\": \"dest\"}" + "}"; - + // tag::NO_CODE_FORMAT + String json = "{" + + " \"source\": {" + + " \"index\": \"src\"" + + " }," + + " \"dest\": {" + + " \"index\": \"dest\"" + + "} }"; + // end::NO_CODE_FORMAT // Should parse with lenient parser createTransformConfigFromString(json, "dummy", true); // Should throw with strict parser diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/latest/LatestConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/latest/LatestConfigTests.java index 19a33964dc769..bf04cf86753c5 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/latest/LatestConfigTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/latest/LatestConfigTests.java @@ -49,7 +49,12 @@ protected Reader instanceReader() { } public void testValidate_ValidConfig() throws IOException { - String json = "{" + " \"unique_key\": [ \"event1\", \"event2\", \"event3\" ]," + " \"sort\": \"timestamp\"" + "}"; + // tag::NO_CODE_FORMAT + String json = "{" + + " \"unique_key\": [ \"event1\", \"event2\", \"event3\" ]," + + " \"sort\": \"timestamp\"" + + "}"; + // end::NO_CODE_FORMAT LatestConfig config = createLatestConfigFromString(json); assertThat(config.validate(null), is(nullValue())); diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/RetentionPolicyToDeleteByQueryRequestConverter.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/RetentionPolicyToDeleteByQueryRequestConverter.java index 5d262af0a0d65..2f6c5a4adc92f 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/RetentionPolicyToDeleteByQueryRequestConverter.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/RetentionPolicyToDeleteByQueryRequestConverter.java @@ -23,7 +23,7 @@ /** * Implementation of `retention_policy` configuration parameter. * - * All implementations of `rentention_policy` are converted to a {@link DeleteByQueryRequest}, which is than executed by the indexer. + * All implementations of `retention_policy` are converted to a {@link DeleteByQueryRequest}, which is then executed by the indexer. */ public final class RetentionPolicyToDeleteByQueryRequestConverter { @@ -36,14 +36,14 @@ public static class RetentionPolicyException extends ElasticsearchException { private RetentionPolicyToDeleteByQueryRequestConverter() {} /** - * Build a {@link DeleteByQueryRequest} from a retention policy. The DBQ should run _after_ a new checkpoint has finished. - * The given checkpoint should be the one that just finished indexing, however the DBQ executes before this checkpoint - * gets exposed. + * Build a {@link DeleteByQueryRequest} from a `retention policy`. The DBQ runs _after_ all data for a new checkpoint + * has been processed (composite agg + bulk indexing) and the index got refreshed. After the DBQ - with a final index refresh - + * the checkpoint is complete. * * @param retentionPolicyConfig The retention policy configuration * @param settingsConfig settings to set certain parameters * @param destConfig the destination config - * @param checkpoint The checkpoint that just finished + * @param nextCheckpoint The checkpoint that just finished * * @return a delete by query request according to the given configurations or null if no delete by query should be executed */ @@ -51,16 +51,16 @@ static DeleteByQueryRequest buildDeleteByQueryRequest( RetentionPolicyConfig retentionPolicyConfig, SettingsConfig settingsConfig, DestConfig destConfig, - TransformCheckpoint checkpoint + TransformCheckpoint nextCheckpoint ) { - if (checkpoint == null || checkpoint.isEmpty()) { + if (nextCheckpoint == null || nextCheckpoint.isEmpty()) { return null; } DeleteByQueryRequest request = new DeleteByQueryRequest(); if (retentionPolicyConfig instanceof TimeRetentionPolicyConfig) { - request.setQuery(getDeleteQueryFromTimeBasedRetentionPolicy((TimeRetentionPolicyConfig) retentionPolicyConfig, checkpoint)); + request.setQuery(getDeleteQueryFromTimeBasedRetentionPolicy((TimeRetentionPolicyConfig) retentionPolicyConfig, nextCheckpoint)); } else { throw new RetentionPolicyException("unsupported retention policy of type [{}]", retentionPolicyConfig.getWriteableName()); } diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformContext.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformContext.java index 5dc8ce143e9e1..aae24f2eb838c 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformContext.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformContext.java @@ -31,7 +31,7 @@ public interface Listener { private final AtomicInteger failureCount; private volatile Instant changesLastDetectedAt; private volatile Instant lastSearchTime; - private volatile boolean shouldStopAtCheckpoint; + private volatile boolean shouldStopAtCheckpoint = false; // the checkpoint of this transform, storing the checkpoint until data indexing from source to dest is _complete_ // Note: Each indexer run creates a new future checkpoint which becomes the current checkpoint only after the indexer run finished @@ -43,7 +43,6 @@ public interface Listener { this.currentCheckpoint = new AtomicLong(currentCheckpoint); this.taskListener = taskListener; this.failureCount = new AtomicInteger(0); - this.shouldStopAtCheckpoint = false; } TransformTaskState getTaskState() { diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformIndexer.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformIndexer.java index 7518765aeeaa6..04310bd3be622 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformIndexer.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformIndexer.java @@ -474,9 +474,12 @@ private void executeRetentionPolicy(ActionListener listener) { // this should not happen as part of checkpointing if (bulkByScrollResponse.getVersionConflicts() > 0) { logger.warn( - "[{}] found [{}] version conflicts when deleting documents as part of the retention policy. Retry at the next checkpoint.", - getJobId(), - bulkByScrollResponse.getDeleted() + new ParameterizedMessage( + "[{}] found [{}] version conflicts when deleting documents as part of the retention policy. " + + "Retry at the next checkpoint.", + getJobId(), + bulkByScrollResponse.getDeleted() + ) ); } // paranoia: we are not expecting dbq to fail for other reasons @@ -484,7 +487,8 @@ private void executeRetentionPolicy(ActionListener listener) { assert false : "delete by query failed unexpectedly" + bulkByScrollResponse; logger.warn( new ParameterizedMessage( - "[{}] found failures when deleting documents as part of the retention policy. Retry at the next checkpoint.", + "[{}] found failures when deleting documents as part of the retention policy. " + + "Retry at the next checkpoint. Response: [{}]", getJobId(), bulkByScrollResponse ) From dac1df18dd48caf3d6a530350ff66374754a8a3d Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Fri, 5 Feb 2021 17:50:58 +0100 Subject: [PATCH 33/39] remove unnecessary if clause --- .../transform/transforms/TransformIndexer.java | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformIndexer.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformIndexer.java index 04310bd3be622..cf81371e6ba91 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformIndexer.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformIndexer.java @@ -454,7 +454,7 @@ private void executeRetentionPolicy(ActionListener listener) { } logger.debug( - new ParameterizedMessage( + () -> new ParameterizedMessage( "[{}] Run delete based on retention policy using dbq [{}] with query: [{}]", getJobId(), deleteByQuery, @@ -464,17 +464,16 @@ private void executeRetentionPolicy(ActionListener listener) { getStats().markStartDelete(); doDeleteByQuery(deleteByQuery, ActionListener.wrap(bulkByScrollResponse -> { - logger.trace(new ParameterizedMessage("[{}] dbq response: [{}]", getJobId(), bulkByScrollResponse)); + logger.trace(() -> new ParameterizedMessage("[{}] dbq response: [{}]", getJobId(), bulkByScrollResponse)); + getStats().markEndDelete(); - if (bulkByScrollResponse.getDeleted() > 0) { - getStats().incrementNumDeletedDocuments(bulkByScrollResponse.getDeleted()); - logger.debug("[{}] deleted [{}] documents as part of the retention policy.", getJobId(), bulkByScrollResponse.getDeleted()); - } + getStats().incrementNumDeletedDocuments(bulkByScrollResponse.getDeleted()); + logger.debug("[{}] deleted [{}] documents as part of the retention policy.", getJobId(), bulkByScrollResponse.getDeleted()); // this should not happen as part of checkpointing if (bulkByScrollResponse.getVersionConflicts() > 0) { logger.warn( - new ParameterizedMessage( + () -> new ParameterizedMessage( "[{}] found [{}] version conflicts when deleting documents as part of the retention policy. " + "Retry at the next checkpoint.", getJobId(), @@ -486,7 +485,7 @@ private void executeRetentionPolicy(ActionListener listener) { if (bulkByScrollResponse.getBulkFailures().size() > 0 || bulkByScrollResponse.getSearchFailures().size() > 0) { assert false : "delete by query failed unexpectedly" + bulkByScrollResponse; logger.warn( - new ParameterizedMessage( + () -> new ParameterizedMessage( "[{}] found failures when deleting documents as part of the retention policy. " + "Retry at the next checkpoint. Response: [{}]", getJobId(), From e3980e9408b4b0bedfe99398a7d6e1e8e055a4c6 Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Mon, 8 Feb 2021 09:33:45 +0100 Subject: [PATCH 34/39] add tests for retention policy classes --- .../org/elasticsearch/client/RestHighLevelClientTests.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java index 1f3bc0dd66e3b..5ede2016c2e42 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java @@ -79,7 +79,9 @@ import org.elasticsearch.client.ml.inference.trainedmodel.ensemble.WeightedSum; import org.elasticsearch.client.ml.inference.trainedmodel.langident.LangIdentNeuralNetwork; import org.elasticsearch.client.ml.inference.trainedmodel.tree.Tree; +import org.elasticsearch.client.transform.transforms.RetentionPolicyConfig; import org.elasticsearch.client.transform.transforms.SyncConfig; +import org.elasticsearch.client.transform.transforms.TimeRetentionPolicyConfig; import org.elasticsearch.client.transform.transforms.TimeSyncConfig; import org.elasticsearch.common.CheckedFunction; import org.elasticsearch.common.bytes.BytesReference; @@ -745,6 +747,8 @@ public void testProvidedNamedXContents() { assertTrue(names.contains(ClassificationStats.NAME.getPreferredName())); assertEquals(Integer.valueOf(1), categories.get(SyncConfig.class)); assertTrue(names.contains(TimeSyncConfig.NAME)); + assertEquals(Integer.valueOf(1), categories.get(RetentionPolicyConfig.class)); + assertTrue(names.contains(TimeRetentionPolicyConfig.NAME)); assertEquals(Integer.valueOf(3), categories.get(org.elasticsearch.client.ml.dataframe.evaluation.Evaluation.class)); assertThat(names, hasItems(OutlierDetection.NAME, Classification.NAME, Regression.NAME)); assertEquals(Integer.valueOf(13), categories.get(org.elasticsearch.client.ml.dataframe.evaluation.EvaluationMetric.class)); From 79a0d0a77f6149d394703b51d56cb093ecedd5e8 Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Mon, 8 Feb 2021 09:48:24 +0100 Subject: [PATCH 35/39] limit refresh to the destination index --- .../xpack/transform/transforms/ClientTransformIndexer.java | 4 ++-- .../xpack/transform/transforms/TransformIndexer.java | 5 ++--- .../transforms/TransformIndexerFailureHandlingTests.java | 3 +-- .../transform/transforms/TransformIndexerStateTests.java | 3 +-- .../xpack/transform/transforms/TransformIndexerTests.java | 3 +-- 5 files changed, 7 insertions(+), 11 deletions(-) diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/ClientTransformIndexer.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/ClientTransformIndexer.java index df08506aa8194..8a527f64c33e5 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/ClientTransformIndexer.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/ClientTransformIndexer.java @@ -215,7 +215,7 @@ protected void doDeleteByQuery(DeleteByQueryRequest deleteByQueryRequest, Action } @Override - protected void refreshDestinationIndex(RefreshRequest refreshRequest, ActionListener responseListener) { + protected void refreshDestinationIndex(ActionListener responseListener) { // note: this gets executed _without_ the headers of the user as the user might not have the rights to call // _refresh for performance reasons. However this refresh is an internal detail of transform and this is only // called for the transform destination index @@ -223,7 +223,7 @@ protected void refreshDestinationIndex(RefreshRequest refreshRequest, ActionList client, ClientHelper.TRANSFORM_ORIGIN, RefreshAction.INSTANCE, - refreshRequest, + new RefreshRequest(transformConfig.getDestination().getIndex()), responseListener ); } diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformIndexer.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformIndexer.java index cf81371e6ba91..40c9253f36953 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformIndexer.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformIndexer.java @@ -13,7 +13,6 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; import org.elasticsearch.action.admin.indices.refresh.RefreshResponse; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.search.SearchRequest; @@ -165,7 +164,7 @@ public TransformIndexer( abstract void doDeleteByQuery(DeleteByQueryRequest deleteByQueryRequest, ActionListener responseListener); - abstract void refreshDestinationIndex(RefreshRequest refreshRequest, ActionListener responseListener); + abstract void refreshDestinationIndex(ActionListener responseListener); public int getPageSize() { return pageSize; @@ -424,7 +423,7 @@ protected void onFinish(ActionListener listener) { return; } - refreshDestinationIndex(new RefreshRequest(transformConfig.getDestination().getIndex()), ActionListener.wrap(response -> { + refreshDestinationIndex(ActionListener.wrap(response -> { if (response.getFailedShards() > 0) { logger.warn( "[{}] failed to refresh transform destination index, not all data might be available after checkpoint.", diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerFailureHandlingTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerFailureHandlingTests.java index c5f5fcf0b9778..b2c9b6ae66a9a 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerFailureHandlingTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerFailureHandlingTests.java @@ -9,7 +9,6 @@ import org.apache.lucene.search.TotalHits; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; import org.elasticsearch.action.admin.indices.refresh.RefreshResponse; import org.elasticsearch.action.bulk.BulkItemResponse; import org.elasticsearch.action.bulk.BulkRequest; @@ -268,7 +267,7 @@ void doDeleteByQuery(DeleteByQueryRequest deleteByQueryRequest, ActionListener responseListener) { + void refreshDestinationIndex(ActionListener responseListener) { responseListener.onResponse(new RefreshResponse(1, 1, 0, Collections.emptyList())); } } diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerStateTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerStateTests.java index 131e89e246056..e1d6cf594dbe5 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerStateTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerStateTests.java @@ -10,7 +10,6 @@ import org.apache.lucene.search.TotalHits; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.LatchedActionListener; -import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; import org.elasticsearch.action.admin.indices.refresh.RefreshResponse; import org.elasticsearch.action.bulk.BulkItemResponse; import org.elasticsearch.action.bulk.BulkRequest; @@ -179,7 +178,7 @@ void doDeleteByQuery(DeleteByQueryRequest deleteByQueryRequest, ActionListener responseListener) { + void refreshDestinationIndex(ActionListener responseListener) { responseListener.onResponse(new RefreshResponse(1, 1, 0, Collections.emptyList())); } diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerTests.java index 6f026d6fcdf23..433aaeabb3235 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerTests.java @@ -9,7 +9,6 @@ import org.apache.lucene.search.TotalHits; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; import org.elasticsearch.action.admin.indices.refresh.RefreshResponse; import org.elasticsearch.action.bulk.BulkItemResponse; import org.elasticsearch.action.bulk.BulkRequest; @@ -187,7 +186,7 @@ void doDeleteByQuery(DeleteByQueryRequest deleteByQueryRequest, ActionListener responseListener) { + void refreshDestinationIndex(ActionListener responseListener) { responseListener.onResponse(new RefreshResponse(1, 1, 0, Collections.emptyList())); } From b03b9e6d1c304a8a82e30ba40289708f6104a812 Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Mon, 8 Feb 2021 11:15:25 +0100 Subject: [PATCH 36/39] add new stats to cat and documentation --- docs/reference/cat/transforms.asciidoc | 6 ++++++ docs/reference/rest-api/common-parms.asciidoc | 9 +++++++++ .../transform/apis/get-transform-stats.asciidoc | 8 ++++++++ .../rest/action/RestCatTransformAction.java | 12 ++++++++++++ 4 files changed, 35 insertions(+) diff --git a/docs/reference/cat/transforms.asciidoc b/docs/reference/cat/transforms.asciidoc index 20752761ca13d..6d4f8c5129656 100644 --- a/docs/reference/cat/transforms.asciidoc +++ b/docs/reference/cat/transforms.asciidoc @@ -74,12 +74,18 @@ The progress of the next checkpoint that is currently in progress. `create_time`, `ct`, `createTime`::: The time the {transform} was created. +`delete_time`, `dtime`::: +include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=delete-time-ms] + `description`, `d`::: The description of the {transform}. `dest_index`, `di`, `destIndex`::: include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=dest-index] +`documents_deleted`, `docd`::: +include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=docs-deleted-transform] + `documents_indexed`, `doci`::: include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=docs-indexed] diff --git a/docs/reference/rest-api/common-parms.asciidoc b/docs/reference/rest-api/common-parms.asciidoc index 4f5096d1bc0c9..030f49288a9ad 100644 --- a/docs/reference/rest-api/common-parms.asciidoc +++ b/docs/reference/rest-api/common-parms.asciidoc @@ -142,6 +142,10 @@ Comma-separated list or wildcard expressions of fields to include in `fielddata` and `suggest` statistics. end::completion-fields[] +tag::delete-time-ms[] +The amount of time spent deleting, in milliseconds. +end::delete-time-ms[] + tag::default_operator[] `default_operator`:: (Optional, string) The default operator for query string query: AND or OR. @@ -195,6 +199,11 @@ Also, {es} creates extra deleted documents to internally track the recent history of operations on a shard. end::docs-deleted[] +tag::docs-deleted-transform[] +The number of documents that have been deleted from the destination index +for the {transform}. +end::docs-deleted-transform[] + tag::docs-indexed[] The number of documents that have been indexed into the destination index for the {transform}. diff --git a/docs/reference/transform/apis/get-transform-stats.asciidoc b/docs/reference/transform/apis/get-transform-stats.asciidoc index 96222a0cc6043..da49ce694e22d 100644 --- a/docs/reference/transform/apis/get-transform-stats.asciidoc +++ b/docs/reference/transform/apis/get-transform-stats.asciidoc @@ -192,6 +192,14 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=state-transform] [%collapsible%open] ==== +`delete_time_in_ms`::: +(long) +include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=delete-time-ms] + +`documents_deleted`::: +(long) +include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=docs-deleted-transform] + `documents_indexed`::: (long) include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=docs-indexed] diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/rest/action/RestCatTransformAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/rest/action/RestCatTransformAction.java index ba19265694c38..b133bbb2e3aaa 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/rest/action/RestCatTransformAction.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/rest/action/RestCatTransformAction.java @@ -169,6 +169,16 @@ private static Table getTableWithHeader() { .setAliases("doci") .build() ) + .addCell( + "delete_time", + TableColumnAttributeBuilder.builder("total time spent deleting documents", false).setAliases("dtime").build() + ) + .addCell( + "documents_deleted", + TableColumnAttributeBuilder.builder("the number of documents deleted from the destination index", false) + .setAliases("docd") + .build() + ) .addCell( "trigger_count", TableColumnAttributeBuilder.builder("the number of times the transform has been triggered", false).setAliases("tc").build() @@ -266,6 +276,8 @@ private Table buildTable(GetTransformAction.Response response, GetTransformStats .addCell(transformIndexerStats == null ? null : TimeValue.timeValueMillis(transformIndexerStats.getIndexTime())) .addCell(transformIndexerStats == null ? null : transformIndexerStats.getOutputDocuments()) + .addCell(transformIndexerStats == null ? null : TimeValue.timeValueMillis(transformIndexerStats.getDeleteTime())) + .addCell(transformIndexerStats == null ? null : transformIndexerStats.getNumDeletedDocuments()) .addCell(transformIndexerStats == null ? null : transformIndexerStats.getNumInvocations()) .addCell(transformIndexerStats == null ? null : transformIndexerStats.getNumPages()) .addCell(transformIndexerStats == null ? null : TimeValue.timeValueMillis(transformIndexerStats.getProcessingTime())) From 69aec685d5e498dc3a31f7371d92ed76befa06eb Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Mon, 8 Feb 2021 11:15:35 +0100 Subject: [PATCH 37/39] fix merge conflict --- .../xpack/transform/integration/TransformInternalIndexIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/integration/TransformInternalIndexIT.java b/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/integration/TransformInternalIndexIT.java index f7eb039fbeb64..daa0c039e8246 100644 --- a/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/integration/TransformInternalIndexIT.java +++ b/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/integration/TransformInternalIndexIT.java @@ -84,7 +84,7 @@ public void testUpdateDeletesOldTransformConfig() throws Exception { assertThat(getTransformResponse.getTransformConfigurations().get(0).getId(), equalTo(transformId)); UpdateTransformAction.Request updateTransformActionRequest = new UpdateTransformAction.Request( - new TransformConfigUpdate(null, null, null, null, "updated", null), + new TransformConfigUpdate(null, null, null, null, "updated", null, null), transformId, false); UpdateTransformAction.Response updateTransformActionResponse = client().execute(UpdateTransformAction.INSTANCE, updateTransformActionRequest).actionGet(); From 2b5c05121a1554443e0323fd4e075dd2a377b16c Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Mon, 8 Feb 2021 11:30:21 +0100 Subject: [PATCH 38/39] fail (with retry) if delete by query fails or returns with unexpected results --- .../transforms/TransformIndexer.java | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformIndexer.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformIndexer.java index 40c9253f36953..68557ce21c347 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformIndexer.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformIndexer.java @@ -45,6 +45,7 @@ import org.elasticsearch.xpack.transform.notifications.TransformAuditor; import org.elasticsearch.xpack.transform.persistence.TransformConfigManager; import org.elasticsearch.xpack.transform.transforms.Function.ChangeCollector; +import org.elasticsearch.xpack.transform.transforms.RetentionPolicyToDeleteByQueryRequestConverter.RetentionPolicyException; import org.elasticsearch.xpack.transform.utils.ExceptionRootCauseFinder; import java.time.Instant; @@ -471,26 +472,25 @@ private void executeRetentionPolicy(ActionListener listener) { // this should not happen as part of checkpointing if (bulkByScrollResponse.getVersionConflicts() > 0) { - logger.warn( - () -> new ParameterizedMessage( - "[{}] found [{}] version conflicts when deleting documents as part of the retention policy. " - + "Retry at the next checkpoint.", - getJobId(), + // note: the failure gets logged by the failure handler + listener.onFailure( + new RetentionPolicyException( + "found [{}] version conflicts when deleting documents as part of the retention policy.", bulkByScrollResponse.getDeleted() ) ); + return; } // paranoia: we are not expecting dbq to fail for other reasons if (bulkByScrollResponse.getBulkFailures().size() > 0 || bulkByScrollResponse.getSearchFailures().size() > 0) { assert false : "delete by query failed unexpectedly" + bulkByScrollResponse; - logger.warn( - () -> new ParameterizedMessage( - "[{}] found failures when deleting documents as part of the retention policy. " - + "Retry at the next checkpoint. Response: [{}]", - getJobId(), + listener.onFailure( + new RetentionPolicyException( + "found failures when deleting documents as part of the retention policy. Response: [{}]", bulkByScrollResponse ) ); + return; } finalizeCheckpoint(listener); From 90681b319ecebcc49748fd41e3559d0a02f38c2a Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Mon, 8 Feb 2021 13:55:55 +0100 Subject: [PATCH 39/39] fix code formating --- .../transforms/latest/LatestConfigTests.java | 35 ++++++++++++++++--- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/latest/LatestConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/latest/LatestConfigTests.java index bf04cf86753c5..ddc688445dfb5 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/latest/LatestConfigTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/latest/LatestConfigTests.java @@ -64,14 +64,24 @@ public void testValidate_ValidConfig() throws IOException { } public void testValidate_EmptyUniqueKey() throws IOException { - String json = "{" + " \"unique_key\": []," + " \"sort\": \"timestamp\"" + "}"; + // tag::NO_CODE_FORMAT + String json = "{" + + " \"unique_key\": []," + + " \"sort\": \"timestamp\"" + + "}"; + // end::NO_CODE_FORMAT LatestConfig config = createLatestConfigFromString(json); assertThat(config.validate(null).validationErrors(), contains("latest.unique_key must be non-empty")); } public void testValidate_EmptyUniqueKeyElement() throws IOException { - String json = "{" + " \"unique_key\": [ \"event1\", \"\", \"event2\", \"\", \"event3\" ]," + " \"sort\": \"timestamp\"" + "}"; + // tag::NO_CODE_FORMAT + String json = "{" + + " \"unique_key\": [ \"event1\", \"\", \"event2\", \"\", \"event3\" ]," + + " \"sort\": \"timestamp\"" + + "}"; + // end::NO_CODE_FORMAT LatestConfig config = createLatestConfigFromString(json); assertThat( @@ -81,7 +91,12 @@ public void testValidate_EmptyUniqueKeyElement() throws IOException { } public void testValidate_DuplicateUniqueKeyElement() throws IOException { - String json = "{" + " \"unique_key\": [ \"event1\", \"event2\", \"event1\" ]," + " \"sort\": \"timestamp\"" + "}"; + // tag::NO_CODE_FORMAT + String json = "{" + + " \"unique_key\": [ \"event1\", \"event2\", \"event1\" ]," + + " \"sort\": \"timestamp\"" + + "}"; + // end::NO_CODE_FORMAT LatestConfig config = createLatestConfigFromString(json); assertThat( @@ -91,14 +106,24 @@ public void testValidate_DuplicateUniqueKeyElement() throws IOException { } public void testValidate_EmptySort() throws IOException { - String json = "{" + " \"unique_key\": [ \"event1\", \"event2\", \"event3\" ]," + " \"sort\": \"\"" + "}"; + // tag::NO_CODE_FORMAT + String json = "{" + + " \"unique_key\": [ \"event1\", \"event2\", \"event3\" ]," + + " \"sort\": \"\"" + + "}"; + // end::NO_CODE_FORMAT LatestConfig config = createLatestConfigFromString(json); assertThat(config.validate(null).validationErrors(), contains("latest.sort must be non-empty")); } public void testValidate_EmptyUniqueKeyAndSort() throws IOException { - String json = "{" + " \"unique_key\": []," + " \"sort\": \"\"" + "}"; + // tag::NO_CODE_FORMAT + String json = "{" + + " \"unique_key\": []," + + " \"sort\": \"\"" + + "}"; + // end::NO_CODE_FORMAT LatestConfig config = createLatestConfigFromString(json); assertThat(