diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/TransformNamedXContentProvider.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/TransformNamedXContentProvider.java index ad21119b929a6..8b7dcdcbe18e0 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/TransformNamedXContentProvider.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/TransformNamedXContentProvider.java @@ -8,7 +8,9 @@ package org.elasticsearch.client.transform; +import org.elasticsearch.client.transform.transforms.RetentionPolicyConfig; import org.elasticsearch.client.transform.transforms.SyncConfig; +import org.elasticsearch.client.transform.transforms.TimeRetentionPolicyConfig; import org.elasticsearch.client.transform.transforms.TimeSyncConfig; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.xcontent.NamedXContentRegistry; @@ -22,9 +24,13 @@ public class TransformNamedXContentProvider implements NamedXContentProvider { @Override public List getNamedXContentParsers() { return Arrays.asList( - new NamedXContentRegistry.Entry(SyncConfig.class, - new ParseField(TimeSyncConfig.NAME), - TimeSyncConfig::fromXContent)); + new NamedXContentRegistry.Entry(SyncConfig.class, new ParseField(TimeSyncConfig.NAME), TimeSyncConfig::fromXContent), + new NamedXContentRegistry.Entry( + RetentionPolicyConfig.class, + new ParseField(TimeRetentionPolicyConfig.NAME), + TimeRetentionPolicyConfig::fromXContent + ) + ); } } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/RetentionPolicyConfig.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/RetentionPolicyConfig.java new file mode 100644 index 0000000000000..9ad72c8db3f6f --- /dev/null +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/RetentionPolicyConfig.java @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.client.transform.transforms; + +import org.elasticsearch.common.xcontent.ToXContentObject; + +public interface RetentionPolicyConfig extends ToXContentObject { + + /** + * Returns the name of the writeable object + */ + String getName(); +} diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/TimeRetentionPolicyConfig.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/TimeRetentionPolicyConfig.java new file mode 100644 index 0000000000000..cbff1ce71e18c --- /dev/null +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/TimeRetentionPolicyConfig.java @@ -0,0 +1,99 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.client.transform.transforms; + +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.io.IOException; +import java.util.Objects; + +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; + +public class TimeRetentionPolicyConfig implements RetentionPolicyConfig { + + public static final String NAME = "time"; + + private static final ParseField FIELD = new ParseField("field"); + private static final ParseField MAX_AGE = new ParseField("max_age"); + + private final String field; + private final TimeValue maxAge; + + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "time_retention_policy_config", + true, + args -> new TimeRetentionPolicyConfig((String) args[0], args[1] != null ? (TimeValue) args[1] : TimeValue.ZERO) + ); + + static { + PARSER.declareString(constructorArg(), FIELD); + PARSER.declareField( + constructorArg(), + (p, c) -> TimeValue.parseTimeValue(p.text(), MAX_AGE.getPreferredName()), + MAX_AGE, + ObjectParser.ValueType.STRING + ); + } + + public static TimeRetentionPolicyConfig fromXContent(XContentParser parser) { + return PARSER.apply(parser, null); + } + + public TimeRetentionPolicyConfig(String field, TimeValue maxAge) { + this.field = field; + this.maxAge = maxAge; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(FIELD.getPreferredName(), field); + builder.field(MAX_AGE.getPreferredName(), maxAge.getStringRep()); + builder.endObject(); + return builder; + } + + public String getField() { + return field; + } + + public TimeValue getMaxAge() { + return maxAge; + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + + if (other == null || getClass() != other.getClass()) { + return false; + } + + final TimeRetentionPolicyConfig that = (TimeRetentionPolicyConfig) other; + + return Objects.equals(this.field, that.field) && Objects.equals(this.maxAge, that.maxAge); + } + + @Override + public int hashCode() { + return Objects.hash(field, maxAge); + } + + @Override + public String getName() { + return NAME; + } +} diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/TransformConfig.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/TransformConfig.java index 21f4e6d95f0f1..8e468c07e2b27 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/TransformConfig.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/TransformConfig.java @@ -21,7 +21,6 @@ import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.common.xcontent.XContentParserUtils; import java.io.IOException; import java.time.Instant; @@ -41,6 +40,7 @@ public class TransformConfig implements ToXContentObject { public static final ParseField SETTINGS = new ParseField("settings"); public static final ParseField VERSION = new ParseField("version"); public static final ParseField CREATE_TIME = new ParseField("create_time"); + public static final ParseField RETENTION_POLICY = new ParseField("retention_policy"); // types of transforms public static final ParseField PIVOT_TRANSFORM = new ParseField("pivot"); public static final ParseField LATEST_TRANSFORM = new ParseField("latest"); @@ -54,6 +54,7 @@ public class TransformConfig implements ToXContentObject { private final PivotConfig pivotConfig; private final LatestConfig latestConfig; private final String description; + private final RetentionPolicyConfig retentionPolicyConfig; private final Version transformVersion; private final Instant createTime; @@ -70,8 +71,9 @@ public class TransformConfig implements ToXContentObject { LatestConfig latestConfig = (LatestConfig) args[6]; String description = (String) args[7]; SettingsConfig settings = (SettingsConfig) args[8]; - Instant createTime = (Instant) args[9]; - String transformVersion = (String) args[10]; + RetentionPolicyConfig retentionPolicyConfig = (RetentionPolicyConfig) args[9]; + Instant createTime = (Instant) args[10]; + String transformVersion = (String) args[11]; return new TransformConfig( id, source, @@ -82,6 +84,7 @@ public class TransformConfig implements ToXContentObject { latestConfig, description, settings, + retentionPolicyConfig, createTime, transformVersion ); @@ -98,11 +101,16 @@ public class TransformConfig implements ToXContentObject { FREQUENCY, ObjectParser.ValueType.STRING ); - PARSER.declareObject(optionalConstructorArg(), (p, c) -> parseSyncConfig(p), SYNC); + PARSER.declareNamedObject(optionalConstructorArg(), (p, c, n) -> p.namedObject(SyncConfig.class, n, c), SYNC); PARSER.declareObject(optionalConstructorArg(), (p, c) -> PivotConfig.fromXContent(p), PIVOT_TRANSFORM); PARSER.declareObject(optionalConstructorArg(), (p, c) -> LatestConfig.fromXContent(p), LATEST_TRANSFORM); PARSER.declareString(optionalConstructorArg(), DESCRIPTION); PARSER.declareObject(optionalConstructorArg(), (p, c) -> SettingsConfig.fromXContent(p), SETTINGS); + PARSER.declareNamedObject( + optionalConstructorArg(), + (p, c, n) -> p.namedObject(RetentionPolicyConfig.class, n, c), + RETENTION_POLICY + ); PARSER.declareField( optionalConstructorArg(), p -> TimeUtil.parseTimeFieldToInstant(p, CREATE_TIME.getPreferredName()), @@ -112,14 +120,6 @@ public class TransformConfig implements ToXContentObject { PARSER.declareString(optionalConstructorArg(), VERSION); } - private static SyncConfig parseSyncConfig(XContentParser parser) throws IOException { - XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser); - XContentParserUtils.ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.nextToken(), parser); - SyncConfig syncConfig = parser.namedObject(SyncConfig.class, parser.currentName(), true); - XContentParserUtils.ensureExpectedToken(XContentParser.Token.END_OBJECT, parser.nextToken(), parser); - return syncConfig; - } - public static TransformConfig fromXContent(final XContentParser parser) { return PARSER.apply(parser, null); } @@ -136,7 +136,7 @@ public static TransformConfig fromXContent(final XContentParser parser) { * @return A TransformConfig to preview, NOTE it will have a {@code null} id, destination and index. */ public static TransformConfig forPreview(final SourceConfig source, final PivotConfig pivotConfig) { - return new TransformConfig(null, source, null, null, null, pivotConfig, null, null, null, null, null); + return new TransformConfig(null, source, null, null, null, pivotConfig, null, null, null, null, null, null); } /** @@ -151,7 +151,7 @@ public static TransformConfig forPreview(final SourceConfig source, final PivotC * @return A TransformConfig to preview, NOTE it will have a {@code null} id, destination and index. */ public static TransformConfig forPreview(final SourceConfig source, final LatestConfig latestConfig) { - return new TransformConfig(null, source, null, null, null, null, latestConfig, null, null, null, null); + return new TransformConfig(null, source, null, null, null, null, latestConfig, null, null, null, null, null); } TransformConfig( @@ -164,6 +164,7 @@ public static TransformConfig forPreview(final SourceConfig source, final Latest final LatestConfig latestConfig, final String description, final SettingsConfig settings, + final RetentionPolicyConfig retentionPolicyConfig, final Instant createTime, final String version ) { @@ -176,6 +177,7 @@ public static TransformConfig forPreview(final SourceConfig source, final Latest this.latestConfig = latestConfig; this.description = description; this.settings = settings; + this.retentionPolicyConfig = retentionPolicyConfig; this.createTime = createTime == null ? null : Instant.ofEpochMilli(createTime.toEpochMilli()); this.transformVersion = version == null ? null : Version.fromString(version); } @@ -226,6 +228,11 @@ public SettingsConfig getSettings() { return settings; } + @Nullable + public RetentionPolicyConfig getRetentionPolicyConfig() { + return retentionPolicyConfig; + } + @Override public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException { builder.startObject(); @@ -258,6 +265,11 @@ public XContentBuilder toXContent(final XContentBuilder builder, final Params pa if (settings != null) { builder.field(SETTINGS.getPreferredName(), settings); } + if (retentionPolicyConfig != null) { + builder.startObject(RETENTION_POLICY.getPreferredName()); + builder.field(retentionPolicyConfig.getName(), retentionPolicyConfig); + builder.endObject(); + } if (createTime != null) { builder.timeField(CREATE_TIME.getPreferredName(), CREATE_TIME.getPreferredName() + "_string", createTime.toEpochMilli()); } @@ -290,13 +302,26 @@ public boolean equals(Object other) { && Objects.equals(this.settings, that.settings) && Objects.equals(this.createTime, that.createTime) && Objects.equals(this.pivotConfig, that.pivotConfig) - && Objects.equals(this.latestConfig, that.latestConfig); + && Objects.equals(this.latestConfig, that.latestConfig) + && Objects.equals(this.retentionPolicyConfig, that.retentionPolicyConfig); } @Override public int hashCode() { return Objects.hash( - id, source, dest, frequency, syncConfig, settings, createTime, transformVersion, pivotConfig, latestConfig, description); + id, + source, + dest, + frequency, + syncConfig, + settings, + createTime, + transformVersion, + pivotConfig, + latestConfig, + description, + retentionPolicyConfig + ); } @Override @@ -319,6 +344,7 @@ public static class Builder { private LatestConfig latestConfig; private SettingsConfig settings; private String description; + private RetentionPolicyConfig retentionPolicyConfig; public Builder setId(String id) { this.id = id; @@ -365,9 +391,26 @@ public Builder setSettings(SettingsConfig settings) { return this; } + public Builder setRetentionPolicyConfig(RetentionPolicyConfig retentionPolicyConfig) { + this.retentionPolicyConfig = retentionPolicyConfig; + return this; + } + public TransformConfig build() { return new TransformConfig( - id, source, dest, frequency, syncConfig, pivotConfig, latestConfig, description, settings, null, null); + id, + source, + dest, + frequency, + syncConfig, + pivotConfig, + latestConfig, + description, + settings, + retentionPolicyConfig, + null, + null + ); } } } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/TransformIndexerStats.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/TransformIndexerStats.java index 8706659538d80..f4952a69fafe9 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/TransformIndexerStats.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/TransformIndexerStats.java @@ -26,10 +26,12 @@ public class TransformIndexerStats { static ParseField PAGES_PROCESSED = new ParseField("pages_processed"); static ParseField DOCUMENTS_PROCESSED = new ParseField("documents_processed"); static ParseField DOCUMENTS_INDEXED = new ParseField("documents_indexed"); + static ParseField DOCUMENTS_DELETED = new ParseField("documents_deleted"); static ParseField TRIGGER_COUNT = new ParseField("trigger_count"); static ParseField INDEX_TIME_IN_MS = new ParseField("index_time_in_ms"); static ParseField SEARCH_TIME_IN_MS = new ParseField("search_time_in_ms"); static ParseField PROCESSING_TIME_IN_MS = new ParseField("processing_time_in_ms"); + static ParseField DELETE_TIME_IN_MS = new ParseField("delete_time_in_ms"); static ParseField INDEX_TOTAL = new ParseField("index_total"); static ParseField SEARCH_TOTAL = new ParseField("search_total"); static ParseField PROCESSING_TOTAL = new ParseField("processing_total"); @@ -52,9 +54,11 @@ public class TransformIndexerStats { unboxSafe(args[9], 0L), unboxSafe(args[10], 0L), unboxSafe(args[11], 0L), - unboxSafe(args[12], 0.0), - unboxSafe(args[13], 0.0), - unboxSafe(args[14], 0.0) + unboxSafe(args[12], 0L), + unboxSafe(args[13], 0L), + unboxSafe(args[14], 0.0), + unboxSafe(args[15], 0.0), + unboxSafe(args[16], 0.0) ) ); @@ -62,10 +66,12 @@ public class TransformIndexerStats { LENIENT_PARSER.declareLong(optionalConstructorArg(), PAGES_PROCESSED); LENIENT_PARSER.declareLong(optionalConstructorArg(), DOCUMENTS_PROCESSED); LENIENT_PARSER.declareLong(optionalConstructorArg(), DOCUMENTS_INDEXED); + LENIENT_PARSER.declareLong(optionalConstructorArg(), DOCUMENTS_DELETED); LENIENT_PARSER.declareLong(optionalConstructorArg(), TRIGGER_COUNT); LENIENT_PARSER.declareLong(optionalConstructorArg(), INDEX_TIME_IN_MS); LENIENT_PARSER.declareLong(optionalConstructorArg(), SEARCH_TIME_IN_MS); LENIENT_PARSER.declareLong(optionalConstructorArg(), PROCESSING_TIME_IN_MS); + LENIENT_PARSER.declareLong(optionalConstructorArg(), DELETE_TIME_IN_MS); LENIENT_PARSER.declareLong(optionalConstructorArg(), INDEX_TOTAL); LENIENT_PARSER.declareLong(optionalConstructorArg(), SEARCH_TOTAL); LENIENT_PARSER.declareLong(optionalConstructorArg(), PROCESSING_TOTAL); @@ -86,12 +92,14 @@ public static TransformIndexerStats fromXContent(XContentParser parser) throws I private final long pagesProcessed; private final long documentsProcessed; private final long documentsIndexed; + private final long documentsDeleted; private final long triggerCount; private final long indexTime; private final long indexTotal; private final long searchTime; private final long searchTotal; private final long processingTime; + private final long deleteTime; private final long processingTotal; private final long indexFailures; private final long searchFailures; @@ -100,10 +108,12 @@ public TransformIndexerStats( long pagesProcessed, long documentsProcessed, long documentsIndexed, + long documentsDeleted, long triggerCount, long indexTime, long searchTime, long processingTime, + long deleteTime, long indexTotal, long searchTotal, long processingTotal, @@ -116,10 +126,12 @@ public TransformIndexerStats( this.pagesProcessed = pagesProcessed; this.documentsProcessed = documentsProcessed; this.documentsIndexed = documentsIndexed; + this.documentsDeleted = documentsDeleted; this.triggerCount = triggerCount; this.indexTime = indexTime; this.indexTotal = indexTotal; this.searchTime = searchTime; + this.deleteTime = deleteTime; this.searchTotal = searchTotal; this.processingTime = processingTime; this.processingTotal = processingTotal; @@ -206,6 +218,13 @@ public long getOutputDocuments() { return getDocumentsIndexed(); } + /** + * Number of documents deleted + */ + public long getDocumentsDeleted() { + return documentsDeleted; + } + /** * Number of index failures that have occurred */ @@ -241,6 +260,13 @@ public long getProcessingTime() { return processingTime; } + /** + * Returns the time spent deleting (cumulative) in milliseconds + */ + public long getDeleteTime() { + return deleteTime; + } + /** * Returns the total number of indexing requests that have been processed * (Note: this is not the number of _documents_ that have been indexed) @@ -278,10 +304,12 @@ public boolean equals(Object other) { return Objects.equals(this.pagesProcessed, that.pagesProcessed) && Objects.equals(this.documentsProcessed, that.documentsProcessed) && Objects.equals(this.documentsIndexed, that.documentsIndexed) + && Objects.equals(this.documentsDeleted, that.documentsDeleted) && Objects.equals(this.triggerCount, that.triggerCount) && Objects.equals(this.indexTime, that.indexTime) && Objects.equals(this.searchTime, that.searchTime) && Objects.equals(this.processingTime, that.processingTime) + && Objects.equals(this.deleteTime, that.deleteTime) && Objects.equals(this.indexFailures, that.indexFailures) && Objects.equals(this.searchFailures, that.searchFailures) && Objects.equals(this.indexTotal, that.indexTotal) @@ -298,10 +326,12 @@ public int hashCode() { pagesProcessed, documentsProcessed, documentsIndexed, + documentsDeleted, triggerCount, indexTime, searchTime, processingTime, + deleteTime, indexFailures, searchFailures, indexTotal, diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java index eb11fafcc4f8b..5ede2016c2e42 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java @@ -9,6 +9,7 @@ package org.elasticsearch.client; import com.fasterxml.jackson.core.JsonParseException; + import org.apache.http.HttpEntity; import org.apache.http.HttpHost; import org.apache.http.HttpResponse; @@ -78,7 +79,9 @@ import org.elasticsearch.client.ml.inference.trainedmodel.ensemble.WeightedSum; import org.elasticsearch.client.ml.inference.trainedmodel.langident.LangIdentNeuralNetwork; import org.elasticsearch.client.ml.inference.trainedmodel.tree.Tree; +import org.elasticsearch.client.transform.transforms.RetentionPolicyConfig; import org.elasticsearch.client.transform.transforms.SyncConfig; +import org.elasticsearch.client.transform.transforms.TimeRetentionPolicyConfig; import org.elasticsearch.client.transform.transforms.TimeSyncConfig; import org.elasticsearch.common.CheckedFunction; import org.elasticsearch.common.bytes.BytesReference; @@ -697,7 +700,7 @@ public void testDefaultNamedXContents() { public void testProvidedNamedXContents() { List namedXContents = RestHighLevelClient.getProvidedNamedXContents(); - assertEquals(76, namedXContents.size()); + assertEquals(77, namedXContents.size()); Map, Integer> categories = new HashMap<>(); List names = new ArrayList<>(); for (NamedXContentRegistry.Entry namedXContent : namedXContents) { @@ -707,7 +710,7 @@ public void testProvidedNamedXContents() { categories.put(namedXContent.categoryClass, counter + 1); } } - assertEquals("Had: " + categories, 14, categories.size()); + assertEquals("Had: " + categories, 15, categories.size()); assertEquals(Integer.valueOf(3), categories.get(Aggregation.class)); assertTrue(names.contains(ChildrenAggregationBuilder.NAME)); assertTrue(names.contains(MatrixStatsAggregationBuilder.NAME)); @@ -744,6 +747,8 @@ public void testProvidedNamedXContents() { assertTrue(names.contains(ClassificationStats.NAME.getPreferredName())); assertEquals(Integer.valueOf(1), categories.get(SyncConfig.class)); assertTrue(names.contains(TimeSyncConfig.NAME)); + assertEquals(Integer.valueOf(1), categories.get(RetentionPolicyConfig.class)); + assertTrue(names.contains(TimeRetentionPolicyConfig.NAME)); assertEquals(Integer.valueOf(3), categories.get(org.elasticsearch.client.ml.dataframe.evaluation.Evaluation.class)); assertThat(names, hasItems(OutlierDetection.NAME, Classification.NAME, Regression.NAME)); assertEquals(Integer.valueOf(13), categories.get(org.elasticsearch.client.ml.dataframe.evaluation.EvaluationMetric.class)); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/TransformIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/TransformIT.java index b8584900a469f..37316a543cd17 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/TransformIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/TransformIT.java @@ -82,18 +82,18 @@ private void createIndex(String indexName) throws IOException { XContentBuilder builder = jsonBuilder(); builder.startObject() - .startObject("properties") - .startObject("timestamp") - .field("type", "date") - .endObject() - .startObject("user_id") - .field("type", "keyword") - .endObject() - .startObject("stars") - .field("type", "integer") - .endObject() - .endObject() - .endObject(); + .startObject("properties") + .startObject("timestamp") + .field("type", "date") + .endObject() + .startObject("user_id") + .field("type", "keyword") + .endObject() + .startObject("stars") + .field("type", "integer") + .endObject() + .endObject() + .endObject(); CreateIndexRequest request = new CreateIndexRequest(indexName); request.mapping(builder); @@ -180,18 +180,17 @@ public void testCreateDelete() throws IOException { TransformConfig transform = validDataFrameTransformConfig(id, sourceIndex, "pivot-dest"); TransformClient client = highLevelClient().transform(); - AcknowledgedResponse ack = execute(new PutTransformRequest(transform), client::putTransform, - client::putTransformAsync); + AcknowledgedResponse ack = execute(new PutTransformRequest(transform), client::putTransform, client::putTransformAsync); assertTrue(ack.isAcknowledged()); - ack = execute(new DeleteTransformRequest(transform.getId()), client::deleteTransform, - client::deleteTransformAsync); + ack = execute(new DeleteTransformRequest(transform.getId()), client::deleteTransform, client::deleteTransformAsync); assertTrue(ack.isAcknowledged()); // The second delete should fail - ElasticsearchStatusException deleteError = expectThrows(ElasticsearchStatusException.class, - () -> execute(new DeleteTransformRequest(transform.getId()), client::deleteTransform, - client::deleteTransformAsync)); + ElasticsearchStatusException deleteError = expectThrows( + ElasticsearchStatusException.class, + () -> execute(new DeleteTransformRequest(transform.getId()), client::deleteTransform, client::deleteTransformAsync) + ); assertThat(deleteError.getMessage(), containsString("Transform with id [test-crud] could not be found")); } @@ -200,25 +199,27 @@ public void testUpdate() throws IOException { createIndex(sourceIndex); String id = "test-update"; - TransformConfig transform = validDataFrameTransformConfigBuilder(id, sourceIndex, "pivot-dest") - .setSyncConfig(new TimeSyncConfig("timefield", TimeValue.timeValueSeconds(60))) - .build(); + TransformConfig transform = validDataFrameTransformConfigBuilder(id, sourceIndex, "pivot-dest").setSyncConfig( + new TimeSyncConfig("timefield", TimeValue.timeValueSeconds(60)) + ).build(); TransformClient client = highLevelClient().transform(); - AcknowledgedResponse ack = execute(new PutTransformRequest(transform), client::putTransform, - client::putTransformAsync); + AcknowledgedResponse ack = execute(new PutTransformRequest(transform), client::putTransform, client::putTransformAsync); assertTrue(ack.isAcknowledged()); String updatedDescription = "my new description"; TransformConfigUpdate update = TransformConfigUpdate.builder().setDescription(updatedDescription).build(); UpdateTransformResponse response = execute( - new UpdateTransformRequest(update, id), client::updateTransform, - client::updateTransformAsync); + new UpdateTransformRequest(update, id), + client::updateTransform, + client::updateTransformAsync + ); assertThat(response.getTransformConfiguration().getDescription(), equalTo(updatedDescription)); - ElasticsearchStatusException updateError = expectThrows(ElasticsearchStatusException.class, - () -> execute(new UpdateTransformRequest(update, "missing-transform"), client::updateTransform, - client::updateTransformAsync)); + ElasticsearchStatusException updateError = expectThrows( + ElasticsearchStatusException.class, + () -> execute(new UpdateTransformRequest(update, "missing-transform"), client::updateTransform, client::updateTransformAsync) + ); assertThat(updateError.getMessage(), containsString("Transform with id [missing-transform] could not be found")); } @@ -233,8 +234,7 @@ public void testCreateDeleteWithDefer() throws IOException { AcknowledgedResponse ack = execute(request, client::putTransform, client::putTransformAsync); assertTrue(ack.isAcknowledged()); - ack = execute(new DeleteTransformRequest(transform.getId()), client::deleteTransform, - client::deleteTransformAsync); + ack = execute(new DeleteTransformRequest(transform.getId()), client::deleteTransform, client::deleteTransformAsync); assertTrue(ack.isAcknowledged()); } @@ -249,8 +249,7 @@ public void testGetTransform() throws IOException { putTransform(transform); GetTransformRequest getRequest = new GetTransformRequest(id); - GetTransformResponse getResponse = execute(getRequest, client::getTransform, - client::getTransformAsync); + GetTransformResponse getResponse = execute(getRequest, client::getTransform, client::getTransformAsync); assertNull(getResponse.getInvalidTransforms()); assertThat(getResponse.getTransformConfigurations(), hasSize(1)); assertEquals(transform.getId(), getResponse.getTransformConfigurations().get(0).getId()); @@ -269,21 +268,18 @@ public void testGetAllAndPageTransforms() throws IOException { putTransform(transform); GetTransformRequest getRequest = new GetTransformRequest("_all"); - GetTransformResponse getResponse = execute(getRequest, client::getTransform, - client::getTransformAsync); + GetTransformResponse getResponse = execute(getRequest, client::getTransform, client::getTransformAsync); assertNull(getResponse.getInvalidTransforms()); assertThat(getResponse.getTransformConfigurations(), hasSize(2)); assertEquals(transform.getId(), getResponse.getTransformConfigurations().get(1).getId()); - getRequest.setPageParams(new PageParams(0,1)); - getResponse = execute(getRequest, client::getTransform, - client::getTransformAsync); + getRequest.setPageParams(new PageParams(0, 1)); + getResponse = execute(getRequest, client::getTransform, client::getTransformAsync); assertNull(getResponse.getInvalidTransforms()); assertThat(getResponse.getTransformConfigurations(), hasSize(1)); GetTransformRequest getMulitple = new GetTransformRequest("test-get-all-1", "test-get-all-2"); - getResponse = execute(getMulitple, client::getTransform, - client::getTransformAsync); + getResponse = execute(getMulitple, client::getTransform, client::getTransformAsync); assertNull(getResponse.getInvalidTransforms()); assertThat(getResponse.getTransformConfigurations(), hasSize(2)); } @@ -291,9 +287,10 @@ public void testGetAllAndPageTransforms() throws IOException { public void testGetMissingTransform() { TransformClient client = highLevelClient().transform(); - ElasticsearchStatusException missingError = expectThrows(ElasticsearchStatusException.class, - () -> execute(new GetTransformRequest("unknown"), client::getTransform, - client::getTransformAsync)); + ElasticsearchStatusException missingError = expectThrows( + ElasticsearchStatusException.class, + () -> execute(new GetTransformRequest("unknown"), client::getTransform, client::getTransformAsync) + ); assertThat(missingError.status(), equalTo(RestStatus.NOT_FOUND)); } @@ -308,31 +305,33 @@ public void testStartStop() throws IOException { putTransform(transform); StartTransformRequest startRequest = new StartTransformRequest(id); - StartTransformResponse startResponse = - execute(startRequest, client::startTransform, client::startTransformAsync); + StartTransformResponse startResponse = execute(startRequest, client::startTransform, client::startTransformAsync); assertTrue(startResponse.isAcknowledged()); assertThat(startResponse.getNodeFailures(), empty()); assertThat(startResponse.getTaskFailures(), empty()); - GetTransformStatsResponse statsResponse = execute(new GetTransformStatsRequest(id), - client::getTransformStats, client::getTransformStatsAsync); + GetTransformStatsResponse statsResponse = execute( + new GetTransformStatsRequest(id), + client::getTransformStats, + client::getTransformStatsAsync + ); assertThat(statsResponse.getTransformsStats(), hasSize(1)); TransformStats.State taskState = statsResponse.getTransformsStats().get(0).getState(); // Since we are non-continuous, the transform could auto-stop between being started earlier and us gathering the statistics - assertThat(taskState, oneOf(TransformStats.State.STARTED, TransformStats.State.INDEXING, - TransformStats.State.STOPPING, TransformStats.State.STOPPED)); + assertThat( + taskState, + oneOf(TransformStats.State.STARTED, TransformStats.State.INDEXING, TransformStats.State.STOPPING, TransformStats.State.STOPPED) + ); StopTransformRequest stopRequest = new StopTransformRequest(id, Boolean.TRUE, null, false); - StopTransformResponse stopResponse = - execute(stopRequest, client::stopTransform, client::stopTransformAsync); + StopTransformResponse stopResponse = execute(stopRequest, client::stopTransform, client::stopTransformAsync); assertTrue(stopResponse.isAcknowledged()); assertThat(stopResponse.getNodeFailures(), empty()); assertThat(stopResponse.getTaskFailures(), empty()); // Calling stop with wait_for_completion assures that we will be in the `STOPPED` state for the transform task - statsResponse = execute(new GetTransformStatsRequest(id), - client::getTransformStats, client::getTransformStatsAsync); + statsResponse = execute(new GetTransformStatsRequest(id), client::getTransformStats, client::getTransformStatsAsync); taskState = statsResponse.getTransformsStats().get(0).getState(); assertThat(taskState, is(TransformStats.State.STOPPED)); } @@ -346,9 +345,11 @@ public void testPreview() throws IOException { TransformConfig transform = validDataFrameTransformConfig("test-preview", sourceIndex, null); TransformClient client = highLevelClient().transform(); - PreviewTransformResponse preview = execute(new PreviewTransformRequest(transform), - client::previewTransform, - client::previewTransformAsync); + PreviewTransformResponse preview = execute( + new PreviewTransformRequest(transform), + client::previewTransform, + client::previewTransformAsync + ); List> docs = preview.getDocs(); assertThat(docs, hasSize(2)); @@ -362,7 +363,7 @@ public void testPreview() throws IOException { Map mappings = preview.getMappings(); assertThat(mappings, hasKey("properties")); - Map fields = (Map)mappings.get("properties"); + Map fields = (Map) mappings.get("properties"); assertThat(fields.get("reviewer"), equalTo(Map.of("type", "keyword"))); assertThat(fields.get("avg_rating"), equalTo(Map.of("type", "double"))); } @@ -372,8 +373,7 @@ private TransformConfig validDataFrameTransformConfig(String id, String source, } private TransformConfig.Builder validDataFrameTransformConfigBuilder(String id, String source, String destination) { - GroupConfig groupConfig = GroupConfig.builder().groupBy("reviewer", - TermsGroupSource.builder().setField("user_id").build()).build(); + GroupConfig groupConfig = GroupConfig.builder().groupBy("reviewer", TermsGroupSource.builder().setField("user_id").build()).build(); AggregatorFactories.Builder aggBuilder = new AggregatorFactories.Builder(); aggBuilder.addAggregator(AggregationBuilders.avg("avg_rating").field("stars")); PivotConfig pivotConfig = PivotConfig.builder().setGroups(groupConfig).setAggregations(aggBuilder).build(); @@ -394,8 +394,7 @@ public void testGetStats() throws Exception { createIndex(sourceIndex); indexData(sourceIndex); - GroupConfig groupConfig = GroupConfig.builder().groupBy("reviewer", - TermsGroupSource.builder().setField("user_id").build()).build(); + GroupConfig groupConfig = GroupConfig.builder().groupBy("reviewer", TermsGroupSource.builder().setField("user_id").build()).build(); AggregatorFactories.Builder aggBuilder = new AggregatorFactories.Builder(); aggBuilder.addAggregator(AggregationBuilders.avg("avg_rating").field("stars")); PivotConfig pivotConfig = PivotConfig.builder().setGroups(groupConfig).setAggregations(aggBuilder).build(); @@ -412,8 +411,11 @@ public void testGetStats() throws Exception { TransformClient client = highLevelClient().transform(); putTransform(transform); - GetTransformStatsResponse statsResponse = execute(new GetTransformStatsRequest(id), - client::getTransformStats, client::getTransformStatsAsync); + GetTransformStatsResponse statsResponse = execute( + new GetTransformStatsRequest(id), + client::getTransformStats, + client::getTransformStatsAsync + ); assertEquals(1, statsResponse.getTransformsStats().size()); TransformStats stats = statsResponse.getTransformsStats().get(0); @@ -432,33 +434,46 @@ public void testGetStats() throws Exception { 0L, 0L, 0L, + 0L, + 0L, 0.0, 0.0, - 0.0); + 0.0 + ); assertEquals(zeroIndexerStats, stats.getIndexerStats()); // start the transform - StartTransformResponse startTransformResponse = execute(new StartTransformRequest(id), + StartTransformResponse startTransformResponse = execute( + new StartTransformRequest(id), client::startTransform, - client::startTransformAsync); + client::startTransformAsync + ); assertThat(startTransformResponse.isAcknowledged(), is(true)); assertBusy(() -> { - GetTransformStatsResponse response = execute(new GetTransformStatsRequest(id), - client::getTransformStats, client::getTransformStatsAsync); + GetTransformStatsResponse response = execute( + new GetTransformStatsRequest(id), + client::getTransformStats, + client::getTransformStatsAsync + ); TransformStats stateAndStats = response.getTransformsStats().get(0); assertNotEquals(zeroIndexerStats, stateAndStats.getIndexerStats()); - assertThat(stateAndStats.getState(), oneOf(TransformStats.State.STARTED, TransformStats.State.INDEXING, - TransformStats.State.STOPPING, TransformStats.State.STOPPED)); + assertThat( + stateAndStats.getState(), + oneOf( + TransformStats.State.STARTED, + TransformStats.State.INDEXING, + TransformStats.State.STOPPING, + TransformStats.State.STOPPED + ) + ); assertThat(stateAndStats.getReason(), is(nullValue())); }); } void putTransform(TransformConfig config) throws IOException { TransformClient client = highLevelClient().transform(); - AcknowledgedResponse ack = execute(new PutTransformRequest(config), client::putTransform, - client::putTransformAsync); + AcknowledgedResponse ack = execute(new PutTransformRequest(config), client::putTransform, client::putTransformAsync); assertTrue(ack.isAcknowledged()); transformsToClean.add(config.getId()); } } - diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/TimeRetentionPolicyConfigTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/TimeRetentionPolicyConfigTests.java new file mode 100644 index 0000000000000..33be3e535d5e6 --- /dev/null +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/TimeRetentionPolicyConfigTests.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.client.transform.transforms; + +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.test.AbstractXContentTestCase; + +import java.io.IOException; + +public class TimeRetentionPolicyConfigTests extends AbstractXContentTestCase { + + public static TimeRetentionPolicyConfig randomTimeRetentionPolicyConfig() { + return new TimeRetentionPolicyConfig(randomAlphaOfLengthBetween(1, 10), new TimeValue(randomNonNegativeLong())); + } + + @Override + protected TimeRetentionPolicyConfig createTestInstance() { + return randomTimeRetentionPolicyConfig(); + } + + @Override + protected TimeRetentionPolicyConfig doParseInstance(XContentParser parser) throws IOException { + return TimeRetentionPolicyConfig.fromXContent(parser); + } + + @Override + protected boolean supportsUnknownFields() { + return true; + } +} diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/TransformConfigTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/TransformConfigTests.java index 30daddc0d9e68..1579d29df7a2b 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/TransformConfigTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/TransformConfigTests.java @@ -52,6 +52,7 @@ public static TransformConfig randomTransformConfig() { latestConfig, randomBoolean() ? null : randomAlphaOfLengthBetween(1, 100), SettingsConfigTests.randomSettingsConfig(), + randomBoolean() ? null : randomRetentionPolicyConfig(), randomBoolean() ? null : Instant.now(), randomBoolean() ? null : Version.CURRENT.toString() ); @@ -61,6 +62,10 @@ public static SyncConfig randomSyncConfig() { return TimeSyncConfigTests.randomTimeSyncConfig(); } + public static RetentionPolicyConfig randomRetentionPolicyConfig() { + return TimeRetentionPolicyConfigTests.randomTimeRetentionPolicyConfig(); + } + @Override protected TransformConfig createTestInstance() { return randomTransformConfig(); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/TransformIndexerStatsTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/TransformIndexerStatsTests.java index 971dbb82622c1..beee811328923 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/TransformIndexerStatsTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/TransformIndexerStatsTests.java @@ -40,6 +40,8 @@ public static TransformIndexerStats randomStats() { randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), + randomNonNegativeLong(), + randomNonNegativeLong(), randomDouble(), randomDouble(), randomDouble() @@ -52,6 +54,7 @@ public static void toXContent(TransformIndexerStats stats, XContentBuilder build builder.field(TransformIndexerStats.PAGES_PROCESSED.getPreferredName(), stats.getPagesProcessed()); builder.field(TransformIndexerStats.DOCUMENTS_PROCESSED.getPreferredName(), stats.getDocumentsProcessed()); builder.field(TransformIndexerStats.DOCUMENTS_INDEXED.getPreferredName(), stats.getDocumentsIndexed()); + builder.field(TransformIndexerStats.DOCUMENTS_DELETED.getPreferredName(), stats.getDocumentsDeleted()); builder.field(TransformIndexerStats.TRIGGER_COUNT.getPreferredName(), stats.getTriggerCount()); builder.field(TransformIndexerStats.INDEX_TIME_IN_MS.getPreferredName(), stats.getIndexTime()); builder.field(TransformIndexerStats.INDEX_TOTAL.getPreferredName(), stats.getIndexTotal()); @@ -60,6 +63,7 @@ public static void toXContent(TransformIndexerStats stats, XContentBuilder build builder.field(TransformIndexerStats.SEARCH_TOTAL.getPreferredName(), stats.getSearchTotal()); builder.field(TransformIndexerStats.PROCESSING_TIME_IN_MS.getPreferredName(), stats.getProcessingTime()); builder.field(TransformIndexerStats.PROCESSING_TOTAL.getPreferredName(), stats.getProcessingTotal()); + builder.field(TransformIndexerStats.DELETE_TIME_IN_MS.getPreferredName(), stats.getDeleteTime()); builder.field(TransformIndexerStats.SEARCH_FAILURES.getPreferredName(), stats.getSearchFailures()); builder.field( TransformIndexerStats.EXPONENTIAL_AVG_CHECKPOINT_DURATION_MS.getPreferredName(), @@ -75,6 +79,7 @@ public static void toXContent(TransformIndexerStats stats, XContentBuilder build xContentFieldIfNotZero(builder, TransformIndexerStats.PAGES_PROCESSED.getPreferredName(), stats.getPagesProcessed()); xContentFieldIfNotZero(builder, TransformIndexerStats.DOCUMENTS_PROCESSED.getPreferredName(), stats.getDocumentsProcessed()); xContentFieldIfNotZero(builder, TransformIndexerStats.DOCUMENTS_INDEXED.getPreferredName(), stats.getDocumentsIndexed()); + xContentFieldIfNotZero(builder, TransformIndexerStats.DOCUMENTS_DELETED.getPreferredName(), stats.getDocumentsDeleted()); xContentFieldIfNotZero(builder, TransformIndexerStats.TRIGGER_COUNT.getPreferredName(), stats.getTriggerCount()); xContentFieldIfNotZero(builder, TransformIndexerStats.INDEX_TIME_IN_MS.getPreferredName(), stats.getIndexTime()); xContentFieldIfNotZero(builder, TransformIndexerStats.INDEX_TOTAL.getPreferredName(), stats.getIndexTotal()); @@ -83,6 +88,7 @@ public static void toXContent(TransformIndexerStats stats, XContentBuilder build xContentFieldIfNotZero(builder, TransformIndexerStats.SEARCH_TOTAL.getPreferredName(), stats.getSearchTotal()); xContentFieldIfNotZero(builder, TransformIndexerStats.PROCESSING_TIME_IN_MS.getPreferredName(), stats.getProcessingTime()); xContentFieldIfNotZero(builder, TransformIndexerStats.PROCESSING_TOTAL.getPreferredName(), stats.getProcessingTotal()); + xContentFieldIfNotZero(builder, TransformIndexerStats.DELETE_TIME_IN_MS.getPreferredName(), stats.getDeleteTime()); xContentFieldIfNotZero(builder, TransformIndexerStats.SEARCH_FAILURES.getPreferredName(), stats.getSearchFailures()); xContentFieldIfNotZero( builder, diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/hlrc/TimeRetentionPolicyConfigTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/hlrc/TimeRetentionPolicyConfigTests.java new file mode 100644 index 0000000000000..6b12921881d71 --- /dev/null +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/hlrc/TimeRetentionPolicyConfigTests.java @@ -0,0 +1,58 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.client.transform.transforms.hlrc; + +import org.elasticsearch.client.AbstractResponseTestCase; +import org.elasticsearch.client.transform.transforms.TimeRetentionPolicyConfig; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; + +import java.io.IOException; + +public class TimeRetentionPolicyConfigTests extends AbstractResponseTestCase< + org.elasticsearch.xpack.core.transform.transforms.TimeRetentionPolicyConfig, + TimeRetentionPolicyConfig> { + + public static org.elasticsearch.xpack.core.transform.transforms.TimeRetentionPolicyConfig randomTimeSyncConfig() { + return new org.elasticsearch.xpack.core.transform.transforms.TimeRetentionPolicyConfig( + randomAlphaOfLengthBetween(1, 10), + new TimeValue(randomNonNegativeLong()) + ); + } + + public static void assertHlrcEquals( + org.elasticsearch.xpack.core.transform.transforms.TimeRetentionPolicyConfig serverTestInstance, + TimeRetentionPolicyConfig clientInstance + ) { + assertEquals(serverTestInstance.getField(), clientInstance.getField()); + assertEquals(serverTestInstance.getMaxAge(), clientInstance.getMaxAge()); + } + + @Override + protected org.elasticsearch.xpack.core.transform.transforms.TimeRetentionPolicyConfig createServerTestInstance( + XContentType xContentType + ) { + return randomTimeSyncConfig(); + } + + @Override + protected TimeRetentionPolicyConfig doParseToClientInstance(XContentParser parser) throws IOException { + return TimeRetentionPolicyConfig.fromXContent(parser); + } + + @Override + protected void assertInstances( + org.elasticsearch.xpack.core.transform.transforms.TimeRetentionPolicyConfig serverTestInstance, + TimeRetentionPolicyConfig clientInstance + ) { + assertHlrcEquals(serverTestInstance, clientInstance); + } + +} diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/hlrc/TransformIndexerStatsTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/hlrc/TransformIndexerStatsTests.java index 6faaa3d2a3510..92a828c63d114 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/hlrc/TransformIndexerStatsTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/hlrc/TransformIndexerStatsTests.java @@ -35,6 +35,8 @@ public static org.elasticsearch.xpack.core.transform.transforms.TransformIndexer randomLongBetween(0L, 10000L), randomLongBetween(0L, 10000L), randomLongBetween(0L, 10000L), + randomLongBetween(0L, 10000L), + randomLongBetween(0L, 10000L), randomDouble(), randomDouble(), randomDouble() diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/hlrc/TransformStatsTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/hlrc/TransformStatsTests.java index 103dd6790d6dd..b989b68aba13b 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/hlrc/TransformStatsTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/transforms/hlrc/TransformStatsTests.java @@ -156,8 +156,10 @@ public static void assertTransformIndexerStats( assertThat(serverTestInstance.getNumDocuments(), equalTo(clientInstance.getDocumentsProcessed())); assertThat(serverTestInstance.getNumInvocations(), equalTo(clientInstance.getTriggerCount())); assertThat(serverTestInstance.getOutputDocuments(), equalTo(clientInstance.getDocumentsIndexed())); + assertThat(serverTestInstance.getNumDeletedDocuments(), equalTo(clientInstance.getDocumentsDeleted())); assertThat(serverTestInstance.getSearchFailures(), equalTo(clientInstance.getSearchFailures())); assertThat(serverTestInstance.getSearchTime(), equalTo(clientInstance.getSearchTime())); assertThat(serverTestInstance.getSearchTotal(), equalTo(clientInstance.getSearchTotal())); + assertThat(serverTestInstance.getDeleteTime(), equalTo(clientInstance.getDeleteTime())); } } diff --git a/docs/reference/cat/transforms.asciidoc b/docs/reference/cat/transforms.asciidoc index 20752761ca13d..6d4f8c5129656 100644 --- a/docs/reference/cat/transforms.asciidoc +++ b/docs/reference/cat/transforms.asciidoc @@ -74,12 +74,18 @@ The progress of the next checkpoint that is currently in progress. `create_time`, `ct`, `createTime`::: The time the {transform} was created. +`delete_time`, `dtime`::: +include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=delete-time-ms] + `description`, `d`::: The description of the {transform}. `dest_index`, `di`, `destIndex`::: include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=dest-index] +`documents_deleted`, `docd`::: +include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=docs-deleted-transform] + `documents_indexed`, `doci`::: include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=docs-indexed] diff --git a/docs/reference/rest-api/common-parms.asciidoc b/docs/reference/rest-api/common-parms.asciidoc index 4f5096d1bc0c9..030f49288a9ad 100644 --- a/docs/reference/rest-api/common-parms.asciidoc +++ b/docs/reference/rest-api/common-parms.asciidoc @@ -142,6 +142,10 @@ Comma-separated list or wildcard expressions of fields to include in `fielddata` and `suggest` statistics. end::completion-fields[] +tag::delete-time-ms[] +The amount of time spent deleting, in milliseconds. +end::delete-time-ms[] + tag::default_operator[] `default_operator`:: (Optional, string) The default operator for query string query: AND or OR. @@ -195,6 +199,11 @@ Also, {es} creates extra deleted documents to internally track the recent history of operations on a shard. end::docs-deleted[] +tag::docs-deleted-transform[] +The number of documents that have been deleted from the destination index +for the {transform}. +end::docs-deleted-transform[] + tag::docs-indexed[] The number of documents that have been indexed into the destination index for the {transform}. diff --git a/docs/reference/transform/apis/get-transform-stats.asciidoc b/docs/reference/transform/apis/get-transform-stats.asciidoc index 96222a0cc6043..da49ce694e22d 100644 --- a/docs/reference/transform/apis/get-transform-stats.asciidoc +++ b/docs/reference/transform/apis/get-transform-stats.asciidoc @@ -192,6 +192,14 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=state-transform] [%collapsible%open] ==== +`delete_time_in_ms`::: +(long) +include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=delete-time-ms] + +`documents_deleted`::: +(long) +include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=docs-deleted-transform] + `documents_indexed`::: (long) include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=docs-indexed] diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/RefreshResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/RefreshResponse.java index e85f08136587d..4c162fa14ef24 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/RefreshResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/refresh/RefreshResponse.java @@ -38,7 +38,12 @@ public class RefreshResponse extends BroadcastResponse { super(in); } - RefreshResponse(int totalShards, int successfulShards, int failedShards, List shardFailures) { + public RefreshResponse( + int totalShards, + int successfulShards, + int failedShards, + List shardFailures + ) { super(totalShards, successfulShards, failedShards, shardFailures); } diff --git a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchResponseTests.java b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchResponseTests.java index 044820d4bb158..8e875c7aa4d39 100644 --- a/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchResponseTests.java +++ b/x-pack/plugin/async-search/src/test/java/org/elasticsearch/xpack/search/AsyncSearchResponseTests.java @@ -14,7 +14,6 @@ import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentElasticsearchExtension; @@ -24,10 +23,6 @@ import org.elasticsearch.search.internal.InternalSearchResponse; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.core.search.action.AsyncSearchResponse; -import org.elasticsearch.xpack.core.transform.TransformField; -import org.elasticsearch.xpack.core.transform.TransformNamedXContentProvider; -import org.elasticsearch.xpack.core.transform.transforms.SyncConfig; -import org.elasticsearch.xpack.core.transform.transforms.TimeSyncConfig; import org.junit.Before; import java.io.IOException; @@ -48,12 +43,6 @@ public void registerNamedObjects() { SearchModule searchModule = new SearchModule(Settings.EMPTY, emptyList()); List namedWriteables = searchModule.getNamedWriteables(); - namedWriteables.add(new NamedWriteableRegistry.Entry(SyncConfig.class, TransformField.TIME_BASED_SYNC.getPreferredName(), - TimeSyncConfig::new)); - - List namedXContents = searchModule.getNamedXContents(); - namedXContents.addAll(new TransformNamedXContentProvider().getNamedXContentParsers()); - namedWriteableRegistry = new NamedWriteableRegistry(namedWriteables); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java index 5965162676d02..d8f43e33029bb 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java @@ -93,7 +93,6 @@ import org.elasticsearch.xpack.core.ml.action.ExplainDataFrameAnalyticsAction; import org.elasticsearch.xpack.core.ml.action.FinalizeJobExecutionAction; import org.elasticsearch.xpack.core.rollup.action.RollupIndexerAction; -import org.elasticsearch.xpack.core.textstructure.action.FindStructureAction; import org.elasticsearch.xpack.core.ml.action.FlushJobAction; import org.elasticsearch.xpack.core.ml.action.ForecastJobAction; import org.elasticsearch.xpack.core.ml.action.GetBucketsAction; @@ -153,12 +152,12 @@ import org.elasticsearch.xpack.core.rollup.action.GetRollupCapsAction; import org.elasticsearch.xpack.core.rollup.action.GetRollupJobsAction; import org.elasticsearch.xpack.core.rollup.action.PutRollupJobAction; +import org.elasticsearch.xpack.core.rollup.action.RollupAction; import org.elasticsearch.xpack.core.rollup.action.RollupSearchAction; import org.elasticsearch.xpack.core.rollup.action.StartRollupJobAction; import org.elasticsearch.xpack.core.rollup.action.StopRollupJobAction; import org.elasticsearch.xpack.core.rollup.job.RollupJob; import org.elasticsearch.xpack.core.rollup.job.RollupJobStatus; -import org.elasticsearch.xpack.core.rollup.action.RollupAction; import org.elasticsearch.xpack.core.runtimefields.RuntimeFieldsFeatureSetUsage; import org.elasticsearch.xpack.core.search.action.GetAsyncSearchAction; import org.elasticsearch.xpack.core.search.action.SubmitAsyncSearchAction; @@ -203,6 +202,7 @@ import org.elasticsearch.xpack.core.spatial.SpatialFeatureSetUsage; import org.elasticsearch.xpack.core.sql.SqlFeatureSetUsage; import org.elasticsearch.xpack.core.ssl.action.GetCertificateInfoAction; +import org.elasticsearch.xpack.core.textstructure.action.FindStructureAction; import org.elasticsearch.xpack.core.transform.TransformFeatureSetUsage; import org.elasticsearch.xpack.core.transform.TransformField; import org.elasticsearch.xpack.core.transform.action.DeleteTransformAction; @@ -212,7 +212,9 @@ import org.elasticsearch.xpack.core.transform.action.PutTransformAction; import org.elasticsearch.xpack.core.transform.action.StartTransformAction; import org.elasticsearch.xpack.core.transform.action.StopTransformAction; +import org.elasticsearch.xpack.core.transform.transforms.RetentionPolicyConfig; import org.elasticsearch.xpack.core.transform.transforms.SyncConfig; +import org.elasticsearch.xpack.core.transform.transforms.TimeRetentionPolicyConfig; import org.elasticsearch.xpack.core.transform.transforms.TimeSyncConfig; import org.elasticsearch.xpack.core.transform.transforms.TransformState; import org.elasticsearch.xpack.core.transform.transforms.TransformTaskParams; @@ -504,7 +506,12 @@ public List getNamedWriteables() { new NamedWriteableRegistry.Entry(PersistentTaskParams.class, TransformField.TASK_NAME, TransformTaskParams::new), new NamedWriteableRegistry.Entry(Task.Status.class, TransformField.TASK_NAME, TransformState::new), new NamedWriteableRegistry.Entry(PersistentTaskState.class, TransformField.TASK_NAME, TransformState::new), - new NamedWriteableRegistry.Entry(SyncConfig.class, TransformField.TIME_BASED_SYNC.getPreferredName(), TimeSyncConfig::new), + new NamedWriteableRegistry.Entry(SyncConfig.class, TransformField.TIME.getPreferredName(), TimeSyncConfig::new), + new NamedWriteableRegistry.Entry( + RetentionPolicyConfig.class, + TransformField.TIME.getPreferredName(), + TimeRetentionPolicyConfig::new + ), // Flattened for backward compatibility with 7.x new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.FLATTENED, FlattenedFeatureSetUsage::new), // Vectors diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformField.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformField.java index f5cf48ead7859..2870bba177b65 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformField.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformField.java @@ -39,9 +39,11 @@ public final class TransformField { public static final ParseField DATES_AS_EPOCH_MILLIS = new ParseField("dates_as_epoch_millis"); public static final ParseField FIELD = new ParseField("field"); public static final ParseField SYNC = new ParseField("sync"); - public static final ParseField TIME_BASED_SYNC = new ParseField("time"); + public static final ParseField TIME = new ParseField("time"); public static final ParseField DELAY = new ParseField("delay"); public static final ParseField DEFER_VALIDATION = new ParseField("defer_validation"); + public static final ParseField RETENTION_POLICY = new ParseField("retention_policy"); + public static final ParseField MAX_AGE = new ParseField("max_age"); public static final ParseField ALLOW_NO_MATCH = new ParseField("allow_no_match"); /** diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformNamedXContentProvider.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformNamedXContentProvider.java index 35a6bbdf15614..447575c4de9f4 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformNamedXContentProvider.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformNamedXContentProvider.java @@ -9,7 +9,9 @@ import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.plugins.spi.NamedXContentProvider; +import org.elasticsearch.xpack.core.transform.transforms.RetentionPolicyConfig; import org.elasticsearch.xpack.core.transform.transforms.SyncConfig; +import org.elasticsearch.xpack.core.transform.transforms.TimeRetentionPolicyConfig; import org.elasticsearch.xpack.core.transform.transforms.TimeSyncConfig; import java.util.Arrays; @@ -20,8 +22,8 @@ public class TransformNamedXContentProvider implements NamedXContentProvider { @Override public List getNamedXContentParsers() { return Arrays.asList( - new NamedXContentRegistry.Entry(SyncConfig.class, - TransformField.TIME_BASED_SYNC, - TimeSyncConfig::parse)); + new NamedXContentRegistry.Entry(SyncConfig.class, TransformField.TIME, TimeSyncConfig::parse), + new NamedXContentRegistry.Entry(RetentionPolicyConfig.class, TransformField.TIME, TimeRetentionPolicyConfig::parse) + ); } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/RetentionPolicyConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/RetentionPolicyConfig.java new file mode 100644 index 0000000000000..2fb1e340a8041 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/RetentionPolicyConfig.java @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.core.transform.transforms; + +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.common.io.stream.NamedWriteable; +import org.elasticsearch.common.xcontent.ToXContentObject; + +public interface RetentionPolicyConfig extends ToXContentObject, NamedWriteable { + ActionRequestValidationException validate(ActionRequestValidationException validationException); +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TimeRetentionPolicyConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TimeRetentionPolicyConfig.java new file mode 100644 index 0000000000000..f70881895b02c --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TimeRetentionPolicyConfig.java @@ -0,0 +1,136 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.core.transform.transforms; + +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.xpack.core.transform.TransformField; +import org.elasticsearch.xpack.core.transform.utils.ExceptionsHelper; + +import java.io.IOException; +import java.util.Objects; + +import static org.elasticsearch.action.ValidateActions.addValidationError; +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; + +public class TimeRetentionPolicyConfig implements RetentionPolicyConfig { + + private static final String NAME = "transform_retention_policy_time"; + private static long MIN_AGE_SECONDS = 60; + + private final String field; + private final TimeValue maxAge; + + private static final ConstructingObjectParser STRICT_PARSER = createParser(false); + private static final ConstructingObjectParser LENIENT_PARSER = createParser(true); + + private static ConstructingObjectParser createParser(boolean lenient) { + ConstructingObjectParser parser = new ConstructingObjectParser<>(NAME, lenient, args -> { + String field = (String) args[0]; + TimeValue maxAge = (TimeValue) args[1]; + return new TimeRetentionPolicyConfig(field, maxAge); + }); + parser.declareString(constructorArg(), TransformField.FIELD); + parser.declareField( + constructorArg(), + (p, c) -> TimeValue.parseTimeValue(p.text(), TransformField.MAX_AGE.getPreferredName()), + TransformField.MAX_AGE, + ObjectParser.ValueType.STRING + ); + return parser; + } + + public TimeRetentionPolicyConfig(final String field, final TimeValue maxAge) { + this.field = ExceptionsHelper.requireNonNull(field, TransformField.FIELD.getPreferredName()); + this.maxAge = ExceptionsHelper.requireNonNull(maxAge, TransformField.MAX_AGE.getPreferredName()); + } + + public TimeRetentionPolicyConfig(StreamInput in) throws IOException { + this.field = in.readString(); + this.maxAge = in.readTimeValue(); + } + + public String getField() { + return field; + } + + public TimeValue getMaxAge() { + return maxAge; + } + + @Override + public ActionRequestValidationException validate(ActionRequestValidationException validationException) { + if (maxAge.getSeconds() < MIN_AGE_SECONDS) { + validationException = addValidationError( + "retention_policy.time.max_age must be more than " + MIN_AGE_SECONDS + "s, found [" + maxAge + "]", + validationException + ); + } + return validationException; + } + + @Override + public void writeTo(final StreamOutput out) throws IOException { + out.writeString(field); + out.writeTimeValue(maxAge); + } + + @Override + public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException { + builder.startObject(); + builder.field(TransformField.FIELD.getPreferredName(), field); + builder.field(TransformField.MAX_AGE.getPreferredName(), maxAge.getStringRep()); + builder.endObject(); + return builder; + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + + if (other == null || getClass() != other.getClass()) { + return false; + } + + final TimeRetentionPolicyConfig that = (TimeRetentionPolicyConfig) other; + + return Objects.equals(this.field, that.field) && Objects.equals(this.maxAge, that.maxAge); + } + + @Override + public int hashCode() { + return Objects.hash(field, maxAge); + } + + @Override + public String toString() { + return Strings.toString(this, true, true); + } + + public static TimeRetentionPolicyConfig parse(final XContentParser parser) { + return LENIENT_PARSER.apply(parser, null); + } + + public static TimeRetentionPolicyConfig fromXContent(final XContentParser parser, boolean lenient) throws IOException { + return lenient ? LENIENT_PARSER.apply(parser, null) : STRICT_PARSER.apply(parser, null); + } + + @Override + public String getWriteableName() { + return TransformField.TIME.getPreferredName(); + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TimeSyncConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TimeSyncConfig.java index 8ab21573fe608..956018a243eb7 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TimeSyncConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TimeSyncConfig.java @@ -131,7 +131,7 @@ public static TimeSyncConfig fromXContent(final XContentParser parser, boolean l @Override public String getWriteableName() { - return TransformField.TIME_BASED_SYNC.getPreferredName(); + return TransformField.TIME.getPreferredName(); } @Override diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java index e56a107465235..2187d2537864a 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java @@ -22,7 +22,6 @@ import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.common.xcontent.XContentParserUtils; import org.elasticsearch.xpack.core.common.time.TimeUtils; import org.elasticsearch.xpack.core.transform.TransformField; import org.elasticsearch.xpack.core.transform.TransformMessages; @@ -61,6 +60,7 @@ public class TransformConfig extends AbstractDiffable implement private final TimeValue frequency; private final SyncConfig syncConfig; private final SettingsConfig settings; + private final RetentionPolicyConfig retentionPolicyConfig; private final String description; // headers store the user context from the creating user, which allows us to run the transform as this user // the header only contains name, groups and other context but no authorization keys @@ -103,8 +103,8 @@ private static ConstructingObjectParser createParser(bo if (lenient == false) { // on strict parsing do not allow injection of headers, transform version, or create time validateStrictParsingParams(args[6], HEADERS.getPreferredName()); - validateStrictParsingParams(args[11], TransformField.CREATE_TIME.getPreferredName()); - validateStrictParsingParams(args[12], TransformField.VERSION.getPreferredName()); + validateStrictParsingParams(args[12], TransformField.CREATE_TIME.getPreferredName()); + validateStrictParsingParams(args[13], TransformField.VERSION.getPreferredName()); // exactly one function must be defined if ((args[7] == null) == (args[8] == null)) { throw new IllegalArgumentException(TransformMessages.TRANSFORM_CONFIGURATION_BAD_FUNCTION_COUNT); @@ -118,6 +118,8 @@ private static ConstructingObjectParser createParser(bo LatestConfig latestConfig = (LatestConfig) args[8]; String description = (String) args[9]; SettingsConfig settings = (SettingsConfig) args[10]; + RetentionPolicyConfig retentionPolicyConfig = (RetentionPolicyConfig) args[11]; + return new TransformConfig( id, source, @@ -129,8 +131,9 @@ private static ConstructingObjectParser createParser(bo latestConfig, description, settings, - (Instant) args[11], - (String) args[12] + retentionPolicyConfig, + (Instant) args[12], + (String) args[13] ); }); @@ -138,13 +141,18 @@ private static ConstructingObjectParser createParser(bo parser.declareObject(constructorArg(), (p, c) -> SourceConfig.fromXContent(p, lenient), TransformField.SOURCE); parser.declareObject(constructorArg(), (p, c) -> DestConfig.fromXContent(p, lenient), TransformField.DESTINATION); parser.declareString(optionalConstructorArg(), TransformField.FREQUENCY); - parser.declareObject(optionalConstructorArg(), (p, c) -> parseSyncConfig(p, lenient), TransformField.SYNC); + parser.declareNamedObject(optionalConstructorArg(), (p, c, n) -> p.namedObject(SyncConfig.class, n, c), TransformField.SYNC); parser.declareString(optionalConstructorArg(), TransformField.INDEX_DOC_TYPE); parser.declareObject(optionalConstructorArg(), (p, c) -> p.mapStrings(), HEADERS); parser.declareObject(optionalConstructorArg(), (p, c) -> PivotConfig.fromXContent(p, lenient), PIVOT_TRANSFORM); parser.declareObject(optionalConstructorArg(), (p, c) -> LatestConfig.fromXContent(p, lenient), LATEST_TRANSFORM); parser.declareString(optionalConstructorArg(), TransformField.DESCRIPTION); parser.declareObject(optionalConstructorArg(), (p, c) -> SettingsConfig.fromXContent(p, lenient), TransformField.SETTINGS); + parser.declareNamedObject( + optionalConstructorArg(), + (p, c, n) -> p.namedObject(RetentionPolicyConfig.class, n, c), + TransformField.RETENTION_POLICY + ); parser.declareField( optionalConstructorArg(), p -> TimeUtils.parseTimeFieldToInstant(p, TransformField.CREATE_TIME.getPreferredName()), @@ -155,14 +163,6 @@ private static ConstructingObjectParser createParser(bo return parser; } - private static SyncConfig parseSyncConfig(XContentParser parser, boolean ignoreUnknownFields) throws IOException { - XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser); - XContentParserUtils.ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.nextToken(), parser); - SyncConfig syncConfig = parser.namedObject(SyncConfig.class, parser.currentName(), ignoreUnknownFields); - XContentParserUtils.ensureExpectedToken(XContentParser.Token.END_OBJECT, parser.nextToken(), parser); - return syncConfig; - } - public static String documentId(String transformId) { return NAME + "-" + transformId; } @@ -178,6 +178,7 @@ public TransformConfig( final LatestConfig latestConfig, final String description, final SettingsConfig settings, + final RetentionPolicyConfig retentionPolicyConfig, final Instant createTime, final String version ) { @@ -191,6 +192,7 @@ public TransformConfig( this.latestConfig = latestConfig; this.description = description; this.settings = settings == null ? new SettingsConfig() : settings; + this.retentionPolicyConfig = retentionPolicyConfig; if (this.description != null && this.description.length() > MAX_DESCRIPTION_LENGTH) { throw new IllegalArgumentException("[description] must be less than 1000 characters in length."); } @@ -225,6 +227,11 @@ public TransformConfig(final StreamInput in) throws IOException { } else { settings = new SettingsConfig(); } + if (in.getVersion().onOrAfter(Version.V_8_0_0)) { // todo: V_7_12_0 + retentionPolicyConfig = in.readOptionalNamedWriteable(RetentionPolicyConfig.class); + } else { + retentionPolicyConfig = null; + } } public String getId() { @@ -292,6 +299,11 @@ public SettingsConfig getSettings() { return settings; } + @Nullable + public RetentionPolicyConfig getRetentionPolicyConfig() { + return retentionPolicyConfig; + } + public ActionRequestValidationException validate(ActionRequestValidationException validationException) { if (pivotConfig != null) { validationException = pivotConfig.validate(validationException); @@ -300,10 +312,16 @@ public ActionRequestValidationException validate(ActionRequestValidationExceptio validationException = latestConfig.validate(validationException); } validationException = settings.validate(validationException); + + if (retentionPolicyConfig != null) { + validationException = retentionPolicyConfig.validate(validationException); + } + return validationException; } public boolean isValid() { + // todo: base this on validate if (pivotConfig != null && pivotConfig.isValid() == false) { return false; } @@ -316,6 +334,10 @@ public boolean isValid() { return false; } + if (retentionPolicyConfig != null && retentionPolicyConfig.validate(null) != null) { + return false; + } + return settings.isValid() && source.isValid() && dest.isValid(); } @@ -344,14 +366,17 @@ public void writeTo(final StreamOutput out) throws IOException { if (out.getVersion().onOrAfter(Version.V_7_8_0)) { settings.writeTo(out); } + if (out.getVersion().onOrAfter(Version.V_8_0_0)) { // todo: V_7_12_0 + out.writeOptionalNamedWriteable(retentionPolicyConfig); + } } @Override public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException { final boolean excludeGenerated = params.paramAsBoolean(TransformField.EXCLUDE_GENERATED, false); final boolean forInternalStorage = params.paramAsBoolean(TransformField.FOR_INTERNAL_STORAGE, false); - assert (forInternalStorage - && excludeGenerated) == false : "unsupported behavior, exclude_generated is true and for_internal_storage is true"; + assert (forInternalStorage && excludeGenerated) == false + : "unsupported behavior, exclude_generated is true and for_internal_storage is true"; builder.startObject(); builder.field(TransformField.ID.getPreferredName(), id); if (excludeGenerated == false) { @@ -392,6 +417,11 @@ public XContentBuilder toXContent(final XContentBuilder builder, final Params pa builder.field(TransformField.DESCRIPTION.getPreferredName(), description); } builder.field(TransformField.SETTINGS.getPreferredName(), settings); + if (retentionPolicyConfig != null) { + builder.startObject(TransformField.RETENTION_POLICY.getPreferredName()); + builder.field(retentionPolicyConfig.getWriteableName(), retentionPolicyConfig); + builder.endObject(); + } builder.endObject(); return builder; } @@ -418,6 +448,7 @@ public boolean equals(Object other) { && Objects.equals(this.latestConfig, that.latestConfig) && Objects.equals(this.description, that.description) && Objects.equals(this.settings, that.settings) + && Objects.equals(this.retentionPolicyConfig, that.retentionPolicyConfig) && Objects.equals(this.createTime, that.createTime) && Objects.equals(this.transformVersion, that.transformVersion); } @@ -435,6 +466,7 @@ public int hashCode() { latestConfig, description, settings, + retentionPolicyConfig, createTime, transformVersion ); @@ -446,7 +478,6 @@ public String toString() { } public static TransformConfig fromXContent(final XContentParser parser, @Nullable final String optionalTransformId, boolean lenient) { - return lenient ? LENIENT_PARSER.apply(parser, optionalTransformId) : STRICT_PARSER.apply(parser, optionalTransformId); } @@ -524,6 +555,7 @@ public static class Builder { private PivotConfig pivotConfig; private LatestConfig latestConfig; private SettingsConfig settings; + private RetentionPolicyConfig retentionPolicyConfig; public Builder() {} @@ -539,6 +571,7 @@ public Builder(TransformConfig config) { this.pivotConfig = config.pivotConfig; this.latestConfig = config.latestConfig; this.settings = config.settings; + this.retentionPolicyConfig = config.retentionPolicyConfig; } public Builder setId(String id) { @@ -640,6 +673,11 @@ Version getVersion() { return transformVersion; } + public Builder setRetentionPolicyConfig(RetentionPolicyConfig retentionPolicyConfig) { + this.retentionPolicyConfig = retentionPolicyConfig; + return this; + } + public TransformConfig build() { return new TransformConfig( id, @@ -652,6 +690,7 @@ public TransformConfig build() { latestConfig, description, settings, + retentionPolicyConfig, createTime, transformVersion == null ? null : transformVersion.toString() ); @@ -679,6 +718,7 @@ public boolean equals(Object other) { && Objects.equals(this.latestConfig, that.latestConfig) && Objects.equals(this.description, that.description) && Objects.equals(this.settings, that.settings) + && Objects.equals(this.retentionPolicyConfig, that.retentionPolicyConfig) && Objects.equals(this.createTime, that.createTime) && Objects.equals(this.transformVersion, that.transformVersion); } @@ -696,6 +736,7 @@ public int hashCode() { latestConfig, description, settings, + retentionPolicyConfig, createTime, transformVersion ); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigUpdate.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigUpdate.java index acd293c52ee8b..0ab6e9cd1f2f9 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigUpdate.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigUpdate.java @@ -47,7 +47,8 @@ public class TransformConfigUpdate implements Writeable { SyncConfig syncConfig = (SyncConfig) args[3]; String description = (String) args[4]; SettingsConfig settings = (SettingsConfig) args[5]; - return new TransformConfigUpdate(source, dest, frequency, syncConfig, description, settings); + RetentionPolicyConfig retentionPolicyConfig = (RetentionPolicyConfig) args[6]; + return new TransformConfigUpdate(source, dest, frequency, syncConfig, description, settings, retentionPolicyConfig); } ); @@ -58,6 +59,7 @@ public class TransformConfigUpdate implements Writeable { PARSER.declareObject(optionalConstructorArg(), (p, c) -> parseSyncConfig(p), TransformField.SYNC); PARSER.declareString(optionalConstructorArg(), TransformField.DESCRIPTION); PARSER.declareObject(optionalConstructorArg(), (p, c) -> SettingsConfig.fromXContent(p, false), TransformField.SETTINGS); + PARSER.declareObject(optionalConstructorArg(), (p, c) -> parseRetentionPolicyConfig(p), TransformField.RETENTION_POLICY); } private static SyncConfig parseSyncConfig(XContentParser parser) throws IOException { @@ -68,12 +70,21 @@ private static SyncConfig parseSyncConfig(XContentParser parser) throws IOExcept return syncConfig; } + private static RetentionPolicyConfig parseRetentionPolicyConfig(XContentParser parser) throws IOException { + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser); + XContentParserUtils.ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.nextToken(), parser); + RetentionPolicyConfig retentionPolicyConfig = parser.namedObject(RetentionPolicyConfig.class, parser.currentName(), true); + XContentParserUtils.ensureExpectedToken(XContentParser.Token.END_OBJECT, parser.nextToken(), parser); + return retentionPolicyConfig; + } + private final SourceConfig source; private final DestConfig dest; private final TimeValue frequency; private final SyncConfig syncConfig; private final String description; private final SettingsConfig settings; + private final RetentionPolicyConfig retentionPolicyConfig; private Map headers; public TransformConfigUpdate( @@ -82,7 +93,8 @@ public TransformConfigUpdate( final TimeValue frequency, final SyncConfig syncConfig, final String description, - final SettingsConfig settings + final SettingsConfig settings, + final RetentionPolicyConfig retentionPolicyConfig ) { this.source = source; this.dest = dest; @@ -93,6 +105,7 @@ public TransformConfigUpdate( throw new IllegalArgumentException("[description] must be less than 1000 characters in length."); } this.settings = settings; + this.retentionPolicyConfig = retentionPolicyConfig; } public TransformConfigUpdate(final StreamInput in) throws IOException { @@ -109,7 +122,11 @@ public TransformConfigUpdate(final StreamInput in) throws IOException { } else { settings = null; } - + if (in.getVersion().onOrAfter(Version.V_8_0_0)) { // todo: V_7_12_0 + retentionPolicyConfig = in.readOptionalNamedWriteable(RetentionPolicyConfig.class); + } else { + retentionPolicyConfig = null; + } } public SourceConfig getSource() { @@ -138,6 +155,11 @@ public SettingsConfig getSettings() { return settings; } + @Nullable + public RetentionPolicyConfig getRetentionPolicyConfig() { + return retentionPolicyConfig; + } + public Map getHeaders() { return headers; } @@ -162,6 +184,9 @@ public void writeTo(final StreamOutput out) throws IOException { if (out.getVersion().onOrAfter(Version.V_7_8_0)) { out.writeOptionalWriteable(settings); } + if (out.getVersion().onOrAfter(Version.V_8_0_0)) { // todo: V_7_12_0 + out.writeOptionalNamedWriteable(retentionPolicyConfig); + } } @Override @@ -182,12 +207,13 @@ public boolean equals(Object other) { && Objects.equals(this.syncConfig, that.syncConfig) && Objects.equals(this.description, that.description) && Objects.equals(this.settings, that.settings) + && Objects.equals(this.retentionPolicyConfig, that.retentionPolicyConfig) && Objects.equals(this.headers, that.headers); } @Override public int hashCode() { - return Objects.hash(source, dest, frequency, syncConfig, description, settings, headers); + return Objects.hash(source, dest, frequency, syncConfig, description, settings, retentionPolicyConfig, headers); } public static TransformConfigUpdate fromXContent(final XContentParser parser) { @@ -201,6 +227,7 @@ && isNullOrEqual(frequency, config.getFrequency()) && isNullOrEqual(syncConfig, config.getSyncConfig()) && isNullOrEqual(description, config.getDescription()) && isNullOrEqual(settings, config.getSettings()) + && isNullOrEqual(retentionPolicyConfig, config.getRetentionPolicyConfig()) && isNullOrEqual(headers, config.getHeaders()); } @@ -253,6 +280,10 @@ public TransformConfig apply(TransformConfig config) { settingsBuilder.update(settings); builder.setSettings(settingsBuilder.build()); } + if (retentionPolicyConfig != null) { + builder.setRetentionPolicyConfig(retentionPolicyConfig); + } + builder.setVersion(Version.CURRENT); return builder.build(); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformIndexerStats.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformIndexerStats.java index 1f1d34a72dc91..2151ebb682aff 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformIndexerStats.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformIndexerStats.java @@ -30,10 +30,12 @@ public class TransformIndexerStats extends IndexerJobStats { public static ParseField NUM_PAGES = new ParseField("pages_processed"); public static ParseField NUM_INPUT_DOCUMENTS = new ParseField("documents_processed"); public static ParseField NUM_OUTPUT_DOCUMENTS = new ParseField("documents_indexed"); + public static ParseField NUM_DELETED_DOCUMENTS = new ParseField("documents_deleted"); public static ParseField NUM_INVOCATIONS = new ParseField("trigger_count"); public static ParseField INDEX_TIME_IN_MS = new ParseField("index_time_in_ms"); public static ParseField SEARCH_TIME_IN_MS = new ParseField("search_time_in_ms"); public static ParseField PROCESSING_TIME_IN_MS = new ParseField("processing_time_in_ms"); + public static ParseField DELETE_TIME_IN_MS = new ParseField("delete_time_in_ms"); public static ParseField INDEX_TOTAL = new ParseField("index_total"); public static ParseField SEARCH_TOTAL = new ParseField("search_total"); public static ParseField PROCESSING_TOTAL = new ParseField("processing_total"); @@ -64,9 +66,11 @@ public class TransformIndexerStats extends IndexerJobStats { unboxSafe(args[9], 0L), unboxSafe(args[10], 0L), unboxSafe(args[11], 0L), - unboxSafe(args[12], 0.0), - unboxSafe(args[13], 0.0), - unboxSafe(args[14], 0.0) + unboxSafe(args[12], 0L), + unboxSafe(args[13], 0L), + unboxSafe(args[14], 0.0), + unboxSafe(args[15], 0.0), + unboxSafe(args[16], 0.0) ) ); @@ -74,10 +78,12 @@ public class TransformIndexerStats extends IndexerJobStats { LENIENT_PARSER.declareLong(optionalConstructorArg(), NUM_PAGES); LENIENT_PARSER.declareLong(optionalConstructorArg(), NUM_INPUT_DOCUMENTS); LENIENT_PARSER.declareLong(optionalConstructorArg(), NUM_OUTPUT_DOCUMENTS); + LENIENT_PARSER.declareLong(optionalConstructorArg(), NUM_DELETED_DOCUMENTS); LENIENT_PARSER.declareLong(optionalConstructorArg(), NUM_INVOCATIONS); LENIENT_PARSER.declareLong(optionalConstructorArg(), INDEX_TIME_IN_MS); LENIENT_PARSER.declareLong(optionalConstructorArg(), SEARCH_TIME_IN_MS); LENIENT_PARSER.declareLong(optionalConstructorArg(), PROCESSING_TIME_IN_MS); + LENIENT_PARSER.declareLong(optionalConstructorArg(), DELETE_TIME_IN_MS); LENIENT_PARSER.declareLong(optionalConstructorArg(), INDEX_TOTAL); LENIENT_PARSER.declareLong(optionalConstructorArg(), SEARCH_TOTAL); LENIENT_PARSER.declareLong(optionalConstructorArg(), PROCESSING_TOTAL); @@ -88,6 +94,11 @@ public class TransformIndexerStats extends IndexerJobStats { LENIENT_PARSER.declareDouble(optionalConstructorArg(), EXPONENTIAL_AVG_DOCUMENTS_PROCESSED); } + private long deleteTime; + private long numDeletedDocuments; + + private long startDeleteTime; + private double expAvgCheckpointDurationMs; private double expAvgDocumentsIndexed; private double expAvgDocumentsProcessed; @@ -103,10 +114,12 @@ public TransformIndexerStats( long numPages, long numInputDocuments, long numOutputDocuments, + long numDeletedDocuments, long numInvocations, long indexTime, long searchTime, long processingTime, + long deleteTime, long indexTotal, long searchTotal, long processingTotal, @@ -130,6 +143,8 @@ public TransformIndexerStats( indexFailures, searchFailures ); + this.numDeletedDocuments = numDeletedDocuments; + this.deleteTime = deleteTime; this.expAvgCheckpointDurationMs = expAvgCheckpointDurationMs; this.expAvgDocumentsIndexed = expAvgDocumentsIndexed; this.expAvgDocumentsProcessed = expAvgDocumentsProcessed; @@ -140,10 +155,12 @@ public TransformIndexerStats(TransformIndexerStats other) { other.numPages, other.numInputDocuments, other.numOuputDocuments, + other.numDeletedDocuments, other.numInvocations, other.indexTime, other.searchTime, other.processingTime, + other.deleteTime, other.indexTotal, other.searchTotal, other.processingTotal, @@ -165,6 +182,10 @@ public TransformIndexerStats(StreamInput in) throws IOException { this.expAvgDocumentsIndexed = in.readDouble(); this.expAvgDocumentsProcessed = in.readDouble(); } + if (in.getVersion().onOrAfter(Version.V_8_0_0)) { // todo: V_7_12_0 + this.numDeletedDocuments = in.readVLong(); + this.deleteTime = in.readVLong(); + } } @Override @@ -178,6 +199,10 @@ public void writeTo(StreamOutput out) throws IOException { out.writeDouble(this.expAvgDocumentsIndexed); out.writeDouble(this.expAvgDocumentsProcessed); } + if (out.getVersion().onOrAfter(Version.V_8_0_0)) { // todo: V_7_12_0 + out.writeVLong(numDeletedDocuments); + out.writeVLong(deleteTime); + } } @Override @@ -186,6 +211,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(NUM_PAGES.getPreferredName(), numPages); builder.field(NUM_INPUT_DOCUMENTS.getPreferredName(), numInputDocuments); builder.field(NUM_OUTPUT_DOCUMENTS.getPreferredName(), numOuputDocuments); + builder.field(NUM_DELETED_DOCUMENTS.getPreferredName(), numDeletedDocuments); builder.field(NUM_INVOCATIONS.getPreferredName(), numInvocations); builder.field(INDEX_TIME_IN_MS.getPreferredName(), indexTime); builder.field(INDEX_TOTAL.getPreferredName(), indexTotal); @@ -195,6 +221,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(SEARCH_FAILURES.getPreferredName(), searchFailures); builder.field(PROCESSING_TIME_IN_MS.getPreferredName(), processingTime); builder.field(PROCESSING_TOTAL.getPreferredName(), processingTotal); + builder.field(DELETE_TIME_IN_MS.getPreferredName(), deleteTime); builder.field(EXPONENTIAL_AVG_CHECKPOINT_DURATION_MS.getPreferredName(), this.expAvgCheckpointDurationMs); builder.field(EXPONENTIAL_AVG_DOCUMENTS_INDEXED.getPreferredName(), this.expAvgDocumentsIndexed); builder.field(EXPONENTIAL_AVG_DOCUMENTS_PROCESSED.getPreferredName(), this.expAvgDocumentsProcessed); @@ -202,6 +229,14 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws return builder; } + public long getDeleteTime() { + return deleteTime; + } + + public long getNumDeletedDocuments() { + return numDeletedDocuments; + } + public double getExpAvgCheckpointDurationMs() { return expAvgCheckpointDurationMs; } @@ -214,6 +249,19 @@ public double getExpAvgDocumentsProcessed() { return expAvgDocumentsProcessed; } + public void incrementNumDeletedDocuments(long n) { + assert (n >= 0); + numDeletedDocuments += n; + } + + public void markStartDelete() { + startDeleteTime = System.nanoTime(); + } + + public void markEndDelete() { + deleteTime += ((System.nanoTime() - startDeleteTime) / 1000000); + } + public void incrementCheckpointExponentialAverages(long checkpointDurationMs, long docsIndexed, long docsProcessed) { assert checkpointDurationMs >= 0; assert docsIndexed >= 0; @@ -249,10 +297,12 @@ public boolean equals(Object other) { return Objects.equals(this.numPages, that.numPages) && Objects.equals(this.numInputDocuments, that.numInputDocuments) && Objects.equals(this.numOuputDocuments, that.numOuputDocuments) + && Objects.equals(this.numDeletedDocuments, that.numDeletedDocuments) && Objects.equals(this.numInvocations, that.numInvocations) && Objects.equals(this.indexTime, that.indexTime) && Objects.equals(this.searchTime, that.searchTime) && Objects.equals(this.processingTime, that.processingTime) + && Objects.equals(this.deleteTime, that.deleteTime) && Objects.equals(this.indexFailures, that.indexFailures) && Objects.equals(this.searchFailures, that.searchFailures) && Objects.equals(this.indexTotal, that.indexTotal) @@ -269,10 +319,12 @@ public int hashCode() { numPages, numInputDocuments, numOuputDocuments, + numDeletedDocuments, numInvocations, indexTime, searchTime, processingTime, + deleteTime, indexFailures, searchFailures, indexTotal, diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/persistence/TransformInternalIndexConstants.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/persistence/TransformInternalIndexConstants.java index 062e648d189b7..1f8472d159e56 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/persistence/TransformInternalIndexConstants.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/persistence/TransformInternalIndexConstants.java @@ -21,6 +21,7 @@ public final class TransformInternalIndexConstants { * - XPackRestTestConstants * - yaml tests under x-pack/qa/ * - upgrade tests under x-pack/qa/rolling-upgrade + * - TransformSurvivesUpgradeIT * * (pro-tip: grep for the constant) */ @@ -28,8 +29,8 @@ public final class TransformInternalIndexConstants { // internal index // version is not a rollover pattern, however padded because sort is string based - public static final Version INDEX_VERSION_LAST_CHANGED = Version.V_7_7_0; - public static final String INDEX_VERSION = "005"; + public static final Version INDEX_VERSION_LAST_CHANGED = Version.V_7_12_0; + public static final String INDEX_VERSION = "006"; public static final String INDEX_PATTERN = ".transform-internal-"; public static final String LATEST_INDEX_VERSIONED_NAME = INDEX_PATTERN + INDEX_VERSION; public static final String LATEST_INDEX_NAME = LATEST_INDEX_VERSIONED_NAME; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/AbstractSerializingTransformTestCase.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/AbstractSerializingTransformTestCase.java similarity index 60% rename from x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/AbstractSerializingTransformTestCase.java rename to x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/AbstractSerializingTransformTestCase.java index 781098f1e7601..2bab028adf052 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/AbstractSerializingTransformTestCase.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/AbstractSerializingTransformTestCase.java @@ -5,9 +5,10 @@ * 2.0. */ -package org.elasticsearch.xpack.core.transform.action; +package org.elasticsearch.xpack.core.transform; import org.elasticsearch.Version; +import org.elasticsearch.common.ParseField; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; @@ -16,21 +17,30 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.ToXContent.Params; +import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.search.SearchModule; +import org.elasticsearch.search.aggregations.AggregationBuilder; +import org.elasticsearch.search.aggregations.BaseAggregationBuilder; import org.elasticsearch.test.AbstractSerializingTestCase; -import org.elasticsearch.xpack.core.transform.TransformField; -import org.elasticsearch.xpack.core.transform.TransformNamedXContentProvider; +import org.elasticsearch.xpack.core.transform.transforms.RetentionPolicyConfig; import org.elasticsearch.xpack.core.transform.transforms.SyncConfig; +import org.elasticsearch.xpack.core.transform.transforms.TimeRetentionPolicyConfig; import org.elasticsearch.xpack.core.transform.transforms.TimeSyncConfig; import org.junit.Before; import java.io.IOException; +import java.util.Collections; import java.util.List; import static java.util.Collections.emptyList; public abstract class AbstractSerializingTransformTestCase extends AbstractSerializingTestCase { + protected static Params TO_XCONTENT_PARAMS = new ToXContent.MapParams( + Collections.singletonMap(TransformField.FOR_INTERNAL_STORAGE, "true") + ); + private NamedWriteableRegistry namedWriteableRegistry; private NamedXContentRegistry namedXContentRegistry; @@ -40,10 +50,42 @@ public void registerNamedObjects() { List namedWriteables = searchModule.getNamedWriteables(); namedWriteables.add( - new NamedWriteableRegistry.Entry(SyncConfig.class, TransformField.TIME_BASED_SYNC.getPreferredName(), TimeSyncConfig::new) + new NamedWriteableRegistry.Entry(QueryBuilder.class, MockDeprecatedQueryBuilder.NAME, MockDeprecatedQueryBuilder::new) + ); + namedWriteables.add( + new NamedWriteableRegistry.Entry( + AggregationBuilder.class, + MockDeprecatedAggregationBuilder.NAME, + MockDeprecatedAggregationBuilder::new + ) + ); + namedWriteables.add( + new NamedWriteableRegistry.Entry(SyncConfig.class, TransformField.TIME.getPreferredName(), TimeSyncConfig::new) + ); + namedWriteables.add( + new NamedWriteableRegistry.Entry( + RetentionPolicyConfig.class, + TransformField.TIME.getPreferredName(), + TimeRetentionPolicyConfig::new + ) ); List namedXContents = searchModule.getNamedXContents(); + namedXContents.add( + new NamedXContentRegistry.Entry( + QueryBuilder.class, + new ParseField(MockDeprecatedQueryBuilder.NAME), + (p, c) -> MockDeprecatedQueryBuilder.fromXContent(p) + ) + ); + namedXContents.add( + new NamedXContentRegistry.Entry( + BaseAggregationBuilder.class, + new ParseField(MockDeprecatedAggregationBuilder.NAME), + (p, c) -> MockDeprecatedAggregationBuilder.fromXContent(p) + ) + ); + namedXContents.addAll(new TransformNamedXContentProvider().getNamedXContentParsers()); namedWriteableRegistry = new NamedWriteableRegistry(namedWriteables); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/MockDeprecatedAggregationBuilder.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/MockDeprecatedAggregationBuilder.java similarity index 85% rename from x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/MockDeprecatedAggregationBuilder.java rename to x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/MockDeprecatedAggregationBuilder.java index a5b295ab1cda6..47ee429eaa391 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/MockDeprecatedAggregationBuilder.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/MockDeprecatedAggregationBuilder.java @@ -5,7 +5,7 @@ * 2.0. */ -package org.elasticsearch.xpack.core.transform.transforms; +package org.elasticsearch.xpack.core.transform; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -34,8 +34,11 @@ public class MockDeprecatedAggregationBuilder extends ValuesSourceAggregationBui private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(MockDeprecatedAggregationBuilder.class); - protected MockDeprecatedAggregationBuilder(MockDeprecatedAggregationBuilder clone, Builder factoriesBuilder, - Map metadata) { + protected MockDeprecatedAggregationBuilder( + MockDeprecatedAggregationBuilder clone, + Builder factoriesBuilder, + Map metadata + ) { super(clone, factoriesBuilder, metadata); } @@ -71,8 +74,7 @@ protected ValuesSourceRegistry.RegistryKey getRegistryKey() { } @Override - protected void innerWriteTo(StreamOutput out) throws IOException { - } + protected void innerWriteTo(StreamOutput out) throws IOException {} @Override public BucketCardinality bucketCardinality() { @@ -80,10 +82,12 @@ public BucketCardinality bucketCardinality() { } @Override - protected ValuesSourceAggregatorFactory innerBuild(AggregationContext context, - ValuesSourceConfig config, - AggregatorFactory parent, - Builder subFactoriesBuilder) throws IOException { + protected ValuesSourceAggregatorFactory innerBuild( + AggregationContext context, + ValuesSourceConfig config, + AggregatorFactory parent, + Builder subFactoriesBuilder + ) throws IOException { return null; } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/MockDeprecatedQueryBuilder.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/MockDeprecatedQueryBuilder.java similarity index 96% rename from x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/MockDeprecatedQueryBuilder.java rename to x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/MockDeprecatedQueryBuilder.java index 8fc28cb0bc320..622f0d00c6a77 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/MockDeprecatedQueryBuilder.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/MockDeprecatedQueryBuilder.java @@ -5,7 +5,7 @@ * 2.0. */ -package org.elasticsearch.xpack.core.transform.transforms; +package org.elasticsearch.xpack.core.transform; import org.apache.lucene.search.Query; import org.elasticsearch.common.ParsingException; @@ -38,8 +38,7 @@ public class MockDeprecatedQueryBuilder extends AbstractQueryBuilder namedWriteables = searchModule.getNamedWriteables(); namedWriteables.add( - new NamedWriteableRegistry.Entry(SyncConfig.class, TransformField.TIME_BASED_SYNC.getPreferredName(), TimeSyncConfig::new) + new NamedWriteableRegistry.Entry(SyncConfig.class, TransformField.TIME.getPreferredName(), TimeSyncConfig::new) + ); + namedWriteables.add( + new NamedWriteableRegistry.Entry( + RetentionPolicyConfig.class, + TransformField.TIME.getPreferredName(), + TimeRetentionPolicyConfig::new + ) ); List namedXContents = searchModule.getNamedXContents(); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/PreviewTransformActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/PreviewTransformActionRequestTests.java index aaa8370e8e775..8f8891af7789b 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/PreviewTransformActionRequestTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/PreviewTransformActionRequestTests.java @@ -12,6 +12,7 @@ import org.elasticsearch.common.xcontent.DeprecationHandler; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.elasticsearch.xpack.core.transform.AbstractSerializingTransformTestCase; import org.elasticsearch.xpack.core.transform.action.PreviewTransformAction.Request; import org.elasticsearch.xpack.core.transform.transforms.DestConfig; import org.elasticsearch.xpack.core.transform.transforms.TransformConfig; @@ -53,6 +54,7 @@ protected Request createTestInstance() { null, null, null, + null, null ); return new Request(config); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/PutTransformActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/PutTransformActionRequestTests.java index b3704059acefb..d226c475e5625 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/PutTransformActionRequestTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/PutTransformActionRequestTests.java @@ -7,24 +7,13 @@ package org.elasticsearch.xpack.core.transform.action; -import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.search.SearchModule; -import org.elasticsearch.test.AbstractWireSerializingTestCase; -import org.elasticsearch.xpack.core.transform.TransformField; import org.elasticsearch.xpack.core.transform.action.PutTransformAction.Request; import org.elasticsearch.xpack.core.transform.transforms.TransformConfig; import org.elasticsearch.xpack.core.transform.transforms.TransformConfigTests; -import org.elasticsearch.xpack.core.transform.transforms.SyncConfig; -import org.elasticsearch.xpack.core.transform.transforms.TimeSyncConfig; import org.junit.Before; -import java.util.List; - -import static java.util.Collections.emptyList; - -public class PutTransformActionRequestTests extends AbstractWireSerializingTestCase { +public class PutTransformActionRequestTests extends AbstractWireSerializingTransformTestCase { private String transformId; @Before @@ -42,15 +31,4 @@ protected Request createTestInstance() { TransformConfig config = TransformConfigTests.randomTransformConfigWithoutHeaders(transformId); return new Request(config, randomBoolean()); } - - @Override - protected NamedWriteableRegistry getNamedWriteableRegistry() { - SearchModule searchModule = new SearchModule(Settings.EMPTY, emptyList()); - - List namedWriteables = searchModule.getNamedWriteables(); - namedWriteables.add( - new NamedWriteableRegistry.Entry(SyncConfig.class, TransformField.TIME_BASED_SYNC.getPreferredName(), TimeSyncConfig::new) - ); - return new NamedWriteableRegistry(namedWriteables); - } } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/UpdateTransformsActionResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/UpdateTransformsActionResponseTests.java index 9d91522a5a186..fe9240583b5d3 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/UpdateTransformsActionResponseTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/action/UpdateTransformsActionResponseTests.java @@ -10,10 +10,12 @@ import org.elasticsearch.Version; import org.elasticsearch.common.io.stream.Writeable.Reader; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.xpack.core.transform.AbstractSerializingTransformTestCase; import org.elasticsearch.xpack.core.transform.action.UpdateTransformAction.Response; import org.elasticsearch.xpack.core.transform.action.compat.UpdateTransformActionPre78; import org.elasticsearch.xpack.core.transform.transforms.TransformConfig; import org.elasticsearch.xpack.core.transform.transforms.TransformConfigTests; +import org.elasticsearch.xpack.core.transform.transforms.pivot.PivotConfigTests; import org.elasticsearch.xpack.core.transform.transforms.pivot.SingleGroupSource; import java.io.IOException; @@ -41,8 +43,13 @@ protected Response doParseInstance(XContentParser parser) throws IOException { } public void testBWCPre78() throws IOException { + // latest has been added in 7.11, so we only need to test pivot Response newResponse = new Response( - TransformConfigTests.randomTransformConfigWithoutHeaders(randomAlphaOfLengthBetween(1, 10)) + TransformConfigTests.randomTransformConfigWithoutHeaders( + randomAlphaOfLengthBetween(1, 10), + PivotConfigTests.randomPivotConfig(), + null + ) ); UpdateTransformActionPre78.Response oldResponse = writeAndReadBWCObject( newResponse, @@ -58,12 +65,19 @@ public void testBWCPre78() throws IOException { assertEquals(newResponse.getConfig().getFrequency(), oldResponse.getConfig().getFrequency()); assertEquals( newResponse.getConfig().getPivotConfig().getGroupConfig().getGroups().keySet(), - oldResponse.getConfig().getPivotConfig().getGroupConfig().getGroups().keySet()); - for (Map.Entry oldResponseGroupEntry - : oldResponse.getConfig().getPivotConfig().getGroupConfig().getGroups().entrySet()) { + oldResponse.getConfig().getPivotConfig().getGroupConfig().getGroups().keySet() + ); + for (Map.Entry oldResponseGroupEntry : oldResponse.getConfig() + .getPivotConfig() + .getGroupConfig() + .getGroups() + .entrySet()) { SingleGroupSource oldResponseGroup = oldResponseGroupEntry.getValue(); - SingleGroupSource newResponseGroup = - newResponse.getConfig().getPivotConfig().getGroupConfig().getGroups().get(oldResponseGroupEntry.getKey()); + SingleGroupSource newResponseGroup = newResponse.getConfig() + .getPivotConfig() + .getGroupConfig() + .getGroups() + .get(oldResponseGroupEntry.getKey()); assertThat(oldResponseGroup.getField(), is(equalTo(newResponseGroup.getField()))); assertThat(oldResponseGroup.getScriptConfig(), is(equalTo(newResponseGroup.getScriptConfig()))); // missing_bucket was added in 7.10 so it is always false after deserializing from 7.7 @@ -71,15 +85,18 @@ public void testBWCPre78() throws IOException { } assertEquals( newResponse.getConfig().getPivotConfig().getAggregationConfig(), - oldResponse.getConfig().getPivotConfig().getAggregationConfig()); + oldResponse.getConfig().getPivotConfig().getAggregationConfig() + ); assertEquals( newResponse.getConfig().getPivotConfig().getMaxPageSearchSize(), - oldResponse.getConfig().getPivotConfig().getMaxPageSearchSize()); + oldResponse.getConfig().getPivotConfig().getMaxPageSearchSize() + ); if (newResponse.getConfig().getSource() != null) { assertThat(newResponse.getConfig().getSource().getIndex(), is(equalTo(newResponse.getConfig().getSource().getIndex()))); assertThat( newResponse.getConfig().getSource().getQueryConfig(), - is(equalTo(newResponse.getConfig().getSource().getQueryConfig()))); + is(equalTo(newResponse.getConfig().getSource().getQueryConfig())) + ); // runtime_mappings was added in 7.12 so it is always empty after deserializing from 7.7 assertThat(oldResponse.getConfig().getSource().getRuntimeMappings(), is(anEmptyMap())); } @@ -102,12 +119,19 @@ public void testBWCPre78() throws IOException { assertEquals(newResponse.getConfig().getFrequency(), newRequestFromOld.getConfig().getFrequency()); assertEquals( newResponse.getConfig().getPivotConfig().getGroupConfig().getGroups().keySet(), - newRequestFromOld.getConfig().getPivotConfig().getGroupConfig().getGroups().keySet()); - for (Map.Entry newRequestFromOldGroupEntry - : newRequestFromOld.getConfig().getPivotConfig().getGroupConfig().getGroups().entrySet()) { + newRequestFromOld.getConfig().getPivotConfig().getGroupConfig().getGroups().keySet() + ); + for (Map.Entry newRequestFromOldGroupEntry : newRequestFromOld.getConfig() + .getPivotConfig() + .getGroupConfig() + .getGroups() + .entrySet()) { SingleGroupSource newRequestFromOldGroup = newRequestFromOldGroupEntry.getValue(); - SingleGroupSource newResponseGroup = - newResponse.getConfig().getPivotConfig().getGroupConfig().getGroups().get(newRequestFromOldGroupEntry.getKey()); + SingleGroupSource newResponseGroup = newResponse.getConfig() + .getPivotConfig() + .getGroupConfig() + .getGroups() + .get(newRequestFromOldGroupEntry.getKey()); assertThat(newRequestFromOldGroup.getField(), is(equalTo(newResponseGroup.getField()))); assertThat(newRequestFromOldGroup.getScriptConfig(), is(equalTo(newResponseGroup.getScriptConfig()))); // missing_bucket was added in 7.10 so it is always false after deserializing from 7.7 @@ -115,15 +139,18 @@ public void testBWCPre78() throws IOException { } assertEquals( newResponse.getConfig().getPivotConfig().getAggregationConfig(), - newRequestFromOld.getConfig().getPivotConfig().getAggregationConfig()); + newRequestFromOld.getConfig().getPivotConfig().getAggregationConfig() + ); assertEquals( newResponse.getConfig().getPivotConfig().getMaxPageSearchSize(), - newRequestFromOld.getConfig().getPivotConfig().getMaxPageSearchSize()); + newRequestFromOld.getConfig().getPivotConfig().getMaxPageSearchSize() + ); if (newResponse.getConfig().getSource() != null) { assertThat(newRequestFromOld.getConfig().getSource().getIndex(), is(equalTo(newResponse.getConfig().getSource().getIndex()))); assertThat( newRequestFromOld.getConfig().getSource().getQueryConfig(), - is(equalTo(newResponse.getConfig().getSource().getQueryConfig()))); + is(equalTo(newResponse.getConfig().getSource().getQueryConfig())) + ); // runtime_mappings was added in 7.12 so it is always empty after deserializing from 7.7 assertThat(newRequestFromOld.getConfig().getSource().getRuntimeMappings(), is(anEmptyMap())); } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/AbstractSerializingTransformTestCase.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/AbstractSerializingTransformTestCase.java deleted file mode 100644 index 9c219b3f78f5a..0000000000000 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/AbstractSerializingTransformTestCase.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.core.transform.transforms; - -import org.elasticsearch.common.ParseField; -import org.elasticsearch.common.io.stream.NamedWriteableRegistry; -import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.xcontent.NamedXContentRegistry; -import org.elasticsearch.common.xcontent.ToXContent; -import org.elasticsearch.common.xcontent.ToXContent.Params; -import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.search.SearchModule; -import org.elasticsearch.search.aggregations.AggregationBuilder; -import org.elasticsearch.search.aggregations.BaseAggregationBuilder; -import org.elasticsearch.test.AbstractSerializingTestCase; -import org.elasticsearch.xpack.core.transform.TransformField; -import org.elasticsearch.xpack.core.transform.TransformNamedXContentProvider; -import org.junit.Before; - -import java.util.Collections; -import java.util.List; - -import static java.util.Collections.emptyList; - -public abstract class AbstractSerializingTransformTestCase - extends AbstractSerializingTestCase { - - protected static Params TO_XCONTENT_PARAMS = new ToXContent.MapParams( - Collections.singletonMap(TransformField.FOR_INTERNAL_STORAGE, "true")); - - /** - * Test case that ensures aggregation named objects are registered - */ - private NamedWriteableRegistry namedWriteableRegistry; - private NamedXContentRegistry namedXContentRegistry; - - @Before - public void registerAggregationNamedObjects() throws Exception { - // register aggregations as NamedWriteable - SearchModule searchModule = new SearchModule(Settings.EMPTY, emptyList()); - - List namedWriteables = searchModule.getNamedWriteables(); - namedWriteables.add(new NamedWriteableRegistry.Entry(QueryBuilder.class, MockDeprecatedQueryBuilder.NAME, - MockDeprecatedQueryBuilder::new)); - namedWriteables.add(new NamedWriteableRegistry.Entry(AggregationBuilder.class, MockDeprecatedAggregationBuilder.NAME, - MockDeprecatedAggregationBuilder::new)); - namedWriteables.add(new NamedWriteableRegistry.Entry(SyncConfig.class, TransformField.TIME_BASED_SYNC.getPreferredName(), - TimeSyncConfig::new)); - - List namedXContents = searchModule.getNamedXContents(); - namedXContents.add(new NamedXContentRegistry.Entry(QueryBuilder.class, - new ParseField(MockDeprecatedQueryBuilder.NAME), (p, c) -> MockDeprecatedQueryBuilder.fromXContent(p))); - namedXContents.add(new NamedXContentRegistry.Entry(BaseAggregationBuilder.class, - new ParseField(MockDeprecatedAggregationBuilder.NAME), (p, c) -> MockDeprecatedAggregationBuilder.fromXContent(p))); - namedXContents.addAll(new TransformNamedXContentProvider().getNamedXContentParsers()); - - namedWriteableRegistry = new NamedWriteableRegistry(namedWriteables); - namedXContentRegistry = new NamedXContentRegistry(namedXContents); - } - - @Override - protected NamedWriteableRegistry getNamedWriteableRegistry() { - return namedWriteableRegistry; - } - - @Override - protected NamedXContentRegistry xContentRegistry() { - return namedXContentRegistry; - } -} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/DestConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/DestConfigTests.java index 744ac287ed0c3..1697311da2de1 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/DestConfigTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/DestConfigTests.java @@ -9,6 +9,7 @@ import org.elasticsearch.common.io.stream.Writeable.Reader; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.xpack.core.transform.AbstractSerializingTransformTestCase; import org.junit.Before; import java.io.IOException; @@ -18,8 +19,7 @@ public class DestConfigTests extends AbstractSerializingTransformTestCase) XContentHelper.convertToMap(BytesReference.bytes(content), true, XContentType.JSON) - .v2(); + .v2(); } catch (IOException e) { // should not happen fail("failed to create random query config"); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/SettingsConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/SettingsConfigTests.java index c2e74bfc03871..273a5c817bc1d 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/SettingsConfigTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/SettingsConfigTests.java @@ -16,6 +16,7 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.elasticsearch.xpack.core.transform.AbstractSerializingTransformTestCase; import org.elasticsearch.xpack.core.watcher.watch.Payload.XContent; import org.junit.Before; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/SourceConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/SourceConfigTests.java index c77728dcc59e5..9cecfbf789ab2 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/SourceConfigTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/SourceConfigTests.java @@ -9,6 +9,7 @@ import org.elasticsearch.common.io.stream.Writeable.Reader; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.xpack.core.transform.AbstractSerializingTransformTestCase; import org.junit.Before; import java.io.IOException; @@ -31,7 +32,8 @@ public static SourceConfig randomSourceConfig() { return new SourceConfig( generateRandomStringArray(10, 10, false, false), QueryConfigTests.randomQueryConfig(), - randomRuntimeMappings()); + randomRuntimeMappings() + ); } public static SourceConfig randomInvalidSourceConfig() { @@ -39,7 +41,8 @@ public static SourceConfig randomInvalidSourceConfig() { return new SourceConfig( generateRandomStringArray(10, 10, false, false), QueryConfigTests.randomInvalidQueryConfig(), - randomRuntimeMappings()); + randomRuntimeMappings() + ); } private static Map randomRuntimeMappings() { @@ -80,50 +83,71 @@ protected Reader instanceReader() { } public void testGetRuntimeMappings_EmptyRuntimeMappings() { - SourceConfig sourceConfig = - new SourceConfig( - generateRandomStringArray(10, 10, false, false), - QueryConfigTests.randomQueryConfig(), - emptyMap()); + SourceConfig sourceConfig = new SourceConfig( + generateRandomStringArray(10, 10, false, false), + QueryConfigTests.randomQueryConfig(), + emptyMap() + ); assertThat(sourceConfig.getRuntimeMappings(), is(anEmptyMap())); assertThat(sourceConfig.getScriptBasedRuntimeMappings(), is(anEmptyMap())); } public void testGetRuntimeMappings_NonEmptyRuntimeMappings() { - Map runtimeMappings = - new HashMap<>() {{ + Map runtimeMappings = new HashMap<>() { + { put("field-A", singletonMap("type", "keyword")); put("field-B", singletonMap("script", "some script")); put("field-C", singletonMap("script", "some other script")); - }}; - Map scriptBasedRuntimeMappings = - new HashMap<>() {{ + } + }; + Map scriptBasedRuntimeMappings = new HashMap<>() { + { put("field-B", singletonMap("script", "some script")); put("field-C", singletonMap("script", "some other script")); - }}; - SourceConfig sourceConfig = - new SourceConfig( - generateRandomStringArray(10, 10, false, false), - QueryConfigTests.randomQueryConfig(), - runtimeMappings); + } + }; + SourceConfig sourceConfig = new SourceConfig( + generateRandomStringArray(10, 10, false, false), + QueryConfigTests.randomQueryConfig(), + runtimeMappings + ); assertThat(sourceConfig.getRuntimeMappings(), is(equalTo(runtimeMappings))); assertThat(sourceConfig.getScriptBasedRuntimeMappings(), is(equalTo(scriptBasedRuntimeMappings))); } public void testRequiresRemoteCluster() { - assertFalse(new SourceConfig(new String [] {"index1", "index2", "index3"}, - QueryConfigTests.randomQueryConfig(), randomRuntimeMappings()).requiresRemoteCluster()); + assertFalse( + new SourceConfig(new String[] { "index1", "index2", "index3" }, QueryConfigTests.randomQueryConfig(), randomRuntimeMappings()) + .requiresRemoteCluster() + ); - assertTrue(new SourceConfig(new String [] {"index1", "remote2:index2", "index3"}, - QueryConfigTests.randomQueryConfig(), randomRuntimeMappings()).requiresRemoteCluster()); - - assertTrue(new SourceConfig(new String [] {"index1", "index2", "remote3:index3"}, - QueryConfigTests.randomQueryConfig(), randomRuntimeMappings()).requiresRemoteCluster()); + assertTrue( + new SourceConfig( + new String[] { "index1", "remote2:index2", "index3" }, + QueryConfigTests.randomQueryConfig(), + randomRuntimeMappings() + ).requiresRemoteCluster() + ); - assertTrue(new SourceConfig(new String [] {"index1", "remote2:index2", "remote3:index3"}, - QueryConfigTests.randomQueryConfig(), randomRuntimeMappings()).requiresRemoteCluster()); + assertTrue( + new SourceConfig( + new String[] { "index1", "index2", "remote3:index3" }, + QueryConfigTests.randomQueryConfig(), + randomRuntimeMappings() + ).requiresRemoteCluster() + ); - assertTrue(new SourceConfig(new String [] {"remote1:index1"}, - QueryConfigTests.randomQueryConfig(), randomRuntimeMappings()).requiresRemoteCluster()); + assertTrue( + new SourceConfig( + new String[] { "index1", "remote2:index2", "remote3:index3" }, + QueryConfigTests.randomQueryConfig(), + randomRuntimeMappings() + ).requiresRemoteCluster() + ); + + assertTrue( + new SourceConfig(new String[] { "remote1:index1" }, QueryConfigTests.randomQueryConfig(), randomRuntimeMappings()) + .requiresRemoteCluster() + ); } } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TimeRetentionPolicyConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TimeRetentionPolicyConfigTests.java new file mode 100644 index 0000000000000..15c3070be8caa --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TimeRetentionPolicyConfigTests.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.core.transform.transforms; + +import org.elasticsearch.common.io.stream.Writeable.Reader; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.test.AbstractSerializingTestCase; + +import java.io.IOException; + +public class TimeRetentionPolicyConfigTests extends AbstractSerializingTestCase { + + public static TimeRetentionPolicyConfig randomTimeRetentionPolicyConfig() { + return new TimeRetentionPolicyConfig(randomAlphaOfLengthBetween(1, 10), new TimeValue(randomLongBetween(60000, Long.MAX_VALUE))); + } + + @Override + protected TimeRetentionPolicyConfig doParseInstance(XContentParser parser) throws IOException { + return TimeRetentionPolicyConfig.fromXContent(parser, false); + } + + @Override + protected TimeRetentionPolicyConfig createTestInstance() { + return randomTimeRetentionPolicyConfig(); + } + + @Override + protected Reader instanceReader() { + return TimeRetentionPolicyConfig::new; + } +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformCheckpointStatsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformCheckpointStatsTests.java index c6c46ae99b15e..ef1ef1fa8c0e8 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformCheckpointStatsTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformCheckpointStatsTests.java @@ -9,6 +9,7 @@ import org.elasticsearch.common.io.stream.Writeable.Reader; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.xpack.core.transform.AbstractSerializingTransformTestCase; import java.io.IOException; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformCheckpointTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformCheckpointTests.java index a0d6c21702203..f84fd8d8b5ce0 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformCheckpointTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformCheckpointTests.java @@ -12,6 +12,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.xpack.core.transform.AbstractSerializingTransformTestCase; import java.io.IOException; import java.util.ArrayList; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformCheckpointingInfoTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformCheckpointingInfoTests.java index d150b6756410e..bd3e087e161a9 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformCheckpointingInfoTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformCheckpointingInfoTests.java @@ -12,6 +12,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.Writeable.Reader; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.xpack.core.transform.AbstractSerializingTransformTestCase; import java.io.IOException; import java.time.Instant; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigTests.java index 91404bc27aa71..2caf21fc53a8c 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigTests.java @@ -17,7 +17,9 @@ import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.xpack.core.transform.AbstractSerializingTransformTestCase; import org.elasticsearch.xpack.core.transform.transforms.latest.LatestConfig; +import org.elasticsearch.xpack.core.transform.transforms.latest.LatestConfigTests; import org.elasticsearch.xpack.core.transform.transforms.pivot.PivotConfig; import org.elasticsearch.xpack.core.transform.transforms.pivot.PivotConfigTests; import org.junit.Before; @@ -43,6 +45,20 @@ public static TransformConfig randomTransformConfigWithoutHeaders() { } public static TransformConfig randomTransformConfigWithoutHeaders(String id) { + PivotConfig pivotConfig; + LatestConfig latestConfig; + if (randomBoolean()) { + pivotConfig = PivotConfigTests.randomPivotConfig(); + latestConfig = null; + } else { + pivotConfig = null; + latestConfig = LatestConfigTests.randomLatestConfig(); + } + + return randomTransformConfigWithoutHeaders(id, pivotConfig, latestConfig); + } + + public static TransformConfig randomTransformConfigWithoutHeaders(String id, PivotConfig pivotConfig, LatestConfig latestConfig) { return new TransformConfig( id, randomSourceConfig(), @@ -50,10 +66,11 @@ public static TransformConfig randomTransformConfigWithoutHeaders(String id) { randomBoolean() ? null : TimeValue.timeValueMillis(randomIntBetween(1_000, 3_600_000)), randomBoolean() ? null : randomSyncConfig(), null, - PivotConfigTests.randomPivotConfig(Version.CURRENT), - null, + pivotConfig, + latestConfig, randomBoolean() ? null : randomAlphaOfLengthBetween(1, 1000), SettingsConfigTests.randomSettingsConfig(), + randomBoolean() ? null : randomRetentionPolicyConfig(), null, null ); @@ -64,7 +81,17 @@ public static TransformConfig randomTransformConfig() { } public static TransformConfig randomTransformConfig(String id) { - return randomTransformConfig(id, PivotConfigTests.randomPivotConfig(Version.CURRENT), null); + PivotConfig pivotConfig; + LatestConfig latestConfig; + if (randomBoolean()) { + pivotConfig = PivotConfigTests.randomPivotConfig(); + latestConfig = null; + } else { + pivotConfig = null; + latestConfig = LatestConfigTests.randomLatestConfig(); + } + + return randomTransformConfig(id, pivotConfig, latestConfig); } public static TransformConfig randomTransformConfig(String id, PivotConfig pivotConfig, LatestConfig latestConfig) { @@ -79,6 +106,7 @@ public static TransformConfig randomTransformConfig(String id, PivotConfig pivot latestConfig, randomBoolean() ? null : randomAlphaOfLengthBetween(1, 1000), randomBoolean() ? null : SettingsConfigTests.randomSettingsConfig(), + randomBoolean() ? null : randomRetentionPolicyConfig(), randomBoolean() ? null : Instant.now(), randomBoolean() ? null : Version.CURRENT.toString() ); @@ -86,6 +114,16 @@ public static TransformConfig randomTransformConfig(String id, PivotConfig pivot public static TransformConfig randomInvalidTransformConfig() { if (randomBoolean()) { + PivotConfig pivotConfig; + LatestConfig latestConfig; + if (randomBoolean()) { + pivotConfig = PivotConfigTests.randomPivotConfig(); + latestConfig = null; + } else { + pivotConfig = null; + latestConfig = LatestConfigTests.randomLatestConfig(); + } + return new TransformConfig( randomAlphaOfLengthBetween(1, 10), randomInvalidSourceConfig(), @@ -93,10 +131,11 @@ public static TransformConfig randomInvalidTransformConfig() { null, randomBoolean() ? randomSyncConfig() : null, randomHeaders(), - PivotConfigTests.randomPivotConfig(), - null, + pivotConfig, + latestConfig, randomBoolean() ? null : randomAlphaOfLengthBetween(1, 1000), null, + randomBoolean() ? null : randomRetentionPolicyConfig(), null, null ); @@ -112,6 +151,7 @@ public static TransformConfig randomInvalidTransformConfig() { null, randomBoolean() ? null : randomAlphaOfLengthBetween(1, 1000), null, + randomBoolean() ? null : randomRetentionPolicyConfig(), null, null ); @@ -121,6 +161,10 @@ public static SyncConfig randomSyncConfig() { return TimeSyncConfigTests.randomTimeSyncConfig(); } + public static RetentionPolicyConfig randomRetentionPolicyConfig() { + return TimeRetentionPolicyConfigTests.randomTimeRetentionPolicyConfig(); + } + @Before public void setUpOptionalId() { transformId = randomAlphaOfLengthBetween(1, 10); @@ -187,11 +231,15 @@ public void testDefaultMatchAll() throws IOException { } public void testConstructor_NoFunctionProvided() throws IOException { + // tag::NO_CODE_FORMAT String json = "{" - + " \"source\": {\"index\": \"src\"}," - + " \"dest\": {\"index\": \"dest\"}" - + "}"; - + + " \"source\": {" + + " \"index\": \"src\"" + + " }," + + " \"dest\": {" + + " \"index\": \"dest\"" + + "} }"; + // end::NO_CODE_FORMAT // Should parse with lenient parser createTransformConfigFromString(json, "dummy", true); // Should throw with strict parser @@ -319,6 +367,7 @@ public void testMaxLengthDescription() { randomAlphaOfLength(1001), null, null, + null, null ) ); @@ -336,6 +385,7 @@ public void testMaxLengthDescription() { description, null, null, + null, null ); assertThat(description, equalTo(config.getDescription())); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigUpdateTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigUpdateTests.java index 341824b624c7d..ef44035a4e423 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigUpdateTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigUpdateTests.java @@ -38,14 +38,11 @@ public static TransformConfigUpdate randomTransformConfigUpdate() { randomBoolean() ? null : TimeValue.timeValueMillis(randomIntBetween(1_000, 3_600_000)), randomBoolean() ? null : randomSyncConfig(), randomBoolean() ? null : randomAlphaOfLengthBetween(1, 1000), - randomBoolean() ? null : SettingsConfigTests.randomSettingsConfig() + randomBoolean() ? null : SettingsConfigTests.randomSettingsConfig(), + randomBoolean() ? null : randomRetentionPolicyConfig() ); } - public static SyncConfig randomSyncConfig() { - return TimeSyncConfigTests.randomTimeSyncConfig(); - } - @Override protected TransformConfigUpdate createTestInstance() { return randomTransformConfigUpdate(); @@ -59,7 +56,7 @@ protected Reader instanceReader() { public void testIsNoop() { for (int i = 0; i < NUMBER_OF_TEST_RUNS; i++) { TransformConfig config = randomTransformConfig(); - TransformConfigUpdate update = new TransformConfigUpdate(null, null, null, null, null, null); + TransformConfigUpdate update = new TransformConfigUpdate(null, null, null, null, null, null, null); assertTrue("null update is not noop", update.isNoop(config)); update = new TransformConfigUpdate( config.getSource(), @@ -67,7 +64,8 @@ public void testIsNoop() { config.getFrequency(), config.getSyncConfig(), config.getDescription(), - config.getSettings() + config.getSettings(), + config.getRetentionPolicyConfig() ); assertTrue("equal update is not noop", update.isNoop(config)); @@ -77,7 +75,8 @@ public void testIsNoop() { config.getFrequency(), config.getSyncConfig(), "this is a new description", - config.getSettings() + config.getSettings(), + config.getRetentionPolicyConfig() ); assertFalse("true update is noop", update.isNoop(config)); } @@ -89,16 +88,17 @@ public void testApply() { randomSourceConfig(), randomDestConfig(), TimeValue.timeValueMillis(randomIntBetween(1_000, 3_600_000)), - TimeSyncConfigTests.randomTimeSyncConfig(), + randomSyncConfig(), Collections.singletonMap("key", "value"), PivotConfigTests.randomPivotConfig(), null, randomBoolean() ? null : randomAlphaOfLengthBetween(1, 1000), SettingsConfigTests.randomNonEmptySettingsConfig(), + randomRetentionPolicyConfig(), randomBoolean() ? null : Instant.now(), randomBoolean() ? null : Version.V_7_2_0.toString() ); - TransformConfigUpdate update = new TransformConfigUpdate(null, null, null, null, null, null); + TransformConfigUpdate update = new TransformConfigUpdate(null, null, null, null, null, null, null); assertThat(config, equalTo(update.apply(config))); SourceConfig sourceConfig = new SourceConfig("the_new_index"); @@ -107,7 +107,16 @@ public void testApply() { SyncConfig syncConfig = new TimeSyncConfig("time_field", TimeValue.timeValueSeconds(30)); String newDescription = "new description"; SettingsConfig settings = new SettingsConfig(4_000, 4_000.400F, true); - update = new TransformConfigUpdate(sourceConfig, destConfig, frequency, syncConfig, newDescription, settings); + RetentionPolicyConfig retentionPolicyConfig = new TimeRetentionPolicyConfig("time_field", new TimeValue(60_000)); + update = new TransformConfigUpdate( + sourceConfig, + destConfig, + frequency, + syncConfig, + newDescription, + settings, + retentionPolicyConfig + ); Map headers = Collections.singletonMap("foo", "bar"); update.setHeaders(headers); @@ -119,6 +128,7 @@ public void testApply() { assertThat(updatedConfig.getSyncConfig(), equalTo(syncConfig)); assertThat(updatedConfig.getDescription(), equalTo(newDescription)); assertThat(updatedConfig.getSettings(), equalTo(settings)); + assertThat(updatedConfig.getRetentionPolicyConfig(), equalTo(retentionPolicyConfig)); assertThat(updatedConfig.getHeaders(), equalTo(headers)); assertThat(updatedConfig.getVersion(), equalTo(Version.CURRENT)); } @@ -129,12 +139,13 @@ public void testApplySettings() { randomSourceConfig(), randomDestConfig(), TimeValue.timeValueMillis(randomIntBetween(1_000, 3_600_000)), - TimeSyncConfigTests.randomTimeSyncConfig(), + randomSyncConfig(), Collections.singletonMap("key", "value"), PivotConfigTests.randomPivotConfig(), null, randomBoolean() ? null : randomAlphaOfLengthBetween(1, 1000), SettingsConfigTests.randomNonEmptySettingsConfig(), + randomRetentionPolicyConfig(), randomBoolean() ? null : Instant.now(), randomBoolean() ? null : Version.V_7_2_0.toString() ); @@ -145,7 +156,8 @@ public void testApplySettings() { null, null, null, - new SettingsConfig(4_000, null, (Boolean) null) + new SettingsConfig(4_000, null, (Boolean) null), + null ); TransformConfig updatedConfig = update.apply(config); @@ -154,18 +166,18 @@ public void testApplySettings() { assertThat(updatedConfig.getSettings().getMaxPageSearchSize(), equalTo(4_000)); assertThat(updatedConfig.getSettings().getDocsPerSecond(), equalTo(config.getSettings().getDocsPerSecond())); - update = new TransformConfigUpdate(null, null, null, null, null, new SettingsConfig(null, 43.244F, (Boolean) null)); + update = new TransformConfigUpdate(null, null, null, null, null, new SettingsConfig(null, 43.244F, (Boolean) null), null); updatedConfig = update.apply(updatedConfig); assertThat(updatedConfig.getSettings().getMaxPageSearchSize(), equalTo(4_000)); assertThat(updatedConfig.getSettings().getDocsPerSecond(), equalTo(43.244F)); // now reset to default using the magic -1 - update = new TransformConfigUpdate(null, null, null, null, null, new SettingsConfig(-1, null, (Boolean) null)); + update = new TransformConfigUpdate(null, null, null, null, null, new SettingsConfig(-1, null, (Boolean) null), null); updatedConfig = update.apply(updatedConfig); assertNull(updatedConfig.getSettings().getMaxPageSearchSize()); assertThat(updatedConfig.getSettings().getDocsPerSecond(), equalTo(43.244F)); - update = new TransformConfigUpdate(null, null, null, null, null, new SettingsConfig(-1, -1F, (Boolean) null)); + update = new TransformConfigUpdate(null, null, null, null, null, new SettingsConfig(-1, -1F, (Boolean) null), null); updatedConfig = update.apply(updatedConfig); assertNull(updatedConfig.getSettings().getMaxPageSearchSize()); assertNull(updatedConfig.getSettings().getDocsPerSecond()); @@ -183,11 +195,20 @@ public void testApplyWithSyncChange() { null, randomBoolean() ? null : randomAlphaOfLengthBetween(1, 1000), SettingsConfigTests.randomNonEmptySettingsConfig(), + randomRetentionPolicyConfig(), randomBoolean() ? null : Instant.now(), randomBoolean() ? null : Version.CURRENT.toString() ); - TransformConfigUpdate update = new TransformConfigUpdate(null, null, null, TimeSyncConfigTests.randomTimeSyncConfig(), null, null); + TransformConfigUpdate update = new TransformConfigUpdate( + null, + null, + null, + TimeSyncConfigTests.randomTimeSyncConfig(), + null, + null, + null + ); ElasticsearchStatusException ex = expectThrows(ElasticsearchStatusException.class, () -> update.apply(batchConfig)); assertThat( @@ -200,17 +221,18 @@ public void testApplyWithSyncChange() { randomSourceConfig(), randomDestConfig(), TimeValue.timeValueMillis(randomIntBetween(1_000, 3_600_000)), - TimeSyncConfigTests.randomTimeSyncConfig(), + randomSyncConfig(), null, PivotConfigTests.randomPivotConfig(), null, randomBoolean() ? null : randomAlphaOfLengthBetween(1, 1000), SettingsConfigTests.randomNonEmptySettingsConfig(), + randomRetentionPolicyConfig(), randomBoolean() ? null : Instant.now(), randomBoolean() ? null : Version.CURRENT.toString() ); - TransformConfigUpdate fooSyncUpdate = new TransformConfigUpdate(null, null, null, new FooSync(), null, null); + TransformConfigUpdate fooSyncUpdate = new TransformConfigUpdate(null, null, null, new FooSync(), null, null, null); ex = expectThrows(ElasticsearchStatusException.class, () -> fooSyncUpdate.apply(timeSyncedConfig)); assertThat( ex.getMessage(), @@ -250,10 +272,23 @@ private void toXContent(TransformConfigUpdate update, XContentBuilder builder) t if (update.getSettings() != null) { builder.field(TransformField.SETTINGS.getPreferredName(), update.getSettings()); } + if (update.getRetentionPolicyConfig() != null) { + builder.startObject(TransformField.RETENTION_POLICY.getPreferredName()); + builder.field(update.getRetentionPolicyConfig().getWriteableName(), update.getRetentionPolicyConfig()); + builder.endObject(); + } builder.endObject(); } + private static SyncConfig randomSyncConfig() { + return TimeSyncConfigTests.randomTimeSyncConfig(); + } + + private static RetentionPolicyConfig randomRetentionPolicyConfig() { + return TimeRetentionPolicyConfigTests.randomTimeRetentionPolicyConfig(); + } + static class FooSync implements SyncConfig { @Override diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformDestIndexSettingsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformDestIndexSettingsTests.java index 93e8f0b2e0bb0..8af85cf69913e 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformDestIndexSettingsTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformDestIndexSettingsTests.java @@ -11,6 +11,7 @@ import org.elasticsearch.common.io.stream.Writeable.Reader; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.xpack.core.transform.AbstractSerializingTransformTestCase; import java.io.IOException; import java.util.HashMap; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformIndexerStatsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformIndexerStatsTests.java index 86cb2bc28783e..3b43d95f0ca55 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformIndexerStatsTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformIndexerStatsTests.java @@ -34,6 +34,8 @@ public static TransformIndexerStats randomStats() { randomLongBetween(0L, 10000L), randomLongBetween(0L, 10000L), randomLongBetween(0L, 10000L), + randomLongBetween(0L, 10000L), + randomLongBetween(0L, 10000L), randomDouble(), randomDouble(), randomDouble() @@ -99,6 +101,7 @@ public static void toXContentIfNotZero(TransformIndexerStats stats, XContentBuil xContentFieldIfNotZero(builder, TransformIndexerStats.NUM_PAGES.getPreferredName(), stats.getNumPages()); xContentFieldIfNotZero(builder, TransformIndexerStats.NUM_INPUT_DOCUMENTS.getPreferredName(), stats.getNumDocuments()); xContentFieldIfNotZero(builder, TransformIndexerStats.NUM_OUTPUT_DOCUMENTS.getPreferredName(), stats.getOutputDocuments()); + xContentFieldIfNotZero(builder, TransformIndexerStats.NUM_DELETED_DOCUMENTS.getPreferredName(), stats.getNumDeletedDocuments()); xContentFieldIfNotZero(builder, TransformIndexerStats.NUM_INVOCATIONS.getPreferredName(), stats.getNumInvocations()); xContentFieldIfNotZero(builder, TransformIndexerStats.INDEX_TIME_IN_MS.getPreferredName(), stats.getIndexTime()); xContentFieldIfNotZero(builder, TransformIndexerStats.INDEX_TOTAL.getPreferredName(), stats.getIndexTotal()); @@ -108,6 +111,7 @@ public static void toXContentIfNotZero(TransformIndexerStats stats, XContentBuil xContentFieldIfNotZero(builder, TransformIndexerStats.PROCESSING_TIME_IN_MS.getPreferredName(), stats.getProcessingTime()); xContentFieldIfNotZero(builder, TransformIndexerStats.PROCESSING_TOTAL.getPreferredName(), stats.getProcessingTotal()); xContentFieldIfNotZero(builder, TransformIndexerStats.SEARCH_FAILURES.getPreferredName(), stats.getSearchFailures()); + xContentFieldIfNotZero(builder, TransformIndexerStats.DELETE_TIME_IN_MS.getPreferredName(), stats.getDeleteTime()); xContentFieldIfNotZero( builder, TransformIndexerStats.EXPONENTIAL_AVG_CHECKPOINT_DURATION_MS.getPreferredName(), diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformProgressTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformProgressTests.java index 8c129d67822ec..541f1c8fd7639 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformProgressTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformProgressTests.java @@ -12,6 +12,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.Writeable.Reader; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.xpack.core.transform.AbstractSerializingTransformTestCase; import java.io.IOException; @@ -26,7 +27,8 @@ public static TransformProgress randomTransformProgress() { return new TransformProgress( randomBoolean() ? null : randomLongBetween(0, 10000), randomBoolean() ? null : randomLongBetween(0, 10000), - randomBoolean() ? null : randomLongBetween(1, 10000)); + randomBoolean() ? null : randomLongBetween(1, 10000) + ); } @Override @@ -65,8 +67,7 @@ public void testPercentComplete() { } public void testConstructor() { - IllegalArgumentException ex = - expectThrows(IllegalArgumentException.class, () -> new TransformProgress(-1L, null, null)); + IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () -> new TransformProgress(-1L, null, null)); assertThat(ex.getMessage(), equalTo("[total_docs] must be >0.")); ex = expectThrows(IllegalArgumentException.class, () -> new TransformProgress(1L, -1L, null)); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformStatsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformStatsTests.java index 6e06a1514678c..bee240f9f3798 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformStatsTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformStatsTests.java @@ -70,7 +70,7 @@ public void testBwcWith73() throws IOException { STARTED, randomBoolean() ? null : randomAlphaOfLength(100), randomBoolean() ? null : NodeAttributeTests.randomNodeAttributes(), - new TransformIndexerStats(1, 2, 3, 4, 5, 6, 0, 8, 9, 0, 11, 12, 0.0, 0.0, 0.0), + new TransformIndexerStats(1, 2, 3, 0, 5, 6, 7, 0, 0, 10, 11, 0, 13, 14, 0.0, 0.0, 0.0), new TransformCheckpointingInfo( new TransformCheckpointStats(0, null, null, 10, 100), new TransformCheckpointStats(0, null, null, 100, 1000), @@ -99,7 +99,7 @@ public void testBwcWith76() throws IOException { STARTED, randomBoolean() ? null : randomAlphaOfLength(100), randomBoolean() ? null : NodeAttributeTests.randomNodeAttributes(), - new TransformIndexerStats(1, 2, 3, 4, 5, 6, 0, 8, 9, 0, 11, 12, 13.0, 14.0, 15.0), + new TransformIndexerStats(1, 2, 3, 0, 5, 6, 7, 0, 0, 10, 11, 0, 13, 14, 15.0, 16.0, 17.0), new TransformCheckpointingInfo( new TransformCheckpointStats(0, null, null, 10, 100), new TransformCheckpointStats(0, null, null, 100, 1000), diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformStoredDocTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformStoredDocTests.java index 83b0c4ac43d40..6c9e71e2b7844 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformStoredDocTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformStoredDocTests.java @@ -10,6 +10,7 @@ import org.elasticsearch.common.io.stream.Writeable.Reader; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.xpack.core.transform.AbstractSerializingTransformTestCase; import org.elasticsearch.xpack.core.transform.TransformField; import java.io.IOException; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformTests.java index e43bb82f98627..ba8969f81b7f9 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformTests.java @@ -13,6 +13,7 @@ import org.elasticsearch.common.io.stream.Writeable.Reader; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.xpack.core.transform.AbstractSerializingTransformTestCase; import java.io.IOException; @@ -27,8 +28,12 @@ protected TransformTaskParams doParseInstance(XContentParser parser) throws IOEx @Override protected TransformTaskParams createTestInstance() { - return new TransformTaskParams(randomAlphaOfLength(10), randomBoolean() ? null : Version.CURRENT, - randomBoolean() ? null : TimeValue.timeValueMillis(randomIntBetween(1_000, 3_600_000)), randomBoolean()); + return new TransformTaskParams( + randomAlphaOfLength(10), + randomBoolean() ? null : Version.CURRENT, + randomBoolean() ? null : TimeValue.timeValueMillis(randomIntBetween(1_000, 3_600_000)), + randomBoolean() + ); } @Override diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/latest/LatestConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/latest/LatestConfigTests.java index 6946b916b8465..ddc688445dfb5 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/latest/LatestConfigTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/latest/LatestConfigTests.java @@ -13,7 +13,7 @@ import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.search.sort.SortBuilders; import org.elasticsearch.search.sort.SortOrder; -import org.elasticsearch.xpack.core.transform.transforms.AbstractSerializingTransformTestCase; +import org.elasticsearch.xpack.core.transform.AbstractSerializingTransformTestCase; import java.io.IOException; import java.util.ArrayList; @@ -49,10 +49,12 @@ protected Reader instanceReader() { } public void testValidate_ValidConfig() throws IOException { + // tag::NO_CODE_FORMAT String json = "{" + " \"unique_key\": [ \"event1\", \"event2\", \"event3\" ]," + " \"sort\": \"timestamp\"" + "}"; + // end::NO_CODE_FORMAT LatestConfig config = createLatestConfigFromString(json); assertThat(config.validate(null), is(nullValue())); @@ -62,59 +64,72 @@ public void testValidate_ValidConfig() throws IOException { } public void testValidate_EmptyUniqueKey() throws IOException { + // tag::NO_CODE_FORMAT String json = "{" + " \"unique_key\": []," + " \"sort\": \"timestamp\"" + "}"; + // end::NO_CODE_FORMAT LatestConfig config = createLatestConfigFromString(json); assertThat(config.validate(null).validationErrors(), contains("latest.unique_key must be non-empty")); } public void testValidate_EmptyUniqueKeyElement() throws IOException { + // tag::NO_CODE_FORMAT String json = "{" + " \"unique_key\": [ \"event1\", \"\", \"event2\", \"\", \"event3\" ]," + " \"sort\": \"timestamp\"" + "}"; + // end::NO_CODE_FORMAT LatestConfig config = createLatestConfigFromString(json); assertThat( config.validate(null).validationErrors(), - containsInAnyOrder("latest.unique_key[1] element must be non-empty", "latest.unique_key[3] element must be non-empty")); + containsInAnyOrder("latest.unique_key[1] element must be non-empty", "latest.unique_key[3] element must be non-empty") + ); } public void testValidate_DuplicateUniqueKeyElement() throws IOException { + // tag::NO_CODE_FORMAT String json = "{" + " \"unique_key\": [ \"event1\", \"event2\", \"event1\" ]," + " \"sort\": \"timestamp\"" + "}"; + // end::NO_CODE_FORMAT LatestConfig config = createLatestConfigFromString(json); assertThat( config.validate(null).validationErrors(), - containsInAnyOrder("latest.unique_key elements must be unique, found duplicate element [event1]")); + containsInAnyOrder("latest.unique_key elements must be unique, found duplicate element [event1]") + ); } public void testValidate_EmptySort() throws IOException { + // tag::NO_CODE_FORMAT String json = "{" + " \"unique_key\": [ \"event1\", \"event2\", \"event3\" ]," + " \"sort\": \"\"" + "}"; + // end::NO_CODE_FORMAT LatestConfig config = createLatestConfigFromString(json); assertThat(config.validate(null).validationErrors(), contains("latest.sort must be non-empty")); } public void testValidate_EmptyUniqueKeyAndSort() throws IOException { + // tag::NO_CODE_FORMAT String json = "{" + " \"unique_key\": []," + " \"sort\": \"\"" + "}"; + // end::NO_CODE_FORMAT LatestConfig config = createLatestConfigFromString(json); assertThat( config.validate(null).validationErrors(), - containsInAnyOrder("latest.unique_key must be non-empty", "latest.sort must be non-empty")); + containsInAnyOrder("latest.unique_key must be non-empty", "latest.sort must be non-empty") + ); } private LatestConfig createLatestConfigFromString(String json) throws IOException { diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/pivot/AggregationConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/pivot/AggregationConfigTests.java index 4685c4621410a..84f0e02fb8a3b 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/pivot/AggregationConfigTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/pivot/AggregationConfigTests.java @@ -20,8 +20,8 @@ import org.elasticsearch.search.aggregations.AggregationBuilder; import org.elasticsearch.search.aggregations.AggregationBuilders; import org.elasticsearch.search.aggregations.AggregatorFactories; -import org.elasticsearch.xpack.core.transform.transforms.AbstractSerializingTransformTestCase; -import org.elasticsearch.xpack.core.transform.transforms.MockDeprecatedAggregationBuilder; +import org.elasticsearch.xpack.core.transform.AbstractSerializingTransformTestCase; +import org.elasticsearch.xpack.core.transform.MockDeprecatedAggregationBuilder; import org.junit.Before; import java.io.IOException; @@ -105,12 +105,12 @@ public void testEmptyAggregation() throws IOException { } public void testFailOnStrictPassOnLenient() throws IOException { - String source = "{\n" + - " \"avg_rating\": { \"some_removed_agg\": { \"field\": \"rating\" } }\n" + - " },\n" + - " {\n" + - " \"max_rating\": { \"max_rating\" : { \"field\" : \"rating\" } }\n" + - " }"; + String source = "{\n" + + " \"avg_rating\": { \"some_removed_agg\": { \"field\": \"rating\" } }\n" + + " },\n" + + " {\n" + + " \"max_rating\": { \"max_rating\" : { \"field\" : \"rating\" } }\n" + + " }"; // lenient, passes but reports invalid try (XContentParser parser = createParser(JsonXContent.jsonXContent, source)) { @@ -136,14 +136,14 @@ public void testDeprecation() throws IOException { private static AggregationBuilder getRandomSupportedAggregation() { final int numberOfSupportedAggs = 4; switch (randomIntBetween(1, numberOfSupportedAggs)) { - case 1: - return AggregationBuilders.avg(randomAlphaOfLengthBetween(1, 10)).field(randomAlphaOfLengthBetween(1, 10)); - case 2: - return AggregationBuilders.min(randomAlphaOfLengthBetween(1, 10)).field(randomAlphaOfLengthBetween(1, 10)); - case 3: - return AggregationBuilders.max(randomAlphaOfLengthBetween(1, 10)).field(randomAlphaOfLengthBetween(1, 10)); - case 4: - return AggregationBuilders.sum(randomAlphaOfLengthBetween(1, 10)).field(randomAlphaOfLengthBetween(1, 10)); + case 1: + return AggregationBuilders.avg(randomAlphaOfLengthBetween(1, 10)).field(randomAlphaOfLengthBetween(1, 10)); + case 2: + return AggregationBuilders.min(randomAlphaOfLengthBetween(1, 10)).field(randomAlphaOfLengthBetween(1, 10)); + case 3: + return AggregationBuilders.max(randomAlphaOfLengthBetween(1, 10)).field(randomAlphaOfLengthBetween(1, 10)); + case 4: + return AggregationBuilders.sum(randomAlphaOfLengthBetween(1, 10)).field(randomAlphaOfLengthBetween(1, 10)); } return null; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/pivot/PivotConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/pivot/PivotConfigTests.java index fff66deb8460f..77dad8f3fc1fc 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/pivot/PivotConfigTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/pivot/PivotConfigTests.java @@ -13,7 +13,7 @@ import org.elasticsearch.common.xcontent.DeprecationHandler; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.xpack.core.transform.transforms.AbstractSerializingTransformTestCase; +import org.elasticsearch.xpack.core.transform.AbstractSerializingTransformTestCase; import java.io.IOException; import java.util.Arrays; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/pivot/ScriptConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/pivot/ScriptConfigTests.java index 7d5851ea416b5..d27741707b7bf 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/pivot/ScriptConfigTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/pivot/ScriptConfigTests.java @@ -19,7 +19,7 @@ import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptType; -import org.elasticsearch.xpack.core.transform.transforms.AbstractSerializingTransformTestCase; +import org.elasticsearch.xpack.core.transform.AbstractSerializingTransformTestCase; import org.junit.Before; import java.io.IOException; diff --git a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformProgressIT.java b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformProgressIT.java index 506dc1a95e729..0fdfa2754e3c9 100644 --- a/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformProgressIT.java +++ b/x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformProgressIT.java @@ -152,8 +152,21 @@ public void assertGetProgress(int userWithMissingBuckets) throws Exception { aggs.addAggregator(AggregationBuilders.avg("avg_rating").field("stars")); AggregationConfig aggregationConfig = new AggregationConfig(Collections.emptyMap(), aggs); PivotConfig pivotConfig = new PivotConfig(histgramGroupConfig, aggregationConfig, null); - TransformConfig config = - new TransformConfig(transformId, sourceConfig, destConfig, null, null, null, pivotConfig, null, null, null, null, null); + TransformConfig config = new TransformConfig( + transformId, + sourceConfig, + destConfig, + null, + null, + null, + pivotConfig, + null, + null, + null, + null, + null, + null + ); Pivot pivot = new Pivot(pivotConfig, new SettingsConfig(), Version.CURRENT); diff --git a/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/integration/TransformInternalIndexIT.java b/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/integration/TransformInternalIndexIT.java index f7eb039fbeb64..daa0c039e8246 100644 --- a/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/integration/TransformInternalIndexIT.java +++ b/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/integration/TransformInternalIndexIT.java @@ -84,7 +84,7 @@ public void testUpdateDeletesOldTransformConfig() throws Exception { assertThat(getTransformResponse.getTransformConfigurations().get(0).getId(), equalTo(transformId)); UpdateTransformAction.Request updateTransformActionRequest = new UpdateTransformAction.Request( - new TransformConfigUpdate(null, null, null, null, "updated", null), + new TransformConfigUpdate(null, null, null, null, "updated", null, null), transformId, false); UpdateTransformAction.Response updateTransformActionResponse = client().execute(UpdateTransformAction.INSTANCE, updateTransformActionRequest).actionGet(); diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/TransformInfoTransportAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/TransformInfoTransportAction.java index 9056d063f282e..706f9256a8e3f 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/TransformInfoTransportAction.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/TransformInfoTransportAction.java @@ -43,10 +43,12 @@ public class TransformInfoTransportAction extends XPackInfoFeatureTransportActio TransformIndexerStats.NUM_PAGES.getPreferredName(), TransformIndexerStats.NUM_INPUT_DOCUMENTS.getPreferredName(), TransformIndexerStats.NUM_OUTPUT_DOCUMENTS.getPreferredName(), + TransformIndexerStats.NUM_DELETED_DOCUMENTS.getPreferredName(), TransformIndexerStats.NUM_INVOCATIONS.getPreferredName(), TransformIndexerStats.INDEX_TIME_IN_MS.getPreferredName(), TransformIndexerStats.SEARCH_TIME_IN_MS.getPreferredName(), TransformIndexerStats.PROCESSING_TIME_IN_MS.getPreferredName(), + TransformIndexerStats.DELETE_TIME_IN_MS.getPreferredName(), TransformIndexerStats.INDEX_TOTAL.getPreferredName(), TransformIndexerStats.SEARCH_TOTAL.getPreferredName(), TransformIndexerStats.PROCESSING_TOTAL.getPreferredName(), @@ -57,10 +59,7 @@ public class TransformInfoTransportAction extends XPackInfoFeatureTransportActio TransformIndexerStats.EXPONENTIAL_AVG_DOCUMENTS_PROCESSED.getPreferredName(), }; @Inject - public TransformInfoTransportAction( - TransportService transportService, - ActionFilters actionFilters - ) { + public TransformInfoTransportAction(TransportService transportService, ActionFilters actionFilters) { super(XPackInfoFeatureAction.TRANSFORM.name(), transportService, actionFilters); } @@ -95,18 +94,20 @@ static TransformIndexerStats parseSearchAggs(SearchResponse searchResponse) { statisticsList.get(0).longValue(), // numPages statisticsList.get(1).longValue(), // numInputDocuments statisticsList.get(2).longValue(), // numOutputDocuments - statisticsList.get(3).longValue(), // numInvocations - statisticsList.get(4).longValue(), // indexTime - statisticsList.get(5).longValue(), // searchTime - statisticsList.get(6).longValue(), // processingTime - statisticsList.get(7).longValue(), // indexTotal - statisticsList.get(8).longValue(), // searchTotal - statisticsList.get(9).longValue(), // processingTotal - statisticsList.get(10).longValue(), // indexFailures - statisticsList.get(11).longValue(), // searchFailures - statisticsList.get(12), // exponential_avg_checkpoint_duration_ms - statisticsList.get(13), // exponential_avg_documents_indexed - statisticsList.get(14) // exponential_avg_documents_processed + statisticsList.get(3).longValue(), // numDeletedDocuments + statisticsList.get(4).longValue(), // numInvocations + statisticsList.get(5).longValue(), // indexTime + statisticsList.get(6).longValue(), // searchTime + statisticsList.get(7).longValue(), // processingTime + statisticsList.get(8).longValue(), // deleteTime + statisticsList.get(9).longValue(), // indexTotal + statisticsList.get(10).longValue(), // searchTotal + statisticsList.get(11).longValue(), // processingTotal + statisticsList.get(12).longValue(), // indexFailures + statisticsList.get(13).longValue(), // searchFailures + statisticsList.get(14), // exponential_avg_checkpoint_duration_ms + statisticsList.get(15), // exponential_avg_documents_indexed + statisticsList.get(16) // exponential_avg_documents_processed ); } diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/TransformInternalIndex.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/TransformInternalIndex.java index 1f322d5f630db..d0875d65faeb3 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/TransformInternalIndex.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/TransformInternalIndex.java @@ -64,6 +64,7 @@ public final class TransformInternalIndex { * version 4 (7.6): state::should_stop_at_checkpoint * checkpoint::checkpoint * version 5 (7.7): stats::processing_time_in_ms, stats::processing_total + * version 6 (7.12):stats::delete_time_in_ms, stats::documents_deleted */ // constants for mappings @@ -241,6 +242,9 @@ private static XContentBuilder addTransformStoredDocMappings(XContentBuilder bui .endObject() .startObject(TransformIndexerStats.NUM_OUTPUT_DOCUMENTS.getPreferredName()) .field(TYPE, LONG) + .endObject() + .startObject(TransformIndexerStats.NUM_DELETED_DOCUMENTS.getPreferredName()) + .field(TYPE, LONG) .endObject() .startObject(TransformIndexerStats.NUM_INVOCATIONS.getPreferredName()) .field(TYPE, LONG) @@ -254,6 +258,9 @@ private static XContentBuilder addTransformStoredDocMappings(XContentBuilder bui .startObject(TransformIndexerStats.PROCESSING_TIME_IN_MS.getPreferredName()) .field(TYPE, LONG) .endObject() + .startObject(TransformIndexerStats.DELETE_TIME_IN_MS.getPreferredName()) + .field(TYPE, LONG) + .endObject() .startObject(TransformIndexerStats.INDEX_TOTAL.getPreferredName()) .field(TYPE, LONG) .endObject() diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/rest/action/RestCatTransformAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/rest/action/RestCatTransformAction.java index ba19265694c38..b133bbb2e3aaa 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/rest/action/RestCatTransformAction.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/rest/action/RestCatTransformAction.java @@ -169,6 +169,16 @@ private static Table getTableWithHeader() { .setAliases("doci") .build() ) + .addCell( + "delete_time", + TableColumnAttributeBuilder.builder("total time spent deleting documents", false).setAliases("dtime").build() + ) + .addCell( + "documents_deleted", + TableColumnAttributeBuilder.builder("the number of documents deleted from the destination index", false) + .setAliases("docd") + .build() + ) .addCell( "trigger_count", TableColumnAttributeBuilder.builder("the number of times the transform has been triggered", false).setAliases("tc").build() @@ -266,6 +276,8 @@ private Table buildTable(GetTransformAction.Response response, GetTransformStats .addCell(transformIndexerStats == null ? null : TimeValue.timeValueMillis(transformIndexerStats.getIndexTime())) .addCell(transformIndexerStats == null ? null : transformIndexerStats.getOutputDocuments()) + .addCell(transformIndexerStats == null ? null : TimeValue.timeValueMillis(transformIndexerStats.getDeleteTime())) + .addCell(transformIndexerStats == null ? null : transformIndexerStats.getNumDeletedDocuments()) .addCell(transformIndexerStats == null ? null : transformIndexerStats.getNumInvocations()) .addCell(transformIndexerStats == null ? null : transformIndexerStats.getNumPages()) .addCell(transformIndexerStats == null ? null : TimeValue.timeValueMillis(transformIndexerStats.getProcessingTime())) diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/ClientTransformIndexer.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/ClientTransformIndexer.java index 73fe07983be4a..8a527f64c33e5 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/ClientTransformIndexer.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/ClientTransformIndexer.java @@ -12,6 +12,9 @@ import org.apache.logging.log4j.message.ParameterizedMessage; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.indices.refresh.RefreshAction; +import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; +import org.elasticsearch.action.admin.indices.refresh.RefreshResponse; import org.elasticsearch.action.bulk.BulkAction; import org.elasticsearch.action.bulk.BulkItemResponse; import org.elasticsearch.action.bulk.BulkRequest; @@ -24,6 +27,9 @@ import org.elasticsearch.common.logging.LoggerMessageFormat; import org.elasticsearch.index.engine.VersionConflictEngineException; import org.elasticsearch.index.mapper.MapperParsingException; +import org.elasticsearch.index.reindex.BulkByScrollResponse; +import org.elasticsearch.index.reindex.DeleteByQueryAction; +import org.elasticsearch.index.reindex.DeleteByQueryRequest; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.core.ClientHelper; import org.elasticsearch.xpack.core.indexing.IndexerState; @@ -196,6 +202,32 @@ protected void doNextBulk(BulkRequest request, ActionListener next ); } + @Override + protected void doDeleteByQuery(DeleteByQueryRequest deleteByQueryRequest, ActionListener responseListener) { + ClientHelper.executeWithHeadersAsync( + transformConfig.getHeaders(), + ClientHelper.TRANSFORM_ORIGIN, + client, + DeleteByQueryAction.INSTANCE, + deleteByQueryRequest, + responseListener + ); + } + + @Override + protected void refreshDestinationIndex(ActionListener responseListener) { + // note: this gets executed _without_ the headers of the user as the user might not have the rights to call + // _refresh for performance reasons. However this refresh is an internal detail of transform and this is only + // called for the transform destination index + ClientHelper.executeAsyncWithOrigin( + client, + ClientHelper.TRANSFORM_ORIGIN, + RefreshAction.INSTANCE, + new RefreshRequest(transformConfig.getDestination().getIndex()), + responseListener + ); + } + @Override void doGetInitialProgress(SearchRequest request, ActionListener responseListener) { ClientHelper.executeWithHeadersAsync( @@ -350,7 +382,7 @@ private void doSaveState(TransformState state, ActionListener listener) { // this should never happen, but indicates a race condition in state persistence: // - there should be only 1 save persistence at a time // - this is not a catastrophic failure, if 2 state persistence calls run at the same time, 1 should succeed and update - // seqNoPrimaryTermAndIndex + // seqNoPrimaryTermAndIndex // - for tests fail(assert), so we can debug the problem logger.error( new ParameterizedMessage( diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/RetentionPolicyToDeleteByQueryRequestConverter.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/RetentionPolicyToDeleteByQueryRequestConverter.java new file mode 100644 index 0000000000000..2f6c5a4adc92f --- /dev/null +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/RetentionPolicyToDeleteByQueryRequestConverter.java @@ -0,0 +1,97 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.transform.transforms; + +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.index.reindex.AbstractBulkByScrollRequest; +import org.elasticsearch.index.reindex.DeleteByQueryRequest; +import org.elasticsearch.xpack.core.transform.transforms.DestConfig; +import org.elasticsearch.xpack.core.transform.transforms.RetentionPolicyConfig; +import org.elasticsearch.xpack.core.transform.transforms.SettingsConfig; +import org.elasticsearch.xpack.core.transform.transforms.TimeRetentionPolicyConfig; +import org.elasticsearch.xpack.core.transform.transforms.TransformCheckpoint; + +import java.time.Instant; + +/** + * Implementation of `retention_policy` configuration parameter. + * + * All implementations of `retention_policy` are converted to a {@link DeleteByQueryRequest}, which is then executed by the indexer. + */ +public final class RetentionPolicyToDeleteByQueryRequestConverter { + + public static class RetentionPolicyException extends ElasticsearchException { + RetentionPolicyException(String msg, Object... args) { + super(msg, args); + } + } + + private RetentionPolicyToDeleteByQueryRequestConverter() {} + + /** + * Build a {@link DeleteByQueryRequest} from a `retention policy`. The DBQ runs _after_ all data for a new checkpoint + * has been processed (composite agg + bulk indexing) and the index got refreshed. After the DBQ - with a final index refresh - + * the checkpoint is complete. + * + * @param retentionPolicyConfig The retention policy configuration + * @param settingsConfig settings to set certain parameters + * @param destConfig the destination config + * @param nextCheckpoint The checkpoint that just finished + * + * @return a delete by query request according to the given configurations or null if no delete by query should be executed + */ + static DeleteByQueryRequest buildDeleteByQueryRequest( + RetentionPolicyConfig retentionPolicyConfig, + SettingsConfig settingsConfig, + DestConfig destConfig, + TransformCheckpoint nextCheckpoint + ) { + if (nextCheckpoint == null || nextCheckpoint.isEmpty()) { + return null; + } + + DeleteByQueryRequest request = new DeleteByQueryRequest(); + + if (retentionPolicyConfig instanceof TimeRetentionPolicyConfig) { + request.setQuery(getDeleteQueryFromTimeBasedRetentionPolicy((TimeRetentionPolicyConfig) retentionPolicyConfig, nextCheckpoint)); + } else { + throw new RetentionPolicyException("unsupported retention policy of type [{}]", retentionPolicyConfig.getWriteableName()); + } + + /* other dbq options not set and why: + * - timeout: we do not timeout for search, so we don't timeout for dbq + * - batch size: don't use max page size search, dbq should be simple + */ + request.setSlices(AbstractBulkByScrollRequest.AUTO_SLICES) + .setBatchSize(AbstractBulkByScrollRequest.DEFAULT_SCROLL_SIZE) + // this should not happen, but still go over version conflicts and report later + .setAbortOnVersionConflict(false) + // refresh the index, so docs are really gone + .setRefresh(true) + // use transforms retry mechanics instead + .setMaxRetries(0) + .indices(destConfig.getIndex()); + + // use the same throttling as for search + if (settingsConfig.getDocsPerSecond() != null) { + request.setRequestsPerSecond(settingsConfig.getDocsPerSecond()); + } + + return request; + } + + private static QueryBuilder getDeleteQueryFromTimeBasedRetentionPolicy( + TimeRetentionPolicyConfig config, + TransformCheckpoint checkpoint + ) { + Instant cutOffDate = Instant.ofEpochMilli(checkpoint.getTimestamp()).minusMillis(config.getMaxAge().getMillis()); + return QueryBuilders.rangeQuery(config.getField()).lt(cutOffDate.toEpochMilli()).format("epoch_millis"); + } +} diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformContext.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformContext.java index c13dae70db742..aae24f2eb838c 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformContext.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformContext.java @@ -31,7 +31,7 @@ public interface Listener { private final AtomicInteger failureCount; private volatile Instant changesLastDetectedAt; private volatile Instant lastSearchTime; - private volatile boolean shouldStopAtCheckpoint; + private volatile boolean shouldStopAtCheckpoint = false; // the checkpoint of this transform, storing the checkpoint until data indexing from source to dest is _complete_ // Note: Each indexer run creates a new future checkpoint which becomes the current checkpoint only after the indexer run finished @@ -43,7 +43,6 @@ public interface Listener { this.currentCheckpoint = new AtomicLong(currentCheckpoint); this.taskListener = taskListener; this.failureCount = new AtomicInteger(0); - this.shouldStopAtCheckpoint = shouldStopAtCheckpoint; } TransformTaskState getTaskState() { diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformIndexer.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformIndexer.java index 05b32c5ff4d24..68557ce21c347 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformIndexer.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/transforms/TransformIndexer.java @@ -13,6 +13,7 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.indices.refresh.RefreshResponse; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; @@ -23,6 +24,8 @@ import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.index.reindex.BulkByScrollResponse; +import org.elasticsearch.index.reindex.DeleteByQueryRequest; import org.elasticsearch.script.ScriptException; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.threadpool.ThreadPool; @@ -42,6 +45,7 @@ import org.elasticsearch.xpack.transform.notifications.TransformAuditor; import org.elasticsearch.xpack.transform.persistence.TransformConfigManager; import org.elasticsearch.xpack.transform.transforms.Function.ChangeCollector; +import org.elasticsearch.xpack.transform.transforms.RetentionPolicyToDeleteByQueryRequestConverter.RetentionPolicyException; import org.elasticsearch.xpack.transform.utils.ExceptionRootCauseFinder; import java.time.Instant; @@ -103,6 +107,7 @@ private enum RunState { // collects changes for continuous mode private ChangeCollector changeCollector; + // position of the change collector, in flux (not yet persisted as we haven't processed changes yet) private Map nextChangeCollectorBucketPosition = null; @@ -158,6 +163,10 @@ public TransformIndexer( abstract void doGetInitialProgress(SearchRequest request, ActionListener responseListener); + abstract void doDeleteByQuery(DeleteByQueryRequest deleteByQueryRequest, ActionListener responseListener); + + abstract void refreshDestinationIndex(ActionListener responseListener); + public int getPageSize() { return pageSize; } @@ -404,17 +413,92 @@ protected boolean initialRun() { @Override protected void onFinish(ActionListener listener) { startIndexerThreadShutdown(); - try { - // This indicates an early exit since no changes were found. - // So, don't treat this like a checkpoint being completed, as no work was done. - if (hasSourceChanged == false) { - if (context.shouldStopAtCheckpoint()) { - stop(); - } - listener.onResponse(null); + + // This indicates an early exit since no changes were found. + // So, don't treat this like a checkpoint being completed, as no work was done. + if (hasSourceChanged == false) { + if (context.shouldStopAtCheckpoint()) { + stop(); + } + listener.onResponse(null); + return; + } + + refreshDestinationIndex(ActionListener.wrap(response -> { + if (response.getFailedShards() > 0) { + logger.warn( + "[{}] failed to refresh transform destination index, not all data might be available after checkpoint.", + getJobId() + ); + } + // delete data defined by retention policy + if (transformConfig.getRetentionPolicyConfig() != null) { + executeRetentionPolicy(listener); + } else { + finalizeCheckpoint(listener); + } + }, listener::onFailure)); + } + + private void executeRetentionPolicy(ActionListener listener) { + DeleteByQueryRequest deleteByQuery = RetentionPolicyToDeleteByQueryRequestConverter.buildDeleteByQueryRequest( + transformConfig.getRetentionPolicyConfig(), + transformConfig.getSettings(), + transformConfig.getDestination(), + nextCheckpoint + ); + + if (deleteByQuery == null) { + finalizeCheckpoint(listener); + return; + } + + logger.debug( + () -> new ParameterizedMessage( + "[{}] Run delete based on retention policy using dbq [{}] with query: [{}]", + getJobId(), + deleteByQuery, + deleteByQuery.getSearchRequest() + ) + ); + getStats().markStartDelete(); + + doDeleteByQuery(deleteByQuery, ActionListener.wrap(bulkByScrollResponse -> { + logger.trace(() -> new ParameterizedMessage("[{}] dbq response: [{}]", getJobId(), bulkByScrollResponse)); + + getStats().markEndDelete(); + getStats().incrementNumDeletedDocuments(bulkByScrollResponse.getDeleted()); + logger.debug("[{}] deleted [{}] documents as part of the retention policy.", getJobId(), bulkByScrollResponse.getDeleted()); + + // this should not happen as part of checkpointing + if (bulkByScrollResponse.getVersionConflicts() > 0) { + // note: the failure gets logged by the failure handler + listener.onFailure( + new RetentionPolicyException( + "found [{}] version conflicts when deleting documents as part of the retention policy.", + bulkByScrollResponse.getDeleted() + ) + ); + return; + } + // paranoia: we are not expecting dbq to fail for other reasons + if (bulkByScrollResponse.getBulkFailures().size() > 0 || bulkByScrollResponse.getSearchFailures().size() > 0) { + assert false : "delete by query failed unexpectedly" + bulkByScrollResponse; + listener.onFailure( + new RetentionPolicyException( + "found failures when deleting documents as part of the retention policy. Response: [{}]", + bulkByScrollResponse + ) + ); return; } + finalizeCheckpoint(listener); + }, listener::onFailure)); + } + + private void finalizeCheckpoint(ActionListener listener) { + try { // reset the page size, so we do not memorize a low page size forever pageSize = function.getInitialPageSize(); // reset the changed bucket to free memory @@ -853,8 +937,7 @@ protected SearchRequest buildSearchRequest() { throw new IllegalStateException("Transform indexer job encountered an illegal state [" + runState + "]"); } - return new SearchRequest(getConfig().getSource().getIndex()) - .allowPartialSearchResults(false) + return new SearchRequest(getConfig().getSource().getIndex()).allowPartialSearchResults(false) .indicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN) .source(sourceBuilder); } diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/TransformInfoTransportActionTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/TransformInfoTransportActionTests.java index d1b71174337b4..fddb7c5a8c568 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/TransformInfoTransportActionTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/TransformInfoTransportActionTests.java @@ -55,18 +55,20 @@ public void testParseSearchAggs() { 1, // numPages 2, // numInputDocuments 3, // numOutputDocuments - 4, // numInvocations - 5, // indexTime - 6, // searchTime - 7, // processingTime - 8, // indexTotal - 9, // searchTotal - 10, // processingTotal - 11, // indexFailures - 12, // searchFailures - 13.0, // exponential_avg_checkpoint_duration_ms - 14.0, // exponential_avg_documents_indexed - 15.0 // exponential_avg_documents_processed + 4, // numDeletedDocuments + 5, // numInvocations + 6, // indexTime + 7, // searchTime + 8, // processingTime + 9, // deleteTime + 10, // indexTotal + 11, // searchTotal + 12, // processingTotal + 13, // indexFailures + 14, // searchFailures + 15.0, // exponential_avg_checkpoint_duration_ms + 16.0, // exponential_avg_documents_indexed + 17.0 // exponential_avg_documents_processed ); int currentStat = 1; diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/checkpoint/MockTimebasedCheckpointProvider.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/checkpoint/MockTimebasedCheckpointProvider.java index 4b51b40a9f673..b874cbf9140b5 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/checkpoint/MockTimebasedCheckpointProvider.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/checkpoint/MockTimebasedCheckpointProvider.java @@ -37,8 +37,9 @@ public void createNextCheckpoint(TransformCheckpoint lastCheckpoint, ActionListe final long timestamp = System.currentTimeMillis(); long timeUpperBound = timestamp - timeSyncConfig.getDelay().millis(); - if (lastCheckpoint == null) { - listener.onResponse(new TransformCheckpoint(transformConfig.getId(), timestamp, 0, Collections.emptyMap(), timeUpperBound)); + if (TransformCheckpoint.isNullOrEmpty(lastCheckpoint)) { + listener.onResponse(new TransformCheckpoint(transformConfig.getId(), timestamp, 1, Collections.emptyMap(), timeUpperBound)); + return; } listener.onResponse( diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/RetentionPolicyConfigToDeleteByQueryTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/RetentionPolicyConfigToDeleteByQueryTests.java new file mode 100644 index 0000000000000..8938467c0548c --- /dev/null +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/RetentionPolicyConfigToDeleteByQueryTests.java @@ -0,0 +1,99 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.transform.transforms; + +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.RangeQueryBuilder; +import org.elasticsearch.index.reindex.DeleteByQueryRequest; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.core.transform.transforms.DestConfig; +import org.elasticsearch.xpack.core.transform.transforms.DestConfigTests; +import org.elasticsearch.xpack.core.transform.transforms.RetentionPolicyConfig; +import org.elasticsearch.xpack.core.transform.transforms.SettingsConfig; +import org.elasticsearch.xpack.core.transform.transforms.SettingsConfigTests; +import org.elasticsearch.xpack.core.transform.transforms.TimeRetentionPolicyConfig; +import org.elasticsearch.xpack.core.transform.transforms.TimeRetentionPolicyConfigTests; +import org.elasticsearch.xpack.core.transform.transforms.TransformCheckpoint; + +import java.util.Collections; + +import static org.hamcrest.CoreMatchers.instanceOf; +import static org.hamcrest.Matchers.equalTo; + +public class RetentionPolicyConfigToDeleteByQueryTests extends ESTestCase { + + public void testRetentionPolicyConfigBasics() { + // null checkpoint + assertNull( + RetentionPolicyToDeleteByQueryRequestConverter.buildDeleteByQueryRequest( + randomRetentionPolicyConfig(), + SettingsConfigTests.randomSettingsConfig(), + DestConfigTests.randomDestConfig(), + null + ) + ); + + // empty checkpoint + assertNull( + RetentionPolicyToDeleteByQueryRequestConverter.buildDeleteByQueryRequest( + randomRetentionPolicyConfig(), + SettingsConfigTests.randomSettingsConfig(), + DestConfigTests.randomDestConfig(), + TransformCheckpoint.EMPTY + ) + ); + } + + public void testTimeBasedRetentionPolicyConfig() { + SettingsConfig settingsConfig = SettingsConfigTests.randomSettingsConfig(); + DestConfig destConfig = DestConfigTests.randomDestConfig(); + + RetentionPolicyConfig retentionPolicyConfig = new TimeRetentionPolicyConfig( + randomAlphaOfLengthBetween(3, 5), + new TimeValue(60_000L) + ); + TransformCheckpoint checkpoint = new TransformCheckpoint( + randomAlphaOfLengthBetween(3, 5), + 10_000_000L, + 1, + Collections.emptyMap(), + 8_000_000L + ); + DeleteByQueryRequest deleteByQueryRequest = RetentionPolicyToDeleteByQueryRequestConverter.buildDeleteByQueryRequest( + retentionPolicyConfig, + settingsConfig, + destConfig, + checkpoint + ); + + assertNotNull(deleteByQueryRequest); + assertNotNull(deleteByQueryRequest.getSearchRequest().source()); + assertNotNull(deleteByQueryRequest.getSearchRequest().source().query()); + + assertTrue(deleteByQueryRequest.isRefresh()); + assertEquals(0, deleteByQueryRequest.getMaxRetries()); + assertEquals(1, deleteByQueryRequest.getSearchRequest().indices().length); + assertEquals(destConfig.getIndex(), deleteByQueryRequest.getSearchRequest().indices()[0]); + + if (settingsConfig.getDocsPerSecond() != null) { + assertEquals(settingsConfig.getDocsPerSecond().floatValue(), deleteByQueryRequest.getRequestsPerSecond(), 1E-15); + } + + QueryBuilder query = deleteByQueryRequest.getSearchRequest().source().query(); + assertThat(query, instanceOf(RangeQueryBuilder.class)); + RangeQueryBuilder rangeQuery = (RangeQueryBuilder) query; + + assertThat(rangeQuery.to(), equalTo(9_940_000L)); + assertTrue(rangeQuery.includeLower()); + } + + private static RetentionPolicyConfig randomRetentionPolicyConfig() { + return TimeRetentionPolicyConfigTests.randomTimeRetentionPolicyConfig(); + } +} diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerFailureHandlingTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerFailureHandlingTests.java new file mode 100644 index 0000000000000..b2c9b6ae66a9a --- /dev/null +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerFailureHandlingTests.java @@ -0,0 +1,636 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.transform.transforms; + +import org.apache.lucene.search.TotalHits; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.indices.refresh.RefreshResponse; +import org.elasticsearch.action.bulk.BulkItemResponse; +import org.elasticsearch.action.bulk.BulkRequest; +import org.elasticsearch.action.bulk.BulkResponse; +import org.elasticsearch.action.search.SearchPhaseExecutionException; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.search.ShardSearchFailure; +import org.elasticsearch.client.Client; +import org.elasticsearch.common.breaker.CircuitBreaker.Durability; +import org.elasticsearch.common.breaker.CircuitBreakingException; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.index.reindex.BulkByScrollResponse; +import org.elasticsearch.index.reindex.BulkByScrollTask; +import org.elasticsearch.index.reindex.DeleteByQueryRequest; +import org.elasticsearch.script.ScriptException; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.SearchHits; +import org.elasticsearch.search.internal.InternalSearchResponse; +import org.elasticsearch.search.profile.SearchProfileShardResults; +import org.elasticsearch.search.suggest.Suggest; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.client.NoOpClient; +import org.elasticsearch.threadpool.TestThreadPool; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xpack.core.common.notifications.Level; +import org.elasticsearch.xpack.core.indexing.IndexerState; +import org.elasticsearch.xpack.core.indexing.IterationResult; +import org.elasticsearch.xpack.core.transform.transforms.QueryConfigTests; +import org.elasticsearch.xpack.core.transform.transforms.SettingsConfig; +import org.elasticsearch.xpack.core.transform.transforms.SourceConfig; +import org.elasticsearch.xpack.core.transform.transforms.SyncConfig; +import org.elasticsearch.xpack.core.transform.transforms.TimeSyncConfig; +import org.elasticsearch.xpack.core.transform.transforms.TransformCheckpoint; +import org.elasticsearch.xpack.core.transform.transforms.TransformConfig; +import org.elasticsearch.xpack.core.transform.transforms.TransformIndexerPosition; +import org.elasticsearch.xpack.core.transform.transforms.TransformIndexerStats; +import org.elasticsearch.xpack.core.transform.transforms.TransformTaskState; +import org.elasticsearch.xpack.core.transform.transforms.latest.LatestConfig; +import org.elasticsearch.xpack.transform.Transform; +import org.elasticsearch.xpack.transform.checkpoint.CheckpointProvider; +import org.elasticsearch.xpack.transform.notifications.MockTransformAuditor; +import org.elasticsearch.xpack.transform.notifications.TransformAuditor; +import org.elasticsearch.xpack.transform.persistence.IndexBasedTransformConfigManager; +import org.junit.After; +import org.junit.Before; + +import java.io.PrintWriter; +import java.io.StringWriter; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; +import java.util.function.Function; + +import static java.util.Collections.emptyMap; +import static java.util.Collections.singletonList; +import static java.util.Collections.singletonMap; +import static org.elasticsearch.xpack.core.transform.transforms.DestConfigTests.randomDestConfig; +import static org.elasticsearch.xpack.core.transform.transforms.SourceConfigTests.randomSourceConfig; +import static org.elasticsearch.xpack.core.transform.transforms.pivot.PivotConfigTests.randomPivotConfig; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.nullValue; +import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.matchesRegex; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.matches; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +public class TransformIndexerFailureHandlingTests extends ESTestCase { + + private Client client; + private ThreadPool threadPool; + + class MockedTransformIndexer extends TransformIndexer { + + private final Function searchFunction; + private final Function bulkFunction; + private final Consumer failureConsumer; + + // used for synchronizing with the test + private CountDownLatch latch; + + MockedTransformIndexer( + ThreadPool threadPool, + String executorName, + IndexBasedTransformConfigManager transformsConfigManager, + CheckpointProvider checkpointProvider, + TransformConfig transformConfig, + Map fieldMappings, + TransformAuditor auditor, + AtomicReference initialState, + TransformIndexerPosition initialPosition, + TransformIndexerStats jobStats, + TransformContext context, + Function searchFunction, + Function bulkFunction, + Consumer failureConsumer + ) { + super( + threadPool, + transformsConfigManager, + checkpointProvider, + auditor, + transformConfig, + fieldMappings, + initialState, + initialPosition, + jobStats, + /* TransformProgress */ null, + TransformCheckpoint.EMPTY, + TransformCheckpoint.EMPTY, + context + ); + this.searchFunction = searchFunction; + this.bulkFunction = bulkFunction; + this.failureConsumer = failureConsumer; + } + + public void initialize() { + this.initializeFunction(); + } + + public CountDownLatch newLatch(int count) { + return latch = new CountDownLatch(count); + } + + @Override + protected void createCheckpoint(ActionListener listener) { + listener.onResponse(TransformCheckpoint.EMPTY); + } + + @Override + protected String getJobId() { + return transformConfig.getId(); + } + + @Override + protected void doNextSearch(long waitTimeInNanos, ActionListener nextPhase) { + assert latch != null; + try { + latch.await(); + } catch (InterruptedException e) { + throw new IllegalStateException(e); + } + + try { + SearchResponse response = searchFunction.apply(buildSearchRequest()); + nextPhase.onResponse(response); + } catch (Exception e) { + nextPhase.onFailure(e); + } + } + + @Override + protected void doNextBulk(BulkRequest request, ActionListener nextPhase) { + assert latch != null; + try { + latch.await(); + } catch (InterruptedException e) { + throw new IllegalStateException(e); + } + + try { + BulkResponse response = bulkFunction.apply(request); + nextPhase.onResponse(response); + } catch (Exception e) { + nextPhase.onFailure(e); + } + } + + @Override + protected void doSaveState(IndexerState state, TransformIndexerPosition position, Runnable next) { + assert state == IndexerState.STARTED || state == IndexerState.INDEXING || state == IndexerState.STOPPED; + next.run(); + } + + @Override + protected void onFailure(Exception exc) { + try { + super.onFailure(exc); + } catch (Exception e) { + final StringWriter sw = new StringWriter(); + final PrintWriter pw = new PrintWriter(sw, true); + e.printStackTrace(pw); + fail("Unexpected failure: " + e.getMessage() + " Trace: " + sw.getBuffer().toString()); + } + } + + @Override + protected void onFinish(ActionListener listener) { + super.onFinish(listener); + listener.onResponse(null); + } + + @Override + protected void onAbort() { + fail("onAbort should not be called"); + } + + @Override + protected void failIndexer(String message) { + if (failureConsumer != null) { + failureConsumer.accept(message); + super.failIndexer(message); + } else { + fail("failIndexer should not be called, received error: " + message); + } + } + + @Override + void doGetInitialProgress(SearchRequest request, ActionListener responseListener) { + responseListener.onResponse( + new SearchResponse( + new InternalSearchResponse( + new SearchHits(new SearchHit[0], new TotalHits(0L, TotalHits.Relation.EQUAL_TO), 0.0f), + // Simulate completely null aggs + null, + new Suggest(Collections.emptyList()), + new SearchProfileShardResults(Collections.emptyMap()), + false, + false, + 1 + ), + "", + 1, + 1, + 0, + 0, + ShardSearchFailure.EMPTY_ARRAY, + SearchResponse.Clusters.EMPTY + ) + ); + } + + @Override + void doDeleteByQuery(DeleteByQueryRequest deleteByQueryRequest, ActionListener responseListener) { + responseListener.onResponse( + new BulkByScrollResponse( + TimeValue.ZERO, + new BulkByScrollTask.Status(Collections.emptyList(), null), + Collections.emptyList(), + Collections.emptyList(), + false + ) + ); + } + + @Override + void refreshDestinationIndex(ActionListener responseListener) { + responseListener.onResponse(new RefreshResponse(1, 1, 0, Collections.emptyList())); + } + } + + @Before + public void setUpMocks() { + client = new NoOpClient(getTestName()); + threadPool = new TestThreadPool(getTestName()); + } + + @After + public void tearDownClient() { + client.close(); + ThreadPool.terminate(threadPool, 30, TimeUnit.SECONDS); + } + + public void testPageSizeAdapt() throws Exception { + Integer pageSize = randomBoolean() ? null : randomIntBetween(500, 10_000); + TransformConfig config = new TransformConfig( + randomAlphaOfLength(10), + randomSourceConfig(), + randomDestConfig(), + null, + null, + null, + randomPivotConfig(), + null, + randomBoolean() ? null : randomAlphaOfLengthBetween(1, 1000), + new SettingsConfig(pageSize, null, (Boolean) null), + null, + null, + null + ); + AtomicReference state = new AtomicReference<>(IndexerState.STOPPED); + final long initialPageSize = pageSize == null ? Transform.DEFAULT_INITIAL_MAX_PAGE_SEARCH_SIZE : pageSize; + Function searchFunction = searchRequest -> { + throw new SearchPhaseExecutionException( + "query", + "Partial shards failure", + new ShardSearchFailure[] { + new ShardSearchFailure(new CircuitBreakingException("to much memory", 110, 100, Durability.TRANSIENT)) } + ); + }; + + Function bulkFunction = bulkRequest -> new BulkResponse(new BulkItemResponse[0], 100); + + TransformAuditor auditor = MockTransformAuditor.createMockAuditor(); + TransformContext context = new TransformContext(TransformTaskState.STARTED, "", 0, mock(TransformContext.Listener.class)); + + MockedTransformIndexer indexer = createMockIndexer( + config, + state, + searchFunction, + bulkFunction, + null, + threadPool, + ThreadPool.Names.GENERIC, + auditor, + context + ); + final CountDownLatch latch = indexer.newLatch(1); + indexer.start(); + assertThat(indexer.getState(), equalTo(IndexerState.STARTED)); + assertTrue(indexer.maybeTriggerAsyncJob(System.currentTimeMillis())); + assertThat(indexer.getState(), equalTo(IndexerState.INDEXING)); + + latch.countDown(); + assertBusy(() -> assertThat(indexer.getState(), equalTo(IndexerState.STARTED)), 10, TimeUnit.MINUTES); + long pageSizeAfterFirstReduction = indexer.getPageSize(); + assertThat(initialPageSize, greaterThan(pageSizeAfterFirstReduction)); + assertThat(pageSizeAfterFirstReduction, greaterThan((long) TransformIndexer.MINIMUM_PAGE_SIZE)); + + // run indexer a 2nd time + final CountDownLatch secondRunLatch = indexer.newLatch(1); + indexer.start(); + assertEquals(pageSizeAfterFirstReduction, indexer.getPageSize()); + assertThat(indexer.getState(), equalTo(IndexerState.STARTED)); + assertTrue(indexer.maybeTriggerAsyncJob(System.currentTimeMillis())); + assertThat(indexer.getState(), equalTo(IndexerState.INDEXING)); + + secondRunLatch.countDown(); + assertBusy(() -> assertThat(indexer.getState(), equalTo(IndexerState.STARTED))); + + // assert that page size has been reduced again + assertThat(pageSizeAfterFirstReduction, greaterThan((long) indexer.getPageSize())); + assertThat(pageSizeAfterFirstReduction, greaterThan((long) TransformIndexer.MINIMUM_PAGE_SIZE)); + } + + public void testDoProcessAggNullCheck() { + Integer pageSize = randomBoolean() ? null : randomIntBetween(500, 10_000); + TransformConfig config = new TransformConfig( + randomAlphaOfLength(10), + randomSourceConfig(), + randomDestConfig(), + null, + null, + null, + randomPivotConfig(), + null, + randomBoolean() ? null : randomAlphaOfLengthBetween(1, 1000), + new SettingsConfig(pageSize, null, (Boolean) null), + null, + null, + null + ); + SearchResponse searchResponse = new SearchResponse( + new InternalSearchResponse( + new SearchHits(new SearchHit[0], new TotalHits(0L, TotalHits.Relation.EQUAL_TO), 0.0f), + // Simulate completely null aggs + null, + new Suggest(Collections.emptyList()), + new SearchProfileShardResults(Collections.emptyMap()), + false, + false, + 1 + ), + "", + 1, + 1, + 0, + 0, + ShardSearchFailure.EMPTY_ARRAY, + SearchResponse.Clusters.EMPTY + ); + AtomicReference state = new AtomicReference<>(IndexerState.STOPPED); + Function searchFunction = searchRequest -> searchResponse; + Function bulkFunction = bulkRequest -> new BulkResponse(new BulkItemResponse[0], 100); + + TransformAuditor auditor = mock(TransformAuditor.class); + TransformContext context = new TransformContext(TransformTaskState.STARTED, "", 0, mock(TransformContext.Listener.class)); + + MockedTransformIndexer indexer = createMockIndexer( + config, + state, + searchFunction, + bulkFunction, + null, + threadPool, + ThreadPool.Names.GENERIC, + auditor, + context + ); + + IterationResult newPosition = indexer.doProcess(searchResponse); + assertThat(newPosition.getToIndex(), is(empty())); + assertThat(newPosition.getPosition(), is(nullValue())); + assertThat(newPosition.isDone(), is(true)); + } + + public void testScriptError() throws Exception { + Integer pageSize = randomBoolean() ? null : randomIntBetween(500, 10_000); + String transformId = randomAlphaOfLength(10); + TransformConfig config = new TransformConfig( + transformId, + randomSourceConfig(), + randomDestConfig(), + null, + null, + null, + randomPivotConfig(), + null, + randomBoolean() ? null : randomAlphaOfLengthBetween(1, 1000), + new SettingsConfig(pageSize, null, (Boolean) null), + null, + null, + null + ); + AtomicReference state = new AtomicReference<>(IndexerState.STOPPED); + Function searchFunction = searchRequest -> { + throw new SearchPhaseExecutionException( + "query", + "Partial shards failure", + new ShardSearchFailure[] { + new ShardSearchFailure( + new ScriptException( + "runtime error", + new ArithmeticException("/ by zero"), + singletonList("stack"), + "test", + "painless" + ) + ) } + + ); + }; + + Function bulkFunction = bulkRequest -> new BulkResponse(new BulkItemResponse[0], 100); + + final AtomicBoolean failIndexerCalled = new AtomicBoolean(false); + final AtomicReference failureMessage = new AtomicReference<>(); + Consumer failureConsumer = message -> { + failIndexerCalled.compareAndSet(false, true); + failureMessage.compareAndSet(null, message); + }; + + MockTransformAuditor auditor = MockTransformAuditor.createMockAuditor(); + TransformContext.Listener contextListener = mock(TransformContext.Listener.class); + TransformContext context = new TransformContext(TransformTaskState.STARTED, "", 0, contextListener); + + MockedTransformIndexer indexer = createMockIndexer( + config, + state, + searchFunction, + bulkFunction, + failureConsumer, + threadPool, + ThreadPool.Names.GENERIC, + auditor, + context + ); + + final CountDownLatch latch = indexer.newLatch(1); + + indexer.start(); + assertThat(indexer.getState(), equalTo(IndexerState.STARTED)); + assertTrue(indexer.maybeTriggerAsyncJob(System.currentTimeMillis())); + assertThat(indexer.getState(), equalTo(IndexerState.INDEXING)); + + latch.countDown(); + assertBusy(() -> assertThat(indexer.getState(), equalTo(IndexerState.STARTED)), 10, TimeUnit.SECONDS); + assertTrue(failIndexerCalled.get()); + verify(contextListener, times(1)).fail( + matches("Failed to execute script with error: \\[.*ArithmeticException: / by zero\\], stack trace: \\[stack\\]"), + any() + ); + + assertThat( + failureMessage.get(), + matchesRegex("Failed to execute script with error: \\[.*ArithmeticException: / by zero\\], stack trace: \\[stack\\]") + ); + } + + public void testInitializeFunction_WithNoWarnings() { + String transformId = randomAlphaOfLength(10); + SourceConfig sourceConfig = new SourceConfig( + generateRandomStringArray(10, 10, false, false), + QueryConfigTests.randomQueryConfig(), + new HashMap<>() { + { + put("field-A", singletonMap("script", "some script")); + put("field-B", emptyMap()); + put("field-C", singletonMap("script", "some script")); + } + } + ); + SyncConfig syncConfig = new TimeSyncConfig("field", null); + LatestConfig latestConfig = new LatestConfig(Arrays.asList("field-A", "field-B"), "sort"); + TransformConfig config = new TransformConfig( + transformId, + sourceConfig, + randomDestConfig(), + null, + syncConfig, + null, + null, + latestConfig, + null, + null, + null, + null, + null + ); + + MockTransformAuditor auditor = MockTransformAuditor.createMockAuditor(); + auditor.addExpectation( + new MockTransformAuditor.UnseenAuditExpectation( + "warn when all the group-by fields are script-based runtime fields", + Level.WARNING, + transformId, + "all the group-by fields are script-based runtime fields" + ) + ); + TransformContext.Listener contextListener = mock(TransformContext.Listener.class); + TransformContext context = new TransformContext(TransformTaskState.STARTED, "", 0, contextListener); + createMockIndexer(config, null, null, null, null, threadPool, ThreadPool.Names.GENERIC, auditor, context); + auditor.assertAllExpectationsMatched(); + } + + public void testInitializeFunction_WithWarnings() { + String transformId = randomAlphaOfLength(10); + SourceConfig sourceConfig = new SourceConfig( + generateRandomStringArray(10, 10, false, false), + QueryConfigTests.randomQueryConfig(), + new HashMap<>() { + { + put("field-A", singletonMap("script", "some script")); + put("field-B", singletonMap("script", "some script")); + put("field-C", singletonMap("script", "some script")); + put("field-t", singletonMap("script", "some script")); + } + } + ); + SyncConfig syncConfig = new TimeSyncConfig("field-t", null); + LatestConfig latestConfig = new LatestConfig(Arrays.asList("field-A", "field-B"), "sort"); + TransformConfig config = new TransformConfig( + transformId, + sourceConfig, + randomDestConfig(), + null, + syncConfig, + null, + null, + latestConfig, + null, + null, + null, + null, + null + ); + + MockTransformAuditor auditor = MockTransformAuditor.createMockAuditor(); + auditor.addExpectation( + new MockTransformAuditor.SeenAuditExpectation( + "warn when all the group-by fields are script-based runtime fields", + Level.WARNING, + transformId, + "all the group-by fields are script-based runtime fields" + ) + ); + auditor.addExpectation( + new MockTransformAuditor.SeenAuditExpectation( + "warn when the sync time field is a script-based runtime field", + Level.WARNING, + transformId, + "sync time field is a script-based runtime field" + ) + ); + TransformContext.Listener contextListener = mock(TransformContext.Listener.class); + TransformContext context = new TransformContext(TransformTaskState.STARTED, "", 0, contextListener); + createMockIndexer(config, null, null, null, null, threadPool, ThreadPool.Names.GENERIC, auditor, context); + auditor.assertAllExpectationsMatched(); + } + + private MockedTransformIndexer createMockIndexer( + TransformConfig config, + AtomicReference state, + Function searchFunction, + Function bulkFunction, + Consumer failureConsumer, + ThreadPool threadPool, + String executorName, + TransformAuditor auditor, + TransformContext context + ) { + MockedTransformIndexer indexer = new MockedTransformIndexer( + threadPool, + executorName, + mock(IndexBasedTransformConfigManager.class), + mock(CheckpointProvider.class), + config, + Collections.emptyMap(), + auditor, + state, + null, + new TransformIndexerStats(), + context, + searchFunction, + bulkFunction, + failureConsumer + ); + + indexer.initialize(); + return indexer; + } + +} diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerStateTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerStateTests.java index 922d19b19afa4..e1d6cf594dbe5 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerStateTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerStateTests.java @@ -10,6 +10,7 @@ import org.apache.lucene.search.TotalHits; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.LatchedActionListener; +import org.elasticsearch.action.admin.indices.refresh.RefreshResponse; import org.elasticsearch.action.bulk.BulkItemResponse; import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.bulk.BulkResponse; @@ -19,6 +20,9 @@ import org.elasticsearch.action.search.ShardSearchFailure; import org.elasticsearch.client.Client; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.index.reindex.BulkByScrollResponse; +import org.elasticsearch.index.reindex.BulkByScrollTask; +import org.elasticsearch.index.reindex.DeleteByQueryRequest; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; import org.elasticsearch.search.internal.InternalSearchResponse; @@ -160,6 +164,24 @@ void doGetInitialProgress(SearchRequest request, ActionListener responseListener.onResponse(ONE_HIT_SEARCH_RESPONSE); } + @Override + void doDeleteByQuery(DeleteByQueryRequest deleteByQueryRequest, ActionListener responseListener) { + responseListener.onResponse( + new BulkByScrollResponse( + TimeValue.ZERO, + new BulkByScrollTask.Status(Collections.emptyList(), null), + Collections.emptyList(), + Collections.emptyList(), + false + ) + ); + } + + @Override + void refreshDestinationIndex(ActionListener responseListener) { + responseListener.onResponse(new RefreshResponse(1, 1, 0, Collections.emptyList())); + } + @Override protected void doNextSearch(long waitTimeInNanos, ActionListener nextPhase) { if (searchLatch != null) { @@ -254,6 +276,7 @@ public void testStopAtCheckpoint() throws Exception { randomBoolean() ? null : randomAlphaOfLengthBetween(1, 1000), null, null, + null, null ); @@ -460,6 +483,7 @@ public void testStopAtCheckpointForThrottledTransform() throws Exception { randomBoolean() ? null : randomAlphaOfLengthBetween(1, 1000), new SettingsConfig(null, Float.valueOf(1.0f), (Boolean) null), null, + null, null ); AtomicReference state = new AtomicReference<>(IndexerState.STARTED); diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerTests.java index 3f779ff7d9ae2..433aaeabb3235 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformIndexerTests.java @@ -9,17 +9,19 @@ import org.apache.lucene.search.TotalHits; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.indices.refresh.RefreshResponse; import org.elasticsearch.action.bulk.BulkItemResponse; import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.bulk.BulkResponse; -import org.elasticsearch.action.search.SearchPhaseExecutionException; +import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.ShardSearchFailure; import org.elasticsearch.client.Client; -import org.elasticsearch.common.breaker.CircuitBreaker.Durability; -import org.elasticsearch.common.breaker.CircuitBreakingException; -import org.elasticsearch.script.ScriptException; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.index.reindex.BulkByScrollResponse; +import org.elasticsearch.index.reindex.BulkByScrollTask; +import org.elasticsearch.index.reindex.DeleteByQueryRequest; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; import org.elasticsearch.search.internal.InternalSearchResponse; @@ -29,88 +31,88 @@ import org.elasticsearch.test.client.NoOpClient; import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.xpack.core.common.notifications.Level; import org.elasticsearch.xpack.core.indexing.IndexerState; import org.elasticsearch.xpack.core.indexing.IterationResult; -import org.elasticsearch.xpack.core.transform.transforms.QueryConfigTests; -import org.elasticsearch.xpack.core.transform.transforms.SettingsConfig; -import org.elasticsearch.xpack.core.transform.transforms.SourceConfig; -import org.elasticsearch.xpack.core.transform.transforms.SyncConfig; +import org.elasticsearch.xpack.core.transform.transforms.TimeRetentionPolicyConfigTests; import org.elasticsearch.xpack.core.transform.transforms.TimeSyncConfig; import org.elasticsearch.xpack.core.transform.transforms.TransformCheckpoint; import org.elasticsearch.xpack.core.transform.transforms.TransformConfig; import org.elasticsearch.xpack.core.transform.transforms.TransformIndexerPosition; import org.elasticsearch.xpack.core.transform.transforms.TransformIndexerStats; import org.elasticsearch.xpack.core.transform.transforms.TransformTaskState; -import org.elasticsearch.xpack.core.transform.transforms.latest.LatestConfig; -import org.elasticsearch.xpack.transform.Transform; import org.elasticsearch.xpack.transform.checkpoint.CheckpointProvider; +import org.elasticsearch.xpack.transform.checkpoint.MockTimebasedCheckpointProvider; import org.elasticsearch.xpack.transform.notifications.MockTransformAuditor; import org.elasticsearch.xpack.transform.notifications.TransformAuditor; -import org.elasticsearch.xpack.transform.persistence.IndexBasedTransformConfigManager; +import org.elasticsearch.xpack.transform.persistence.InMemoryTransformConfigManager; +import org.elasticsearch.xpack.transform.persistence.TransformConfigManager; import org.junit.After; import org.junit.Before; -import java.io.PrintWriter; -import java.io.StringWriter; -import java.util.Arrays; import java.util.Collections; -import java.util.HashMap; import java.util.Map; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; -import java.util.function.Function; -import static java.util.Collections.emptyMap; -import static java.util.Collections.singletonList; -import static java.util.Collections.singletonMap; import static org.elasticsearch.xpack.core.transform.transforms.DestConfigTests.randomDestConfig; import static org.elasticsearch.xpack.core.transform.transforms.SourceConfigTests.randomSourceConfig; import static org.elasticsearch.xpack.core.transform.transforms.pivot.PivotConfigTests.randomPivotConfig; -import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.CoreMatchers.nullValue; -import static org.hamcrest.Matchers.empty; -import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; -import static org.hamcrest.Matchers.matchesRegex; -import static org.mockito.Matchers.any; -import static org.mockito.Matchers.matches; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; public class TransformIndexerTests extends ESTestCase { + private static final SearchResponse ONE_HIT_SEARCH_RESPONSE = new SearchResponse( + new InternalSearchResponse( + new SearchHits(new SearchHit[] { new SearchHit(1) }, new TotalHits(1L, TotalHits.Relation.EQUAL_TO), 1.0f), + // Simulate completely null aggs + null, + new Suggest(Collections.emptyList()), + new SearchProfileShardResults(Collections.emptyMap()), + false, + false, + 1 + ), + "", + 1, + 1, + 0, + 0, + ShardSearchFailure.EMPTY_ARRAY, + SearchResponse.Clusters.EMPTY + ); + private Client client; private ThreadPool threadPool; + private TransformAuditor auditor; + private TransformConfigManager transformConfigManager; class MockedTransformIndexer extends TransformIndexer { - private final Function searchFunction; - private final Function bulkFunction; - private final Consumer failureConsumer; + private final ThreadPool threadPool; + private int deleteByQueryCallCount = 0; // used for synchronizing with the test - private CountDownLatch latch; + private CountDownLatch searchLatch; + private CountDownLatch doProcessLatch; + + // how many loops to execute until reporting done + private int numberOfLoops; MockedTransformIndexer( + int numberOfLoops, ThreadPool threadPool, - String executorName, - IndexBasedTransformConfigManager transformsConfigManager, + TransformConfigManager transformsConfigManager, CheckpointProvider checkpointProvider, + TransformAuditor auditor, TransformConfig transformConfig, Map fieldMappings, - TransformAuditor auditor, AtomicReference initialState, TransformIndexerPosition initialPosition, TransformIndexerStats jobStats, - TransformContext context, - Function searchFunction, - Function bulkFunction, - Consumer failureConsumer + TransformContext context ) { super( threadPool, @@ -127,133 +129,122 @@ class MockedTransformIndexer extends TransformIndexer { TransformCheckpoint.EMPTY, context ); - this.searchFunction = searchFunction; - this.bulkFunction = bulkFunction; - this.failureConsumer = failureConsumer; + this.threadPool = threadPool; + this.numberOfLoops = numberOfLoops; } public void initialize() { this.initializeFunction(); } - public CountDownLatch newLatch(int count) { - return latch = new CountDownLatch(count); + public CountDownLatch createAwaitForSearchLatch(int count) { + return searchLatch = new CountDownLatch(count); } - @Override - protected void createCheckpoint(ActionListener listener) { - listener.onResponse(TransformCheckpoint.EMPTY); + public CountDownLatch createCountDownOnResponseLatch(int count) { + return doProcessLatch = new CountDownLatch(count); } @Override - protected String getJobId() { - return transformConfig.getId(); + void doGetInitialProgress(SearchRequest request, ActionListener responseListener) { + responseListener.onResponse(ONE_HIT_SEARCH_RESPONSE); } @Override - protected void doNextSearch(long waitTimeInNanos, ActionListener nextPhase) { - assert latch != null; + void doDeleteByQuery(DeleteByQueryRequest deleteByQueryRequest, ActionListener responseListener) { + deleteByQueryCallCount++; try { - latch.await(); + // yes, I know, a sleep, how dare you, this is to test stats collection and this requires a resolution of a millisecond + Thread.sleep(1); } catch (InterruptedException e) { - throw new IllegalStateException(e); - } - - try { - SearchResponse response = searchFunction.apply(buildSearchRequest()); - nextPhase.onResponse(response); - } catch (Exception e) { - nextPhase.onFailure(e); + fail("unexpected exception during sleep: " + e); } + responseListener.onResponse( + new BulkByScrollResponse( + TimeValue.ZERO, + new BulkByScrollTask.Status( + 0, + 0L, + 0L, + 0L, + /*deleted*/ 42L, + 0, + 0L, + 0L, + 0L, + 0L, + TimeValue.ZERO, + 0.0f, + null, + TimeValue.ZERO + ), + Collections.emptyList(), + Collections.emptyList(), + false + ) + ); } @Override - protected void doNextBulk(BulkRequest request, ActionListener nextPhase) { - assert latch != null; - try { - latch.await(); - } catch (InterruptedException e) { - throw new IllegalStateException(e); - } - - try { - BulkResponse response = bulkFunction.apply(request); - nextPhase.onResponse(response); - } catch (Exception e) { - nextPhase.onFailure(e); - } + void refreshDestinationIndex(ActionListener responseListener) { + responseListener.onResponse(new RefreshResponse(1, 1, 0, Collections.emptyList())); } @Override - protected void doSaveState(IndexerState state, TransformIndexerPosition position, Runnable next) { - assert state == IndexerState.STARTED || state == IndexerState.INDEXING || state == IndexerState.STOPPED; - next.run(); + protected void doNextSearch(long waitTimeInNanos, ActionListener nextPhase) { + if (searchLatch != null) { + try { + searchLatch.await(); + } catch (InterruptedException e) { + throw new IllegalStateException(e); + } + } + threadPool.executor(ThreadPool.Names.GENERIC).execute(() -> nextPhase.onResponse(ONE_HIT_SEARCH_RESPONSE)); } @Override - protected void onFailure(Exception exc) { - try { - super.onFailure(exc); - } catch (Exception e) { - final StringWriter sw = new StringWriter(); - final PrintWriter pw = new PrintWriter(sw, true); - e.printStackTrace(pw); - fail("Unexpected failure: " + e.getMessage() + " Trace: " + sw.getBuffer().toString()); + protected void doNextBulk(BulkRequest request, ActionListener nextPhase) { + if (doProcessLatch != null) { + doProcessLatch.countDown(); } + threadPool.executor(ThreadPool.Names.GENERIC) + .execute(() -> nextPhase.onResponse(new BulkResponse(new BulkItemResponse[0], 100))); } @Override - protected void onFinish(ActionListener listener) { - super.onFinish(listener); - listener.onResponse(null); + protected void doSaveState(IndexerState state, TransformIndexerPosition position, Runnable next) { + assert state == IndexerState.STARTED || state == IndexerState.INDEXING || state == IndexerState.STOPPED; + next.run(); } @Override - protected void onAbort() { - fail("onAbort should not be called"); + protected IterationResult doProcess(SearchResponse searchResponse) { + assert numberOfLoops > 0; + --numberOfLoops; + // pretend that we processed 10k documents for each call + getStats().incrementNumDocuments(10_000); + return new IterationResult<>( + Collections.singletonList(new IndexRequest()), + new TransformIndexerPosition(null, null), + numberOfLoops == 0 + ); } - @Override - protected void failIndexer(String message) { - if (failureConsumer != null) { - failureConsumer.accept(message); - super.failIndexer(message); - } else { - fail("failIndexer should not be called, received error: " + message); - } + public boolean waitingForNextSearch() { + return super.getScheduledNextSearch() != null; } - @Override - void doGetInitialProgress(SearchRequest request, ActionListener responseListener) { - responseListener.onResponse( - new SearchResponse( - new InternalSearchResponse( - new SearchHits(new SearchHit[0], new TotalHits(0L, TotalHits.Relation.EQUAL_TO), 0.0f), - // Simulate completely null aggs - null, - new Suggest(Collections.emptyList()), - new SearchProfileShardResults(Collections.emptyMap()), - false, - false, - 1 - ), - "", - 1, - 1, - 0, - 0, - ShardSearchFailure.EMPTY_ARRAY, - SearchResponse.Clusters.EMPTY - ) - ); + public int getDeleteByQueryCallCount() { + return deleteByQueryCallCount; } - } @Before public void setUpMocks() { + auditor = MockTransformAuditor.createMockAuditor(); + transformConfigManager = new InMemoryTransformConfigManager(); client = new NoOpClient(getTestName()); - threadPool = new TestThreadPool(getTestName()); + threadPool = new TestThreadPool(ThreadPool.Names.GENERIC); } @After @@ -262,316 +253,120 @@ public void tearDownClient() { ThreadPool.terminate(threadPool, 30, TimeUnit.SECONDS); } - public void testPageSizeAdapt() throws Exception { - Integer pageSize = randomBoolean() ? null : randomIntBetween(500, 10_000); + public void testRetentionPolicyExecution() throws Exception { TransformConfig config = new TransformConfig( randomAlphaOfLength(10), randomSourceConfig(), randomDestConfig(), null, - null, + new TimeSyncConfig("timestamp", TimeValue.timeValueSeconds(1)), null, randomPivotConfig(), null, randomBoolean() ? null : randomAlphaOfLengthBetween(1, 1000), - new SettingsConfig(pageSize, null, (Boolean) null), + null, + TimeRetentionPolicyConfigTests.randomTimeRetentionPolicyConfig(), null, null ); - AtomicReference state = new AtomicReference<>(IndexerState.STOPPED); - final long initialPageSize = pageSize == null ? Transform.DEFAULT_INITIAL_MAX_PAGE_SEARCH_SIZE : pageSize; - Function searchFunction = searchRequest -> { - throw new SearchPhaseExecutionException( - "query", - "Partial shards failure", - new ShardSearchFailure[] { - new ShardSearchFailure(new CircuitBreakingException("to much memory", 110, 100, Durability.TRANSIENT)) } + AtomicReference state = new AtomicReference<>(IndexerState.STARTED); + { + TransformContext context = new TransformContext(TransformTaskState.STARTED, "", 0, mock(TransformContext.Listener.class)); + final MockedTransformIndexer indexer = createMockIndexer( + 10, + config, + state, + null, + threadPool, + auditor, + new TransformIndexerStats(), + context ); - }; - Function bulkFunction = bulkRequest -> new BulkResponse(new BulkItemResponse[0], 100); + indexer.start(); + assertTrue(indexer.maybeTriggerAsyncJob(System.currentTimeMillis())); + assertEquals(indexer.getState(), IndexerState.INDEXING); - TransformAuditor auditor = MockTransformAuditor.createMockAuditor(); - TransformContext context = new TransformContext(TransformTaskState.STARTED, "", 0, mock(TransformContext.Listener.class)); + assertBusy(() -> assertEquals(1L, indexer.getLastCheckpoint().getCheckpoint()), 5, TimeUnit.HOURS); - MockedTransformIndexer indexer = createMockIndexer( - config, - state, - searchFunction, - bulkFunction, - null, - threadPool, - ThreadPool.Names.GENERIC, - auditor, - context - ); - final CountDownLatch latch = indexer.newLatch(1); - indexer.start(); - assertThat(indexer.getState(), equalTo(IndexerState.STARTED)); - assertTrue(indexer.maybeTriggerAsyncJob(System.currentTimeMillis())); - assertThat(indexer.getState(), equalTo(IndexerState.INDEXING)); - - latch.countDown(); - assertBusy(() -> assertThat(indexer.getState(), equalTo(IndexerState.STARTED)), 10, TimeUnit.MINUTES); - long pageSizeAfterFirstReduction = indexer.getPageSize(); - assertThat(initialPageSize, greaterThan(pageSizeAfterFirstReduction)); - assertThat(pageSizeAfterFirstReduction, greaterThan((long) TransformIndexer.MINIMUM_PAGE_SIZE)); - - // run indexer a 2nd time - final CountDownLatch secondRunLatch = indexer.newLatch(1); - indexer.start(); - assertEquals(pageSizeAfterFirstReduction, indexer.getPageSize()); - assertThat(indexer.getState(), equalTo(IndexerState.STARTED)); - assertTrue(indexer.maybeTriggerAsyncJob(System.currentTimeMillis())); - assertThat(indexer.getState(), equalTo(IndexerState.INDEXING)); - - secondRunLatch.countDown(); - assertBusy(() -> assertThat(indexer.getState(), equalTo(IndexerState.STARTED))); - - // assert that page size has been reduced again - assertThat(pageSizeAfterFirstReduction, greaterThan((long) indexer.getPageSize())); - assertThat(pageSizeAfterFirstReduction, greaterThan((long) TransformIndexer.MINIMUM_PAGE_SIZE)); - } + // delete by query has been executed + assertEquals(1, indexer.getDeleteByQueryCallCount()); + assertEquals(42L, indexer.getStats().getNumDeletedDocuments()); + assertThat(indexer.getStats().getDeleteTime(), greaterThan(0L)); + } - public void testDoProcessAggNullCheck() { - Integer pageSize = randomBoolean() ? null : randomIntBetween(500, 10_000); - TransformConfig config = new TransformConfig( + // test without retention + config = new TransformConfig( randomAlphaOfLength(10), randomSourceConfig(), randomDestConfig(), null, - null, + new TimeSyncConfig("timestamp", TimeValue.timeValueSeconds(1)), null, randomPivotConfig(), null, randomBoolean() ? null : randomAlphaOfLengthBetween(1, 1000), - new SettingsConfig(pageSize, null, (Boolean) null), - null, - null - ); - SearchResponse searchResponse = new SearchResponse( - new InternalSearchResponse( - new SearchHits(new SearchHit[0], new TotalHits(0L, TotalHits.Relation.EQUAL_TO), 0.0f), - // Simulate completely null aggs - null, - new Suggest(Collections.emptyList()), - new SearchProfileShardResults(Collections.emptyMap()), - false, - false, - 1 - ), - "", - 1, - 1, - 0, - 0, - ShardSearchFailure.EMPTY_ARRAY, - SearchResponse.Clusters.EMPTY - ); - AtomicReference state = new AtomicReference<>(IndexerState.STOPPED); - Function searchFunction = searchRequest -> searchResponse; - Function bulkFunction = bulkRequest -> new BulkResponse(new BulkItemResponse[0], 100); - - TransformAuditor auditor = mock(TransformAuditor.class); - TransformContext context = new TransformContext(TransformTaskState.STARTED, "", 0, mock(TransformContext.Listener.class)); - - MockedTransformIndexer indexer = createMockIndexer( - config, - state, - searchFunction, - bulkFunction, null, - threadPool, - ThreadPool.Names.GENERIC, - auditor, - context - ); - - IterationResult newPosition = indexer.doProcess(searchResponse); - assertThat(newPosition.getToIndex(), is(empty())); - assertThat(newPosition.getPosition(), is(nullValue())); - assertThat(newPosition.isDone(), is(true)); - } - - public void testScriptError() throws Exception { - Integer pageSize = randomBoolean() ? null : randomIntBetween(500, 10_000); - String transformId = randomAlphaOfLength(10); - TransformConfig config = new TransformConfig( - transformId, - randomSourceConfig(), - randomDestConfig(), - null, - null, - null, - randomPivotConfig(), null, - randomBoolean() ? null : randomAlphaOfLengthBetween(1, 1000), - new SettingsConfig(pageSize, null, (Boolean) null), null, null ); - AtomicReference state = new AtomicReference<>(IndexerState.STOPPED); - Function searchFunction = searchRequest -> { - throw new SearchPhaseExecutionException( - "query", - "Partial shards failure", - new ShardSearchFailure[] { - new ShardSearchFailure( - new ScriptException( - "runtime error", - new ArithmeticException("/ by zero"), - singletonList("stack"), - "test", - "painless" - ) - ) } + state = new AtomicReference<>(IndexerState.STARTED); + { + TransformContext context = new TransformContext(TransformTaskState.STARTED, "", 0, mock(TransformContext.Listener.class)); + final MockedTransformIndexer indexer = createMockIndexer( + 10, + config, + state, + null, + threadPool, + auditor, + new TransformIndexerStats(), + context ); - }; - - Function bulkFunction = bulkRequest -> new BulkResponse(new BulkItemResponse[0], 100); - - final AtomicBoolean failIndexerCalled = new AtomicBoolean(false); - final AtomicReference failureMessage = new AtomicReference<>(); - Consumer failureConsumer = message -> { - failIndexerCalled.compareAndSet(false, true); - failureMessage.compareAndSet(null, message); - }; - - MockTransformAuditor auditor = MockTransformAuditor.createMockAuditor(); - TransformContext.Listener contextListener = mock(TransformContext.Listener.class); - TransformContext context = new TransformContext(TransformTaskState.STARTED, "", 0, contextListener); - - MockedTransformIndexer indexer = createMockIndexer( - config, - state, - searchFunction, - bulkFunction, - failureConsumer, - threadPool, - ThreadPool.Names.GENERIC, - auditor, - context - ); - - final CountDownLatch latch = indexer.newLatch(1); - indexer.start(); - assertThat(indexer.getState(), equalTo(IndexerState.STARTED)); - assertTrue(indexer.maybeTriggerAsyncJob(System.currentTimeMillis())); - assertThat(indexer.getState(), equalTo(IndexerState.INDEXING)); - - latch.countDown(); - assertBusy(() -> assertThat(indexer.getState(), equalTo(IndexerState.STARTED)), 10, TimeUnit.SECONDS); - assertTrue(failIndexerCalled.get()); - verify(contextListener, times(1)).fail( - matches("Failed to execute script with error: \\[.*ArithmeticException: / by zero\\], stack trace: \\[stack\\]"), - any() - ); - - assertThat( - failureMessage.get(), - matchesRegex("Failed to execute script with error: \\[.*ArithmeticException: / by zero\\], stack trace: \\[stack\\]") - ); - } + indexer.start(); + assertTrue(indexer.maybeTriggerAsyncJob(System.currentTimeMillis())); + assertEquals(indexer.getState(), IndexerState.INDEXING); - public void testInitializeFunction_WithNoWarnings() { - String transformId = randomAlphaOfLength(10); - SourceConfig sourceConfig = - new SourceConfig( - generateRandomStringArray(10, 10, false, false), - QueryConfigTests.randomQueryConfig(), - new HashMap<>() {{ - put("field-A", singletonMap("script", "some script")); - put("field-B", emptyMap()); - put("field-C", singletonMap("script", "some script")); - }}); - SyncConfig syncConfig = new TimeSyncConfig("field", null); - LatestConfig latestConfig = new LatestConfig(Arrays.asList("field-A", "field-B"), "sort"); - TransformConfig config = - new TransformConfig( - transformId, sourceConfig, randomDestConfig(), null, syncConfig, null, null, latestConfig, null, null, null, null); - - MockTransformAuditor auditor = MockTransformAuditor.createMockAuditor(); - auditor.addExpectation( - new MockTransformAuditor.UnseenAuditExpectation( - "warn when all the group-by fields are script-based runtime fields", - Level.WARNING, - transformId, - "all the group-by fields are script-based runtime fields")); - TransformContext.Listener contextListener = mock(TransformContext.Listener.class); - TransformContext context = new TransformContext(TransformTaskState.STARTED, "", 0, contextListener); - createMockIndexer(config, null, null, null, null, threadPool, ThreadPool.Names.GENERIC, auditor, context); - auditor.assertAllExpectationsMatched(); - } + assertBusy(() -> assertEquals(1L, indexer.getLastCheckpoint().getCheckpoint()), 5, TimeUnit.SECONDS); - public void testInitializeFunction_WithWarnings() { - String transformId = randomAlphaOfLength(10); - SourceConfig sourceConfig = - new SourceConfig( - generateRandomStringArray(10, 10, false, false), - QueryConfigTests.randomQueryConfig(), - new HashMap<>() {{ - put("field-A", singletonMap("script", "some script")); - put("field-B", singletonMap("script", "some script")); - put("field-C", singletonMap("script", "some script")); - put("field-t", singletonMap("script", "some script")); - }}); - SyncConfig syncConfig = new TimeSyncConfig("field-t", null); - LatestConfig latestConfig = new LatestConfig(Arrays.asList("field-A", "field-B"), "sort"); - TransformConfig config = - new TransformConfig( - transformId, sourceConfig, randomDestConfig(), null, syncConfig, null, null, latestConfig, null, null, null, null); - - MockTransformAuditor auditor = MockTransformAuditor.createMockAuditor(); - auditor.addExpectation( - new MockTransformAuditor.SeenAuditExpectation( - "warn when all the group-by fields are script-based runtime fields", - Level.WARNING, - transformId, - "all the group-by fields are script-based runtime fields")); - auditor.addExpectation( - new MockTransformAuditor.SeenAuditExpectation( - "warn when the sync time field is a script-based runtime field", - Level.WARNING, - transformId, - "sync time field is a script-based runtime field")); - TransformContext.Listener contextListener = mock(TransformContext.Listener.class); - TransformContext context = new TransformContext(TransformTaskState.STARTED, "", 0, contextListener); - createMockIndexer(config, null, null, null, null, threadPool, ThreadPool.Names.GENERIC, auditor, context); - auditor.assertAllExpectationsMatched(); + // delete by query has _not_ been executed + assertEquals(0, indexer.getDeleteByQueryCallCount()); + assertEquals(0L, indexer.getStats().getNumDeletedDocuments()); + assertEquals(0L, indexer.getStats().getDeleteTime()); + } } private MockedTransformIndexer createMockIndexer( + int numberOfLoops, TransformConfig config, AtomicReference state, - Function searchFunction, - Function bulkFunction, Consumer failureConsumer, ThreadPool threadPool, - String executorName, TransformAuditor auditor, + TransformIndexerStats jobStats, TransformContext context ) { + CheckpointProvider checkpointProvider = new MockTimebasedCheckpointProvider(config); + transformConfigManager.putTransformConfiguration(config, ActionListener.wrap(r -> {}, e -> {})); + MockedTransformIndexer indexer = new MockedTransformIndexer( + numberOfLoops, threadPool, - executorName, - mock(IndexBasedTransformConfigManager.class), - mock(CheckpointProvider.class), + transformConfigManager, + checkpointProvider, + auditor, config, Collections.emptyMap(), - auditor, state, null, - new TransformIndexerStats(), - context, - searchFunction, - bulkFunction, - failureConsumer + jobStats, + context ); indexer.initialize(); return indexer; } - } diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/80_transform_jobs_crud.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/80_transform_jobs_crud.yml index 9d2f27565d2ad..0993d3c9f3bb9 100644 --- a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/80_transform_jobs_crud.yml +++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/80_transform_jobs_crud.yml @@ -298,11 +298,11 @@ setup: - do: warnings: - - "this request accesses system indices: [.transform-internal-005], but in a future major version, direct access to system indices will be prevented by default" + - "this request accesses system indices: [.transform-internal-006], but in a future major version, direct access to system indices will be prevented by default" indices.get_mapping: - index: .transform-internal-005 - - match: { \.transform-internal-005.mappings.dynamic: "false" } - - match: { \.transform-internal-005.mappings.properties.id.type: "keyword" } + index: .transform-internal-006 + - match: { \.transform-internal-006.mappings.dynamic: "false" } + - match: { \.transform-internal-006.mappings.properties.id.type: "keyword" } - do: indices.get_mapping: index: .transform-notifications-000002