[Transform] add retention policy to documentation IT (#68728)

Hendrik Muhs · Hendrik Muhs · commit a0bf31576244 · 2021-02-11T16:00:33.000+01:00
add retention policy and sync to documentation IT, add builders for sync and retention policy relates #67832
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/TimeRetentionPolicyConfig.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/TimeRetentionPolicyConfig.java
@@ -50,7 +50,7 @@ public static TimeRetentionPolicyConfig fromXContent(XContentParser parser) {
         return PARSER.apply(parser, null);
     }
 
-    public TimeRetentionPolicyConfig(String field, TimeValue maxAge) {
+    TimeRetentionPolicyConfig(String field, TimeValue maxAge) {
         this.field = field;
         this.maxAge = maxAge;
     }
@@ -96,4 +96,37 @@ public int hashCode() {
     public String getName() {
         return NAME;
     }
+
+    public static Builder builder() {
+        return new Builder();
+    }
+
+    public static class Builder {
+        private String field;
+        private TimeValue maxAge;
+
+        /**
+         * The time field used to calculate the age of a document.
+         * @param field The field name to be used to execute the retention policy
+         * @return The {@link Builder} with the field set.
+         */
+        public Builder setField(String field) {
+            this.field = field;
+            return this;
+        }
+
+        /**
+         * The max age, all documents that are older will be deleted.
+         * @param maxAge The maximum age of a document
+         * @return The {@link Builder} with max age set.
+         */
+        public Builder setMaxAge(TimeValue maxAge) {
+            this.maxAge = maxAge;
+            return this;
+        }
+
+        public TimeRetentionPolicyConfig build() {
+            return new TimeRetentionPolicyConfig(field, maxAge);
+        }
+    }
 }
diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/TimeSyncConfig.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/TimeSyncConfig.java
@@ -44,6 +44,8 @@ public static TimeSyncConfig fromXContent(XContentParser parser) {
         return PARSER.apply(parser, null);
     }
 
+    // Deprecated, the public modifier will be removed in 8.0: use the builder instead
+    @Deprecated
     public TimeSyncConfig(String field, TimeValue delay) {
         this.field = field;
         this.delay = delay;
@@ -94,4 +96,37 @@ public String getName() {
         return NAME;
     }
 
+    public static Builder builder() {
+        return new Builder();
+    }
+
+    public static class Builder {
+        private String field;
+        private TimeValue delay = TimeValue.ZERO;
+
+        /**
+         * The date field that is used to identify new documents in the source.
+         * @param field The field name of the timestamp field used for synchronizing
+         * @return The {@link Builder} with the field set.
+         */
+        public Builder setField(String field) {
+            this.field = field;
+            return this;
+        }
+
+        /**
+         * The time delay between the current time and the latest input data time.
+         * The default value is 60s.
+         * @param delay the delay to use when checking for changes
+         * @return The {@link Builder} with delay set.
+         */
+        public Builder setDelay(TimeValue delay) {
+            this.delay = delay;
+            return this;
+        }
+
+        public TimeSyncConfig build() {
+            return new TimeSyncConfig(field, delay);
+        }
+    }
 }
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/TransformIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/TransformIT.java
@@ -201,7 +201,7 @@ public void testUpdate() throws IOException {
 
         String id = "test-update";
         TransformConfig transform = validDataFrameTransformConfigBuilder(id, sourceIndex, "pivot-dest").setSyncConfig(
-            new TimeSyncConfig("timefield", TimeValue.timeValueSeconds(60))
+            TimeSyncConfig.builder().setField("timestamp").setDelay(TimeValue.timeValueSeconds(60)).build()
         ).build();
 
         TransformClient client = highLevelClient().transform();
diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/TransformDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/TransformDocumentationIT.java
@@ -34,8 +34,11 @@
 import org.elasticsearch.client.transform.transforms.DestConfig;
 import org.elasticsearch.client.transform.transforms.NodeAttributes;
 import org.elasticsearch.client.transform.transforms.QueryConfig;
+import org.elasticsearch.client.transform.transforms.RetentionPolicyConfig;
 import org.elasticsearch.client.transform.transforms.SettingsConfig;
 import org.elasticsearch.client.transform.transforms.SourceConfig;
+import org.elasticsearch.client.transform.transforms.SyncConfig;
+import org.elasticsearch.client.transform.transforms.TimeRetentionPolicyConfig;
 import org.elasticsearch.client.transform.transforms.TimeSyncConfig;
 import org.elasticsearch.client.transform.transforms.TransformConfig;
 import org.elasticsearch.client.transform.transforms.TransformConfigUpdate;
@@ -147,6 +150,18 @@ public void testPutTransform() throws IOException, InterruptedException {
             .setMaxPageSearchSize(1000) // <1>
             .build();
         // end::put-transform-settings-config
+        // tag::put-transform-retention-policy-config
+        RetentionPolicyConfig retentionPolicy = TimeRetentionPolicyConfig.builder()
+            .setField("time-field") // <1>
+            .setMaxAge(TimeValue.timeValueDays(30)) // <2>
+            .build();
+        // end::put-transform-retention-policy-config
+        // tag::put-transform-sync-config
+        SyncConfig syncConfig = TimeSyncConfig.builder()
+            .setField("time-field") // <1>
+            .setDelay(TimeValue.timeValueSeconds(30)) // <2>
+            .build();
+        // end::put-transform-sync-config
         // tag::put-transform-config
         TransformConfig transformConfig = TransformConfig
             .builder()
@@ -157,6 +172,8 @@ public void testPutTransform() throws IOException, InterruptedException {
             .setPivotConfig(pivotConfig) // <5>
             .setDescription("This is my test transform") // <6>
             .setSettings(settings) // <7>
+            .setRetentionPolicyConfig(retentionPolicy) // <8>
+            .setSyncConfig(syncConfig) // <9>
             .build();
         // end::put-transform-config
 
@@ -230,7 +247,7 @@ public void testUpdateTransform() throws IOException, InterruptedException {
             .setSource(SourceConfig.builder().setIndex("source-data").setQueryConfig(queryConfig).build())
             .setDest(DestConfig.builder().setIndex("pivot-dest").build())
             .setPivotConfig(pivotConfig)
-            .setSyncConfig(new TimeSyncConfig("time-field", TimeValue.timeValueSeconds(120)))
+            .setSyncConfig(TimeSyncConfig.builder().setField("time-field").setDelay(TimeValue.timeValueSeconds(120)).build())
             .build();
 
         client.transform().putTransform(new PutTransformRequest(transformConfig), RequestOptions.DEFAULT);
@@ -246,8 +263,10 @@ public void testUpdateTransform() throws IOException, InterruptedException {
                 .setIndex("pivot-dest")
                 .build()) // <2>
             .setFrequency(TimeValue.timeValueSeconds(15)) // <3>
-            .setSyncConfig(new TimeSyncConfig("time-field",
-                TimeValue.timeValueSeconds(120))) // <4>
+            .setSyncConfig(TimeSyncConfig.builder()
+                .setField("time-field")
+                .setDelay(TimeValue.timeValueSeconds(120))
+                .build()) // <4>
             .setDescription("This is my updated transform") // <5>
             .build();
         // end::update-transform-config
diff --git a/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformIT.java b/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformIT.java
@@ -135,7 +135,7 @@ public void testContinuousTransformCrud() throws Exception {
         TransformConfig config =
             createTransformConfigBuilder(transformId, "reviews-by-user-business-day", QueryBuilders.matchAllQuery(), indexName)
                 .setPivotConfig(createPivotConfig(groups, aggs))
-                .setSyncConfig(new TimeSyncConfig("timestamp", TimeValue.timeValueSeconds(1)))
+                .setSyncConfig(TimeSyncConfig.builder().setField("timestamp").setDelay(TimeValue.timeValueSeconds(1)).build())
                 .build();
 
         assertTrue(putTransform(config, RequestOptions.DEFAULT).isAcknowledged());
@@ -183,7 +183,7 @@ public void testContinuousTransformUpdate() throws Exception {
         TransformConfig config =
             createTransformConfigBuilder(id, dest, QueryBuilders.matchAllQuery(), indexName)
                 .setPivotConfig(createPivotConfig(groups, aggs))
-                .setSyncConfig(new TimeSyncConfig("timestamp", TimeValue.timeValueSeconds(1)))
+                .setSyncConfig(TimeSyncConfig.builder().setField("timestamp").setDelay(TimeValue.timeValueSeconds(1)).build())
                 .build();
 
         assertTrue(putTransform(config, RequestOptions.DEFAULT).isAcknowledged());
@@ -269,7 +269,7 @@ public void testStopWaitForCheckpoint() throws Exception {
         TransformConfig config =
             createTransformConfigBuilder(transformId, "reviews-by-user-business-day", QueryBuilders.matchAllQuery(), indexName)
                 .setPivotConfig(createPivotConfig(groups, aggs))
-                .setSyncConfig(new TimeSyncConfig("timestamp", TimeValue.timeValueSeconds(1)))
+                .setSyncConfig(TimeSyncConfig.builder().setField("timestamp").setDelay(TimeValue.timeValueSeconds(1)).build())
                 .build();
 
         assertTrue(putTransform(config, RequestOptions.DEFAULT).isAcknowledged());
@@ -335,7 +335,7 @@ public void testContinuousTransformRethrottle() throws Exception {
         TransformConfig config =
             createTransformConfigBuilder(transformId, "reviews-by-user-business-day", QueryBuilders.matchAllQuery(), indexName)
                 .setPivotConfig(createPivotConfig(groups, aggs))
-                .setSyncConfig(new TimeSyncConfig("timestamp", TimeValue.timeValueSeconds(1)))
+                .setSyncConfig(TimeSyncConfig.builder().setField("timestamp").setDelay(TimeValue.timeValueSeconds(1)).build())
                 // set requests per second and page size low enough to fail the test if update does not succeed,
                 .setSettings(SettingsConfig.builder().setRequestsPerSecond(1F).setMaxPageSearchSize(10).build())
                 .build();
diff --git a/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/ContinuousTestCase.java b/x-pack/plugin/transform/qa/multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/continuous/ContinuousTestCase.java
@@ -136,6 +136,6 @@ private static class TestRestHighLevelClient extends RestHighLevelClient {
     }
 
     private SyncConfig getSyncConfig() {
-        return new TimeSyncConfig("timestamp", SYNC_DELAY);
+        return TimeSyncConfig.builder().setField("timestamp").setDelay(SYNC_DELAY).build();
     }
 }
diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TransformSurvivesUpgradeIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TransformSurvivesUpgradeIT.java
@@ -176,7 +176,7 @@ private void createAndStartContinuousTransform() throws Exception {
         }
         long totalDocsWritten = totalDocsWrittenSum;
         TransformConfig config = TransformConfig.builder()
-            .setSyncConfig(new TimeSyncConfig("timestamp", TimeValue.timeValueSeconds(1)))
+            .setSyncConfig(TimeSyncConfig.builder().setField("timestamp").setDelay(TimeValue.timeValueSeconds(1)).build())
             .setPivotConfig(PivotConfig.builder()
                 .setAggregations(new AggregatorFactories.Builder().addAggregator(AggregationBuilders.avg("stars").field("stars")))
                 .setGroups(GroupConfig.builder().groupBy("user_id", TermsGroupSource.builder().setField("user_id").build()).build())

Original file line number	Diff line number	Diff line change
`@@ -136,6 +136,6 @@ private static class TestRestHighLevelClient extends RestHighLevelClient {`
`136`	`136`	`}`
`137`	`137`
`138`	`138`	`private SyncConfig getSyncConfig() {`
`139`		`- return new TimeSyncConfig("timestamp", SYNC_DELAY);`
	`139`	`+ return TimeSyncConfig.builder().setField("timestamp").setDelay(SYNC_DELAY).build();`
`140`	`140`	`}`
`141`	`141`	`}`
Original file line number	Diff line number	Diff line change
`@@ -176,7 +176,7 @@ private void createAndStartContinuousTransform() throws Exception {`
`176`	`176`	`}`
`177`	`177`	`long totalDocsWritten = totalDocsWrittenSum;`
`178`	`178`	`TransformConfig config = TransformConfig.builder()`
`179`		`- .setSyncConfig(new TimeSyncConfig("timestamp", TimeValue.timeValueSeconds(1)))`
	`179`	`+ .setSyncConfig(TimeSyncConfig.builder().setField("timestamp").setDelay(TimeValue.timeValueSeconds(1)).build())`
`180`	`180`	`.setPivotConfig(PivotConfig.builder()`
`181`	`181`	`.setAggregations(new AggregatorFactories.Builder().addAggregator(AggregationBuilders.avg("stars").field("stars")))`
`182`	`182`	`.setGroups(GroupConfig.builder().groupBy("user_id", TermsGroupSource.builder().setField("user_id").build()).build())`