From cf89967a39c492d1afeabc9583b0cc3f9b96badd Mon Sep 17 00:00:00 2001 From: Esduard Date: Sun, 27 Jun 2021 13:44:55 -0300 Subject: [PATCH 1/9] changing hard coded 10k page size limit to 65k --- .../xpack/core/transform/transforms/SettingsConfig.java | 4 ++-- .../xpack/core/transform/transforms/pivot/PivotConfig.java | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/SettingsConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/SettingsConfig.java index d8dce0bbb005c..e85b3964c9495 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/SettingsConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/SettingsConfig.java @@ -98,9 +98,9 @@ public Integer getDatesAsEpochMillisForUpdate() { public ActionRequestValidationException validate(ActionRequestValidationException validationException) { // TODO: make this dependent on search.max_buckets - if (maxPageSearchSize != null && (maxPageSearchSize < 10 || maxPageSearchSize > 10_000)) { + if (maxPageSearchSize != null && (maxPageSearchSize < 10 || maxPageSearchSize > 65_536)) { validationException = addValidationError( - "settings.max_page_search_size [" + maxPageSearchSize + "] must be greater than 10 and less than 10,000", + "settings.max_page_search_size [" + maxPageSearchSize + "] must be greater than 10 and less than 65,536", validationException ); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/pivot/PivotConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/pivot/PivotConfig.java index a6c39316d9497..5dcc2be7f5a19 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/pivot/PivotConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/pivot/PivotConfig.java @@ -170,9 +170,9 @@ public int hashCode() { } public ActionRequestValidationException validate(ActionRequestValidationException validationException) { - if (maxPageSearchSize != null && (maxPageSearchSize < 10 || maxPageSearchSize > 10_000)) { + if (maxPageSearchSize != null && (maxPageSearchSize < 10 || maxPageSearchSize > 65_536)) { validationException = addValidationError( - "pivot.max_page_search_size [" + maxPageSearchSize + "] must be greater than 10 and less than 10,000", + "pivot.max_page_search_size [" + maxPageSearchSize + "] must be greater than 10 and less than 65,536", validationException ); } From 249c954c4b70eba3d67a174673f33ce57432b8a5 Mon Sep 17 00:00:00 2001 From: Esduard Date: Wed, 30 Jun 2021 00:13:36 -0300 Subject: [PATCH 2/9] Importing Variable Max Buckets import 'max_buckets' constant --- .../xpack/core/transform/transforms/SettingsConfig.java | 6 +++--- .../xpack/core/transform/transforms/pivot/PivotConfig.java | 6 ++++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/SettingsConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/SettingsConfig.java index e85b3964c9495..f6d0e7bd9d071 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/SettingsConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/SettingsConfig.java @@ -18,6 +18,7 @@ import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.search.aggregations.MultiBucketConsumerService; import org.elasticsearch.xpack.core.transform.TransformField; import java.io.IOException; @@ -97,10 +98,9 @@ public Integer getDatesAsEpochMillisForUpdate() { } public ActionRequestValidationException validate(ActionRequestValidationException validationException) { - // TODO: make this dependent on search.max_buckets - if (maxPageSearchSize != null && (maxPageSearchSize < 10 || maxPageSearchSize > 65_536)) { + if (maxPageSearchSize != null && (maxPageSearchSize < 10 || maxPageSearchSize > MultiBucketConsumerService.DEFAULT_MAX_BUCKETS)) { validationException = addValidationError( - "settings.max_page_search_size [" + maxPageSearchSize + "] must be greater than 10 and less than 65,536", + "settings.max_page_search_size [" + maxPageSearchSize + "] is out of range. The minimum value is 10 and the maximum is " + MultiBucketConsumerService.DEFAULT_MAX_BUCKETS, validationException ); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/pivot/PivotConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/pivot/PivotConfig.java index 5dcc2be7f5a19..1c908c3736ffd 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/pivot/PivotConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/pivot/PivotConfig.java @@ -18,6 +18,7 @@ import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.search.aggregations.MultiBucketConsumerService; import org.elasticsearch.search.aggregations.bucket.composite.CompositeAggregationBuilder; import org.elasticsearch.xpack.core.transform.TransformField; import org.elasticsearch.xpack.core.transform.utils.ExceptionsHelper; @@ -170,9 +171,10 @@ public int hashCode() { } public ActionRequestValidationException validate(ActionRequestValidationException validationException) { - if (maxPageSearchSize != null && (maxPageSearchSize < 10 || maxPageSearchSize > 65_536)) { + + if (maxPageSearchSize != null && (maxPageSearchSize < 10 || maxPageSearchSize > MultiBucketConsumerService.DEFAULT_MAX_BUCKETS)) { validationException = addValidationError( - "pivot.max_page_search_size [" + maxPageSearchSize + "] must be greater than 10 and less than 65,536", + "pivot.max_page_search_size [" + maxPageSearchSize + "] is out of range. The minimum value is 10 and the maximum is " + MultiBucketConsumerService.DEFAULT_MAX_BUCKETS, validationException ); } From 9bdef7767aa729592c237654b36a7ac60c4d78f6 Mon Sep 17 00:00:00 2001 From: Esduard Date: Wed, 30 Jun 2021 00:13:54 -0300 Subject: [PATCH 3/9] updating test cases for page size limit --- .../rest-api-spec/test/transform/transforms_crud.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/transform/transforms_crud.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/transform/transforms_crud.yml index bd8a1e21724eb..62438e9246923 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/transform/transforms_crud.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/transform/transforms_crud.yml @@ -480,7 +480,7 @@ setup: --- "Test put config with invalid pivot size": - do: - catch: /pivot\.max_page_search_size \[5\] must be greater than 10 and less than 10,000/ + catch: /pivot\.max_page_search_size \[5\] is out of range. The minimum value is 10 and the maximum is 65536/ transform.put_transform: transform_id: "airline-transform" body: > @@ -494,7 +494,8 @@ setup: } } - do: - catch: /pivot\.max_page_search_size \[15000\] must be greater than 10 and less than 10,000/ + catch: /pivot\.max_page_search_size \[75000\] is out of range. The minimum value is 10 and the maximum is 65536/ + transform.put_transform: transform_id: "airline-transform" body: > @@ -502,7 +503,7 @@ setup: "source": { "index": "airline-data" }, "dest": { "index": "airline-dest-index" }, "pivot": { - "max_page_search_size": 15000, + "max_page_search_size": 75000, "group_by": { "airline": {"terms": {"field": "airline"}}}, "aggs": {"avg_response": {"avg": {"field": "responsetime"}}} } From 3bbe3588a6dfbd027e08b7ed235ae565e7b132bf Mon Sep 17 00:00:00 2001 From: Esduard Date: Wed, 30 Jun 2021 00:21:38 -0300 Subject: [PATCH 4/9] updating documentation updating documentation with new max page size --- docs/reference/rest-api/common-parms.asciidoc | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/reference/rest-api/common-parms.asciidoc b/docs/reference/rest-api/common-parms.asciidoc index 8752c26a20f33..f2a80f40f054e 100644 --- a/docs/reference/rest-api/common-parms.asciidoc +++ b/docs/reference/rest-api/common-parms.asciidoc @@ -602,8 +602,8 @@ tag::bulk-dynamic-templates[] `dynamic_templates`:: (Optional, map) A map from the full name of fields to the name of <. -Defaults to an empty map. If a name matches a dynamic template, then that template will be -applied regardless of other match predicates defined in the template. And if a field is +Defaults to an empty map. If a name matches a dynamic template, then that template will be +applied regardless of other match predicates defined in the template. And if a field is already defined in the mapping, then this parameter won't be used. end::bulk-dynamic-templates[] @@ -896,11 +896,11 @@ end::source-transforms[] tag::source-index-transforms[] The _source indices_ for the {transform}. It can be a single index, an index pattern (for example, `"my-index-*"`), an array of indices (for example, -`["my-index-000001", "my-index-000002"]`), or an array of index patterns (for -example, `["my-index-*", "my-other-index-*"]`. For remote indices use the syntax +`["my-index-000001", "my-index-000002"]`), or an array of index patterns (for +example, `["my-index-*", "my-other-index-*"]`. For remote indices use the syntax `"remote_name:index_name"`. -NOTE: If any indices are in remote clusters then the master node and at least +NOTE: If any indices are in remote clusters then the master node and at least one transform node must have the `remote_cluster_client` node role. end::source-index-transforms[] @@ -910,8 +910,8 @@ A query clause that retrieves a subset of data from the source index. See end::source-query-transforms[] tag::source-runtime-mappings-transforms[] -Definitions of search-time runtime fields that can be used by the transform. For -search runtime fields all data nodes, including remote nodes, must be 7.12 or +Definitions of search-time runtime fields that can be used by the transform. For +search runtime fields all data nodes, including remote nodes, must be 7.12 or later. end::source-runtime-mappings-transforms[] @@ -971,7 +971,7 @@ unique key. end::transform-latest[] tag::transform-retention[] -Defines a retention policy for the {transform}. Data that meets the defined +Defines a retention policy for the {transform}. Data that meets the defined criteria is deleted from the destination index. end::transform-retention[] @@ -984,7 +984,7 @@ The date field that is used to calculate the age of the document. end::transform-retention-time-field[] tag::transform-retention-time-max-age[] -Specifies the maximum age of a document in the destination index. Documents that +Specifies the maximum age of a document in the destination index. Documents that are older than the configured value are removed from the destination index. end::transform-retention-time-max-age[] @@ -1008,7 +1008,7 @@ end::transform-settings-docs-per-second[] tag::transform-settings-max-page-search-size[] Defines the initial page size to use for the composite aggregation for each checkpoint. If circuit breaker exceptions occur, the page size is dynamically -adjusted to a lower value. The minimum value is `10` and the maximum is `10,000`. +adjusted to a lower value. The minimum value is `10` and the maximum is `65,536`. The default value is `500`. end::transform-settings-max-page-search-size[] From be526ffd77a6672527b159eeac81de14d6c68511 Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Wed, 30 Jun 2021 08:46:07 +0200 Subject: [PATCH 5/9] fix checkstyle line length --- .../xpack/core/transform/transforms/SettingsConfig.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/SettingsConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/SettingsConfig.java index f6d0e7bd9d071..4385631a311a7 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/SettingsConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/SettingsConfig.java @@ -100,7 +100,11 @@ public Integer getDatesAsEpochMillisForUpdate() { public ActionRequestValidationException validate(ActionRequestValidationException validationException) { if (maxPageSearchSize != null && (maxPageSearchSize < 10 || maxPageSearchSize > MultiBucketConsumerService.DEFAULT_MAX_BUCKETS)) { validationException = addValidationError( - "settings.max_page_search_size [" + maxPageSearchSize + "] is out of range. The minimum value is 10 and the maximum is " + MultiBucketConsumerService.DEFAULT_MAX_BUCKETS, + validationException = addValidationError( + "settings.max_page_search_size [" + + maxPageSearchSize + + "] is out of range. The minimum value is 10 and the maximum is " + + MultiBucketConsumerService.DEFAULT_MAX_BUCKETS, validationException ); } From ed6473ccd8225da01a2fab5ae11e75327d11925b Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Wed, 30 Jun 2021 08:49:05 +0200 Subject: [PATCH 6/9] fix checkstyle line length --- .../xpack/core/transform/transforms/pivot/PivotConfig.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/pivot/PivotConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/pivot/PivotConfig.java index 1c908c3736ffd..0746230e984d3 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/pivot/PivotConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/pivot/PivotConfig.java @@ -174,7 +174,10 @@ public ActionRequestValidationException validate(ActionRequestValidationExceptio if (maxPageSearchSize != null && (maxPageSearchSize < 10 || maxPageSearchSize > MultiBucketConsumerService.DEFAULT_MAX_BUCKETS)) { validationException = addValidationError( - "pivot.max_page_search_size [" + maxPageSearchSize + "] is out of range. The minimum value is 10 and the maximum is " + MultiBucketConsumerService.DEFAULT_MAX_BUCKETS, + "settings.max_page_search_size [" + + maxPageSearchSize + + "] is out of range. The minimum value is 10 and the maximum is " + + MultiBucketConsumerService.DEFAULT_MAX_BUCKETS, validationException ); } From fadb50402c3715abc731463b0d6050964f48ba61 Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Wed, 30 Jun 2021 08:49:31 +0200 Subject: [PATCH 7/9] fix checkstyle --- .../xpack/core/transform/transforms/SettingsConfig.java | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/SettingsConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/SettingsConfig.java index 4385631a311a7..0de9f49b1978d 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/SettingsConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/SettingsConfig.java @@ -100,7 +100,6 @@ public Integer getDatesAsEpochMillisForUpdate() { public ActionRequestValidationException validate(ActionRequestValidationException validationException) { if (maxPageSearchSize != null && (maxPageSearchSize < 10 || maxPageSearchSize > MultiBucketConsumerService.DEFAULT_MAX_BUCKETS)) { validationException = addValidationError( - validationException = addValidationError( "settings.max_page_search_size [" + maxPageSearchSize + "] is out of range. The minimum value is 10 and the maximum is " From ebb831b81c7aa1bb4688a09e0732f85bd705a7e1 Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Wed, 30 Jun 2021 08:50:47 +0200 Subject: [PATCH 8/9] Update PivotConfig.java --- .../xpack/core/transform/transforms/pivot/PivotConfig.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/pivot/PivotConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/pivot/PivotConfig.java index 0746230e984d3..ecc0a7e2816b6 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/pivot/PivotConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/pivot/PivotConfig.java @@ -174,7 +174,7 @@ public ActionRequestValidationException validate(ActionRequestValidationExceptio if (maxPageSearchSize != null && (maxPageSearchSize < 10 || maxPageSearchSize > MultiBucketConsumerService.DEFAULT_MAX_BUCKETS)) { validationException = addValidationError( - "settings.max_page_search_size [" + "pivot.max_page_search_size [" + maxPageSearchSize + "] is out of range. The minimum value is 10 and the maximum is " + MultiBucketConsumerService.DEFAULT_MAX_BUCKETS, From 4255f0d84a4a94a94701809b7b4c599c4c8e768f Mon Sep 17 00:00:00 2001 From: Esduard Date: Wed, 30 Jun 2021 11:32:06 -0300 Subject: [PATCH 9/9] blacklisting invalid pivot size test temporarily --- x-pack/plugin/build.gradle | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugin/build.gradle b/x-pack/plugin/build.gradle index 7448fb38fa5e6..e09f812fd7224 100644 --- a/x-pack/plugin/build.gradle +++ b/x-pack/plugin/build.gradle @@ -167,6 +167,7 @@ tasks.named("yamlRestCompatTest").configure { 'transform/transforms_cat_apis/Test cat transform stats hiding headers', 'transform/transforms_cat_apis/Test cat transform stats with column selection', 'transform/transforms_cat_apis/Test cat transform stats with continuous transform', + 'transform/transforms_crud/Test put config with invalid pivot size', // disabled until backport of #74651 has finished 'vectors/10_dense_vector_basic/Deprecated function signature', 'vectors/30_sparse_vector_basic/Cosine Similarity', 'vectors/30_sparse_vector_basic/Deprecated function signature',