From d09a4b674d2520832447fbc91064a16f3629f7c6 Mon Sep 17 00:00:00 2001 From: Craig Taverner Date: Tue, 14 Jun 2022 11:18:08 +0200 Subject: [PATCH 1/8] Update release notes for BC4 --- docs/reference/release-notes/8.3.0.asciidoc | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/docs/reference/release-notes/8.3.0.asciidoc b/docs/reference/release-notes/8.3.0.asciidoc index 193dd95762e94..ee1fb34e52235 100644 --- a/docs/reference/release-notes/8.3.0.asciidoc +++ b/docs/reference/release-notes/8.3.0.asciidoc @@ -12,6 +12,7 @@ Also see <>. Aggregations:: * Allow `serial_diff` under `min_doc_count` aggs {es-pull}86401[#86401] * Allow bucket paths to specify `_count` within a bucket {es-pull}85720[#85720] +* Fix a bug with flattened fields in terms aggregations {es-pull}87392[#87392] * Fix flaky `top_metrics` test {es-pull}86582[#86582] (issue: {es-issue}86377[#86377]) * Fix: check field existence before trying to merge running stats {es-pull}86926[#86926] * Fix: ordering terms aggregation on top metrics null values {es-pull}85774[#85774] @@ -44,6 +45,7 @@ Distributed:: * Enforce external id uniqueness during `DesiredNode` construction {es-pull}84227[#84227] Engine:: +* Fork to WRITE thread before failing shard in `updateCheckPoints` {es-pull}87458[#87458] (issue: {es-issue}87094[#87094]) * Removing Blocking Wait for Close in `RecoverySourceHandler` {es-pull}86127[#86127] (issue: {es-issue}85839[#85839]) Features:: @@ -73,6 +75,9 @@ Infra/Core:: Infra/Logging:: * Temporarily provide `SystemPropertiesPropertySource` {es-pull}87149[#87149] +Infra/Node Lifecycle:: +* Upgrade folders after settings validation {es-pull}87319[#87319] + Infra/Plugins:: * Use Windows newlines when listing plugin information on Windows {es-pull}86408[#86408] (issue: {es-issue}86352[#86352]) @@ -82,6 +87,7 @@ Infra/REST API:: Infra/Scripting:: * Allow to sort by script value using `SemVer` semantics {es-pull}85990[#85990] (issues: {es-issue}85989[#85989], {es-issue}82287[#82287]) * Script: Fix setter shortcut for unbridged setters {es-pull}86868[#86868] +* Script: Load Whitelists as Resource {es-pull}87539[#87539] Infra/Settings:: * Permit removal of archived index settings {es-pull}86107[#86107] @@ -97,16 +103,24 @@ License:: Machine Learning:: * Fix ML task auditor exception early in cluster lifecycle {es-pull}87023[#87023] (issue: {es-issue}87002[#87002]) +* Fix `WordPiece` tokenization of unknown words with known subwords {es-pull}87510[#87510] * Fix distribution change check for `change_point` aggregation {es-pull}86423[#86423] +* Fixes inference timeout handling bug that throws unexpected `NullPointerException` {es-pull}87533[#87533] * Correct logic for restart from failover fine tuning hyperparameters for training classification and regression models {ml-pull}2251[#2251] * Fix possible source of "x = NaN, distribution = class boost::math::normal_distribution<..." log errors training classification and regression models {ml-pull}2249[#2249] * Fix some bugs affecting decision to stop optimizing hyperparameters for training classification and regression models {ml-pull}2259[#2259] * Fix cause of "Must provide points at which to evaluate function" log error training classification and regression models {ml-pull}2268[#2268] * Fix a source of "Discarding sample = nan, weights = ..." log errors for time series anomaly detection {ml-pull}2286[#2286] +Mapping:: +* Don't run `include_in_parent` when in `copy_to` context {es-pull}87123[#87123] (issue: {es-issue}87036[#87036]) + Network:: * Reject `openConnection` attempt while closing {es-pull}86315[#86315] (issue: {es-issue}86249[#86249]) +Recovery:: +* Fail shard if STARTED after master failover {es-pull}87451[#87451] (issue: {es-issue}87367[#87367]) + SQL:: * Fix FORMAT function to comply with Microsoft SQL Server specification {es-pull}86225[#86225] (issue: {es-issue}66560[#66560]) * Implement binary format support for SQL clear cursor {es-pull}84230[#84230] (issue: {es-issue}53359[#53359]) @@ -120,6 +134,7 @@ Search:: Security:: * Make user and role name constraint consistent with max document ID {es-pull}86728[#86728] (issue: {es-issue}66020[#66020]) +* Security plugin close releasable realms {es-pull}87429[#87429] (issue: {es-issue}86286[#86286]) Snapshot/Restore:: * DONE should mean fully processed in snapshot status {es-pull}86414[#86414] @@ -189,6 +204,7 @@ Geo:: Health:: * Add a basic check for tier preference and allocation filter clashing {es-pull}85071[#85071] * Add preflight checks to Health API to ensure health is obtainable {es-pull}86404[#86404] +* Add tier information on health api migrate tiers user actions {es-pull}87486[#87486] * Health api add indicator doc links {es-pull}86904[#86904] (issue: {es-issue}86892[#86892]) * Health api copy editing {es-pull}87010[#87010] * Return a default user action if no actions could be determined {es-pull}87079[#87079] From 65e3b0b322fabe9676df74736407152aceeb2163 Mon Sep 17 00:00:00 2001 From: Craig Taverner Date: Tue, 14 Jun 2022 19:28:45 +0200 Subject: [PATCH 2/8] Adding missing changelog for geo_grid --- docs/changelog/86596.yaml | 34 +++++++++++++++++++ docs/reference/release-notes/8.3.0.asciidoc | 3 ++ .../release-notes/highlights.asciidoc | 28 +++++++++++++++ 3 files changed, 65 insertions(+) create mode 100644 docs/changelog/86596.yaml diff --git a/docs/changelog/86596.yaml b/docs/changelog/86596.yaml new file mode 100644 index 0000000000000..68db2ec7a9b38 --- /dev/null +++ b/docs/changelog/86596.yaml @@ -0,0 +1,34 @@ +pr: 86596 +summary: New geo_grid query to be used with geogrid aggregations +area: Geo +type: feature +issues: + - 85727 +highlight: + title: New geo_grid query + body: |- + With the geo grid query, you can now natively return all the documents that overlap a specific geo tile. + There is no need to reconstruct the geometry or the actual boundaries of the spatial cluster as Elasticsearch + can do this for you which saves you time and reduces complexity. + This is especially useful when geometries are spread across tiles like on a soccer ball football. + While hexagon tiles line the sphere each tile is not exactly straight forward to calculate its boundary. + + ``` + GET /example/_search + { + "query": { + "geo_grid" :{ + "location" : { + "geotile" : "6/32/22" + } + } + } + } + ``` + + Geo grid query can also help determine the single source of truth of containment. + With geo grid query, you can match exactly the intersection-test of Elasticsearch. + As an example, if a client has bounds for a grid-cell at a higher (or lower) precision than what is used + by Elasticsearch when running a corresponding aggregation, the containment-check might be slightly different. + This side-steps any disconnect based on projection/datum difference between client and Elasticsearch. + notable: true diff --git a/docs/reference/release-notes/8.3.0.asciidoc b/docs/reference/release-notes/8.3.0.asciidoc index ee1fb34e52235..4cdca2665a03a 100644 --- a/docs/reference/release-notes/8.3.0.asciidoc +++ b/docs/reference/release-notes/8.3.0.asciidoc @@ -292,6 +292,9 @@ Transform:: Authorization:: * Has privileges API for profiles {es-pull}85898[#85898] +Geo:: +* New geo_grid query to be used with geogrid aggregations {es-pull}86596[#86596] (issue: {es-issue}85727[#85727]) + Health:: * Add support for `impact_areas` to health impacts {es-pull}85830[#85830] (issue: {es-issue}85829[#85829]) * Add troubleshooting guides to shards allocation actions {es-pull}87078[#87078] diff --git a/docs/reference/release-notes/highlights.asciidoc b/docs/reference/release-notes/highlights.asciidoc index 8deb96bd98601..7803008dc6ffa 100644 --- a/docs/reference/release-notes/highlights.asciidoc +++ b/docs/reference/release-notes/highlights.asciidoc @@ -29,6 +29,34 @@ for compliance or regulatory reasons, the occasional lookback or investigation, or to rehydrate parts of it. Access to the data is expected to be infrequent, and can therefore happen with limited performance and query capabilities. +[discrete] +[[new_geo_grid_query]] +=== New geo_grid query +With the geo grid query, you can now natively return all the documents that overlap a specific geo tile. +There is no need to reconstruct the geometry or the actual boundaries of the spatial cluster as Elasticsearch +can do this for you which saves you time and reduces complexity. +This is especially useful when geometries are spread across tiles like on a soccer ball football. +While hexagon tiles line the sphere each tile is not exactly straight forward to calculate its boundary. + +``` +GET /example/_search +{ + "query": { + "geo_grid" :{ + "location" : { + "geotile" : "6/32/22" + } + } + } +} +``` + +Geo grid query can also help determine the single source of truth of containment. +With geo grid query, you can match exactly the intersection-test of Elasticsearch. +As an example, if a client has bounds for a grid-cell at a higher (or lower) precision than what is used +by Elasticsearch when running a corresponding aggregation, the containment-check might be slightly different. +This side-steps any disconnect based on projection/datum difference between client and Elasticsearch. + [discrete] [[add_support_for_dots_in_field_names_for_metrics_usecases]] === Add support for dots in field names for metrics usecases From 5a31fc983881d46820afff62011d2f41c7595b4d Mon Sep 17 00:00:00 2001 From: Craig Taverner Date: Wed, 15 Jun 2022 11:45:26 +0200 Subject: [PATCH 3/8] Added missing highlight notes for 84250 --- docs/changelog/84250.yaml | 8 ++++++++ docs/reference/release-notes/highlights.asciidoc | 8 ++++++++ 2 files changed, 16 insertions(+) diff --git a/docs/changelog/84250.yaml b/docs/changelog/84250.yaml index 28ce1d9b50d9f..8d6cd71ee43dd 100644 --- a/docs/changelog/84250.yaml +++ b/docs/changelog/84250.yaml @@ -4,3 +4,11 @@ area: Ingest type: enhancement issues: - 84274 +highlight: + title: Ingest performance improvement + body: |- + We have improved the pipeline execution logic for pipelines with processors that are synchronous + by avoiding (deep) recursion. + On our nightly benchmark that simulate a Logging use-case, this resulted in a 10% reduction of + CPU time spent on ingest pipelines and a 3% overall ingestion speedup. + notable: true diff --git a/docs/reference/release-notes/highlights.asciidoc b/docs/reference/release-notes/highlights.asciidoc index 7803008dc6ffa..46f4607462a1e 100644 --- a/docs/reference/release-notes/highlights.asciidoc +++ b/docs/reference/release-notes/highlights.asciidoc @@ -57,6 +57,14 @@ As an example, if a client has bounds for a grid-cell at a higher (or lower) pre by Elasticsearch when running a corresponding aggregation, the containment-check might be slightly different. This side-steps any disconnect based on projection/datum difference between client and Elasticsearch. +[discrete] +[[ingest_performance_improvement]] +=== Ingest performance improvement +We have improved the pipeline execution logic for pipelines with processors that are synchronous +by avoiding (deep) recursion. +On our nightly benchmark that simulate a Logging use-case, this resulted in a 10% reduction of +CPU time spent on ingest pipelines and a 3% overall ingestion speedup. + [discrete] [[add_support_for_dots_in_field_names_for_metrics_usecases]] === Add support for dots in field names for metrics usecases From 6836322218801bbbbc100468c3b81e11724d664c Mon Sep 17 00:00:00 2001 From: Craig Taverner Date: Wed, 15 Jun 2022 18:01:33 +0200 Subject: [PATCH 4/8] Update docs/reference/release-notes/highlights.asciidoc Co-authored-by: David Kilfoyle <41695641+kilfoyle@users.noreply.github.com> --- docs/reference/release-notes/highlights.asciidoc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/reference/release-notes/highlights.asciidoc b/docs/reference/release-notes/highlights.asciidoc index 46f4607462a1e..74481f6e87b01 100644 --- a/docs/reference/release-notes/highlights.asciidoc +++ b/docs/reference/release-notes/highlights.asciidoc @@ -34,9 +34,9 @@ and can therefore happen with limited performance and query capabilities. === New geo_grid query With the geo grid query, you can now natively return all the documents that overlap a specific geo tile. There is no need to reconstruct the geometry or the actual boundaries of the spatial cluster as Elasticsearch -can do this for you which saves you time and reduces complexity. -This is especially useful when geometries are spread across tiles like on a soccer ball football. -While hexagon tiles line the sphere each tile is not exactly straight forward to calculate its boundary. +can do this for you, which saves you time and reduces complexity. +This is especially useful when geometries are spread across tiles like on a soccer ball or football. +While hexagon tiles line the sphere, calculating the boundary of each tile is not straightforward. ``` GET /example/_search From a97459844cac6ccaf2fb79713f8bb86fd25bd77a Mon Sep 17 00:00:00 2001 From: Craig Taverner Date: Wed, 15 Jun 2022 18:02:26 +0200 Subject: [PATCH 5/8] Update docs/reference/release-notes/highlights.asciidoc Co-authored-by: David Kilfoyle <41695641+kilfoyle@users.noreply.github.com> --- docs/reference/release-notes/highlights.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/release-notes/highlights.asciidoc b/docs/reference/release-notes/highlights.asciidoc index 74481f6e87b01..6ccce33d3dc95 100644 --- a/docs/reference/release-notes/highlights.asciidoc +++ b/docs/reference/release-notes/highlights.asciidoc @@ -62,7 +62,7 @@ This side-steps any disconnect based on projection/datum difference between clie === Ingest performance improvement We have improved the pipeline execution logic for pipelines with processors that are synchronous by avoiding (deep) recursion. -On our nightly benchmark that simulate a Logging use-case, this resulted in a 10% reduction of +On our nightly benchmark that simulates a Logging use-case, this resulted in a 10% reduction of CPU time spent on ingest pipelines and a 3% overall ingestion speedup. [discrete] From 5a831d0f9eae03cd5c5831a1a805f442ad14c8f8 Mon Sep 17 00:00:00 2001 From: Craig Taverner Date: Thu, 16 Jun 2022 11:21:44 +0200 Subject: [PATCH 6/8] Backported fixes to original yaml --- docs/changelog/84250.yaml | 2 +- docs/changelog/86596.yaml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/changelog/84250.yaml b/docs/changelog/84250.yaml index 8d6cd71ee43dd..87c9b5161e443 100644 --- a/docs/changelog/84250.yaml +++ b/docs/changelog/84250.yaml @@ -9,6 +9,6 @@ highlight: body: |- We have improved the pipeline execution logic for pipelines with processors that are synchronous by avoiding (deep) recursion. - On our nightly benchmark that simulate a Logging use-case, this resulted in a 10% reduction of + On our nightly benchmark that simulates a Logging use-case, this resulted in a 10% reduction of CPU time spent on ingest pipelines and a 3% overall ingestion speedup. notable: true diff --git a/docs/changelog/86596.yaml b/docs/changelog/86596.yaml index 68db2ec7a9b38..b03671d75b863 100644 --- a/docs/changelog/86596.yaml +++ b/docs/changelog/86596.yaml @@ -9,9 +9,9 @@ highlight: body: |- With the geo grid query, you can now natively return all the documents that overlap a specific geo tile. There is no need to reconstruct the geometry or the actual boundaries of the spatial cluster as Elasticsearch - can do this for you which saves you time and reduces complexity. - This is especially useful when geometries are spread across tiles like on a soccer ball football. - While hexagon tiles line the sphere each tile is not exactly straight forward to calculate its boundary. + can do this for you, which saves you time and reduces complexity. + This is especially useful when geometries are spread across tiles like on a soccer ball or football. + While hexagon tiles line the sphere, calculating the boundary of each tile is not straightforward. ``` GET /example/_search From e91015e303c7509bf5bf638450c088e92aae2603 Mon Sep 17 00:00:00 2001 From: Craig Taverner Date: Thu, 16 Jun 2022 12:18:22 +0200 Subject: [PATCH 7/8] Control sort order of release highlights So that small changes to an individual highlight don't completely shuffle the entire document. We also added links to the PRs from the highlight titles, for convenience. Otherwise readers need to search the release notes for the changelog entry and click there, which is a lot more work. --- .../internal/release/ChangelogEntry.java | 6 + .../release/ReleaseHighlightsGenerator.java | 2 + .../templates/release-highlights.asciidoc | 4 +- .../release-notes/highlights.asciidoc | 130 +++++++++--------- 4 files changed, 75 insertions(+), 67 deletions(-) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ChangelogEntry.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ChangelogEntry.java index 7827b34f57e43..093a7a19a42f9 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ChangelogEntry.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ChangelogEntry.java @@ -99,6 +99,7 @@ public Highlight getHighlight() { public void setHighlight(Highlight highlight) { this.highlight = highlight; + if (this.highlight != null) this.highlight.pr = this.pr; } public Breaking getBreaking() { @@ -160,6 +161,7 @@ public static class Highlight { private boolean notable; private String title; private String body; + private Integer pr; public boolean isNotable() { return notable; @@ -189,6 +191,10 @@ public String getAnchor() { return generatedAnchor(this.title); } + public Integer getPr() { + return pr; + } + @Override public boolean equals(Object o) { if (this == o) { diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseHighlightsGenerator.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseHighlightsGenerator.java index 92de8c2ec1752..a562f0f3583f1 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseHighlightsGenerator.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseHighlightsGenerator.java @@ -17,6 +17,7 @@ import java.io.IOException; import java.nio.file.Files; import java.util.ArrayList; +import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -50,6 +51,7 @@ static String generateFile(QualifiedVersion version, String template, List> groupedHighlights = entries.stream() .map(ChangelogEntry::getHighlight) .filter(Objects::nonNull) + .sorted(Comparator.comparingInt(ChangelogEntry.Highlight::getPr)) .collect(Collectors.groupingBy(ChangelogEntry.Highlight::isNotable, Collectors.toList())); final List notableHighlights = groupedHighlights.getOrDefault(true, List.of()); diff --git a/build-tools-internal/src/main/resources/templates/release-highlights.asciidoc b/build-tools-internal/src/main/resources/templates/release-highlights.asciidoc index 8b8d99babbc7f..f07ba9c5d4db3 100644 --- a/build-tools-internal/src/main/resources/templates/release-highlights.asciidoc +++ b/build-tools-internal/src/main/resources/templates/release-highlights.asciidoc @@ -32,7 +32,7 @@ if (notableHighlights.isEmpty()) { %> <% for (highlight in notableHighlights) { %> [discrete] [[${ highlight.anchor }]] -=== ${highlight.title} +=== {es-pull}${highlight.pr}[${highlight.title}] ${highlight.body.trim()} <% } %> // end::notable-highlights[] @@ -40,6 +40,6 @@ ${highlight.body.trim()} <% for (highlight in nonNotableHighlights) { %> [discrete] [[${ highlight.anchor }]] -=== ${highlight.title} +=== {es-pull}${highlight.pr}[${highlight.title}] ${highlight.body.trim()} <% } %> diff --git a/docs/reference/release-notes/highlights.asciidoc b/docs/reference/release-notes/highlights.asciidoc index 6ccce33d3dc95..6d5bdecd3c722 100644 --- a/docs/reference/release-notes/highlights.asciidoc +++ b/docs/reference/release-notes/highlights.asciidoc @@ -19,55 +19,49 @@ Other versions: // tag::notable-highlights[] [discrete] -[[reading_indices_from_older_elasticsearch_versions]] -=== Reading indices from older Elasticsearch versions -Elasticsearch has full query and write support for indices created in the previous major -version. If you have indices created in Elasticsearch versions 5 or 6, you can now use -the archive functionality to import and query these indices as well. -The archive functionality provides slower read-only access to older data, -for compliance or regulatory reasons, the occasional lookback or investigation, -or to rehydrate parts of it. Access to the data is expected to be infrequent, -and can therefore happen with limited performance and query capabilities. +[[ingest_performance_improvement]] +=== {es-pull}84250[Ingest performance improvement] +We have improved the pipeline execution logic for pipelines with processors that are synchronous +by avoiding (deep) recursion. +On our nightly benchmark that simulates a Logging use-case, this resulted in a 10% reduction of +CPU time spent on ingest pipelines and a 3% overall ingestion speedup. [discrete] -[[new_geo_grid_query]] -=== New geo_grid query -With the geo grid query, you can now natively return all the documents that overlap a specific geo tile. -There is no need to reconstruct the geometry or the actual boundaries of the spatial cluster as Elasticsearch -can do this for you, which saves you time and reduces complexity. -This is especially useful when geometries are spread across tiles like on a soccer ball or football. -While hexagon tiles line the sphere, calculating the boundary of each tile is not straightforward. +[[execute_self_reference_checks_once_per_pipeline]] +=== {es-pull}85926[Execute self-reference checks once per pipeline] +Ingest pipelines have a mechanism to prevent circular references in the records +they process, so that they are serializable. Prior to this change, this check was +performed after each `script` processor execution, and was ignorable. +Because of this check, a script processor configured with ``` -GET /example/_search -{ - "query": { - "geo_grid" :{ - "location" : { - "geotile" : "6/32/22" - } - } - } -} +"source": """ + def x = ctx; + ctx.x = x; +""" ``` -Geo grid query can also help determine the single source of truth of containment. -With geo grid query, you can match exactly the intersection-test of Elasticsearch. -As an example, if a client has bounds for a grid-cell at a higher (or lower) precision than what is used -by Elasticsearch when running a corresponding aggregation, the containment-check might be slightly different. -This side-steps any disconnect based on projection/datum difference between client and Elasticsearch. +...would error with `"type" : "illegal_argument_exception", "reason" : "Iterable +object is self-referencing itself (ingest script)"`. -[discrete] -[[ingest_performance_improvement]] -=== Ingest performance improvement -We have improved the pipeline execution logic for pipelines with processors that are synchronous -by avoiding (deep) recursion. -On our nightly benchmark that simulates a Logging use-case, this resulted in a 10% reduction of -CPU time spent on ingest pipelines and a 3% overall ingestion speedup. +If the script processor also had +``` +"ignore_failure" true +``` + +...then the handling thread would actually crash from an unrecoverable +StackOverflowError when trying to serialize the resulting event. + +Now, this check is performed once per pipeline, remediating the potential for +a StackOverflowError. There are some side effects also: + +- The resulting error message specifies which pipeline is causing the issue. +- There is a slight speed up for pipelines with multiple scripts, and a slight +slowdown for pipelines without scripts. [discrete] [[add_support_for_dots_in_field_names_for_metrics_usecases]] -=== Add support for dots in field names for metrics usecases +=== {es-pull}86166[Add support for dots in field names for metrics usecases] Metrics data can often be made of several fields with dots in their names, sharing common prefixes, like in the following example: @@ -107,37 +101,43 @@ without expanding dots in field names to the corresponding object structure. That makes it possible to store the metrics document above. [discrete] -[[execute_self_reference_checks_once_per_pipeline]] -=== Execute self-reference checks once per pipeline -Ingest pipelines have a mechanism to prevent circular references in the records -they process, so that they are serializable. Prior to this change, this check was -performed after each `script` processor execution, and was ignorable. - -Because of this check, a script processor configured with -``` -"source": """ - def x = ctx; - ctx.x = x; -""" -``` +[[reading_indices_from_older_elasticsearch_versions]] +=== {es-pull}86261[Reading indices from older Elasticsearch versions] +Elasticsearch has full query and write support for indices created in the previous major +version. If you have indices created in Elasticsearch versions 5 or 6, you can now use +the archive functionality to import and query these indices as well. +The archive functionality provides slower read-only access to older data, +for compliance or regulatory reasons, the occasional lookback or investigation, +or to rehydrate parts of it. Access to the data is expected to be infrequent, +and can therefore happen with limited performance and query capabilities. -...would error with `"type" : "illegal_argument_exception", "reason" : "Iterable -object is self-referencing itself (ingest script)"`. +[discrete] +[[new_geo_grid_query]] +=== {es-pull}86596[New geo_grid query] +With the geo grid query, you can now natively return all the documents that overlap a specific geo tile. +There is no need to reconstruct the geometry or the actual boundaries of the spatial cluster as Elasticsearch +can do this for you, which saves you time and reduces complexity. +This is especially useful when geometries are spread across tiles like on a soccer ball or football. +While hexagon tiles line the sphere, calculating the boundary of each tile is not straightforward. -If the script processor also had ``` -"ignore_failure" true +GET /example/_search +{ + "query": { + "geo_grid" :{ + "location" : { + "geotile" : "6/32/22" + } + } + } +} ``` -...then the handling thread would actually crash from an unrecoverable -StackOverflowError when trying to serialize the resulting event. - -Now, this check is performed once per pipeline, remediating the potential for -a StackOverflowError. There are some side effects also: - -- The resulting error message specifies which pipeline is causing the issue. -- There is a slight speed up for pipelines with multiple scripts, and a slight -slowdown for pipelines without scripts. +Geo grid query can also help determine the single source of truth of containment. +With geo grid query, you can match exactly the intersection-test of Elasticsearch. +As an example, if a client has bounds for a grid-cell at a higher (or lower) precision than what is used +by Elasticsearch when running a corresponding aggregation, the containment-check might be slightly different. +This side-steps any disconnect based on projection/datum difference between client and Elasticsearch. // end::notable-highlights[] From 9faec8dcb17a97a7166996ea38bc3053574d43f2 Mon Sep 17 00:00:00 2001 From: Craig Taverner Date: Thu, 16 Jun 2022 14:03:51 +0200 Subject: [PATCH 8/8] Fixed failing test after new ordered highlights Also made test verify re-ordering of highlights --- .../ReleaseHighlightsGeneratorTest.java | 37 ++++++++----------- ...hlightsGeneratorTest.generateFile.asciidoc | 6 +-- 2 files changed, 18 insertions(+), 25 deletions(-) diff --git a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/ReleaseHighlightsGeneratorTest.java b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/ReleaseHighlightsGeneratorTest.java index 0a478fa1a2500..7f510bef22661 100644 --- a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/ReleaseHighlightsGeneratorTest.java +++ b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/ReleaseHighlightsGeneratorTest.java @@ -60,31 +60,24 @@ public void generateFile_rendersCorrectMarkup() throws Exception { } private List getEntries() { - ChangelogEntry entry1 = new ChangelogEntry(); - ChangelogEntry.Highlight highlight1 = new ChangelogEntry.Highlight(); - entry1.setHighlight(highlight1); - - highlight1.setNotable(true); - highlight1.setTitle("Notable release highlight number 1"); - highlight1.setBody("Notable release body number 1"); - - ChangelogEntry entry2 = new ChangelogEntry(); - ChangelogEntry.Highlight highlight2 = new ChangelogEntry.Highlight(); - entry2.setHighlight(highlight2); - - highlight2.setNotable(true); - highlight2.setTitle("Notable release highlight number 2"); - highlight2.setBody("Notable release body number 2"); + ChangelogEntry entry1 = makeChangelogEntry(1, true); + ChangelogEntry entry2 = makeChangelogEntry(2, true); + ChangelogEntry entry3 = makeChangelogEntry(3, false); + // Return unordered list, to test correct re-ordering + return List.of(entry2, entry1, entry3); + } - ChangelogEntry entry3 = new ChangelogEntry(); - ChangelogEntry.Highlight highlight3 = new ChangelogEntry.Highlight(); - entry3.setHighlight(highlight3); + private ChangelogEntry makeChangelogEntry(int pr, boolean notable) { + ChangelogEntry entry = new ChangelogEntry(); + entry.setPr(pr); + ChangelogEntry.Highlight highlight = new ChangelogEntry.Highlight(); + entry.setHighlight(highlight); - highlight3.setNotable(false); - highlight3.setTitle("Notable release highlight number 3"); - highlight3.setBody("Notable release body number 3"); + highlight.setNotable(notable); + highlight.setTitle("Notable release highlight number " + pr); + highlight.setBody("Notable release body number " + pr); - return List.of(entry1, entry2, entry3); + return entry; } private String getResource(String name) throws Exception { diff --git a/build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/ReleaseHighlightsGeneratorTest.generateFile.asciidoc b/build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/ReleaseHighlightsGeneratorTest.generateFile.asciidoc index 7fee857cea210..a55a590a8bca5 100644 --- a/build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/ReleaseHighlightsGeneratorTest.generateFile.asciidoc +++ b/build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/ReleaseHighlightsGeneratorTest.generateFile.asciidoc @@ -21,12 +21,12 @@ Other versions: [discrete] [[notable_release_highlight_number_1]] -=== Notable release highlight number 1 +=== {es-pull}1[Notable release highlight number 1] Notable release body number 1 [discrete] [[notable_release_highlight_number_2]] -=== Notable release highlight number 2 +=== {es-pull}2[Notable release highlight number 2] Notable release body number 2 // end::notable-highlights[] @@ -34,6 +34,6 @@ Notable release body number 2 [discrete] [[notable_release_highlight_number_3]] -=== Notable release highlight number 3 +=== {es-pull}3[Notable release highlight number 3] Notable release body number 3