diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ChangelogEntry.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ChangelogEntry.java index 7827b34f57e43..093a7a19a42f9 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ChangelogEntry.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ChangelogEntry.java @@ -99,6 +99,7 @@ public Highlight getHighlight() { public void setHighlight(Highlight highlight) { this.highlight = highlight; + if (this.highlight != null) this.highlight.pr = this.pr; } public Breaking getBreaking() { @@ -160,6 +161,7 @@ public static class Highlight { private boolean notable; private String title; private String body; + private Integer pr; public boolean isNotable() { return notable; @@ -189,6 +191,10 @@ public String getAnchor() { return generatedAnchor(this.title); } + public Integer getPr() { + return pr; + } + @Override public boolean equals(Object o) { if (this == o) { diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseHighlightsGenerator.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseHighlightsGenerator.java index 92de8c2ec1752..a562f0f3583f1 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseHighlightsGenerator.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseHighlightsGenerator.java @@ -17,6 +17,7 @@ import java.io.IOException; import java.nio.file.Files; import java.util.ArrayList; +import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -50,6 +51,7 @@ static String generateFile(QualifiedVersion version, String template, List> groupedHighlights = entries.stream() .map(ChangelogEntry::getHighlight) .filter(Objects::nonNull) + .sorted(Comparator.comparingInt(ChangelogEntry.Highlight::getPr)) .collect(Collectors.groupingBy(ChangelogEntry.Highlight::isNotable, Collectors.toList())); final List notableHighlights = groupedHighlights.getOrDefault(true, List.of()); diff --git a/build-tools-internal/src/main/resources/templates/release-highlights.asciidoc b/build-tools-internal/src/main/resources/templates/release-highlights.asciidoc index 8b8d99babbc7f..f07ba9c5d4db3 100644 --- a/build-tools-internal/src/main/resources/templates/release-highlights.asciidoc +++ b/build-tools-internal/src/main/resources/templates/release-highlights.asciidoc @@ -32,7 +32,7 @@ if (notableHighlights.isEmpty()) { %> <% for (highlight in notableHighlights) { %> [discrete] [[${ highlight.anchor }]] -=== ${highlight.title} +=== {es-pull}${highlight.pr}[${highlight.title}] ${highlight.body.trim()} <% } %> // end::notable-highlights[] @@ -40,6 +40,6 @@ ${highlight.body.trim()} <% for (highlight in nonNotableHighlights) { %> [discrete] [[${ highlight.anchor }]] -=== ${highlight.title} +=== {es-pull}${highlight.pr}[${highlight.title}] ${highlight.body.trim()} <% } %> diff --git a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/ReleaseHighlightsGeneratorTest.java b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/ReleaseHighlightsGeneratorTest.java index 0a478fa1a2500..7f510bef22661 100644 --- a/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/ReleaseHighlightsGeneratorTest.java +++ b/build-tools-internal/src/test/java/org/elasticsearch/gradle/internal/release/ReleaseHighlightsGeneratorTest.java @@ -60,31 +60,24 @@ public void generateFile_rendersCorrectMarkup() throws Exception { } private List getEntries() { - ChangelogEntry entry1 = new ChangelogEntry(); - ChangelogEntry.Highlight highlight1 = new ChangelogEntry.Highlight(); - entry1.setHighlight(highlight1); - - highlight1.setNotable(true); - highlight1.setTitle("Notable release highlight number 1"); - highlight1.setBody("Notable release body number 1"); - - ChangelogEntry entry2 = new ChangelogEntry(); - ChangelogEntry.Highlight highlight2 = new ChangelogEntry.Highlight(); - entry2.setHighlight(highlight2); - - highlight2.setNotable(true); - highlight2.setTitle("Notable release highlight number 2"); - highlight2.setBody("Notable release body number 2"); + ChangelogEntry entry1 = makeChangelogEntry(1, true); + ChangelogEntry entry2 = makeChangelogEntry(2, true); + ChangelogEntry entry3 = makeChangelogEntry(3, false); + // Return unordered list, to test correct re-ordering + return List.of(entry2, entry1, entry3); + } - ChangelogEntry entry3 = new ChangelogEntry(); - ChangelogEntry.Highlight highlight3 = new ChangelogEntry.Highlight(); - entry3.setHighlight(highlight3); + private ChangelogEntry makeChangelogEntry(int pr, boolean notable) { + ChangelogEntry entry = new ChangelogEntry(); + entry.setPr(pr); + ChangelogEntry.Highlight highlight = new ChangelogEntry.Highlight(); + entry.setHighlight(highlight); - highlight3.setNotable(false); - highlight3.setTitle("Notable release highlight number 3"); - highlight3.setBody("Notable release body number 3"); + highlight.setNotable(notable); + highlight.setTitle("Notable release highlight number " + pr); + highlight.setBody("Notable release body number " + pr); - return List.of(entry1, entry2, entry3); + return entry; } private String getResource(String name) throws Exception { diff --git a/build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/ReleaseHighlightsGeneratorTest.generateFile.asciidoc b/build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/ReleaseHighlightsGeneratorTest.generateFile.asciidoc index 7fee857cea210..a55a590a8bca5 100644 --- a/build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/ReleaseHighlightsGeneratorTest.generateFile.asciidoc +++ b/build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/ReleaseHighlightsGeneratorTest.generateFile.asciidoc @@ -21,12 +21,12 @@ Other versions: [discrete] [[notable_release_highlight_number_1]] -=== Notable release highlight number 1 +=== {es-pull}1[Notable release highlight number 1] Notable release body number 1 [discrete] [[notable_release_highlight_number_2]] -=== Notable release highlight number 2 +=== {es-pull}2[Notable release highlight number 2] Notable release body number 2 // end::notable-highlights[] @@ -34,6 +34,6 @@ Notable release body number 2 [discrete] [[notable_release_highlight_number_3]] -=== Notable release highlight number 3 +=== {es-pull}3[Notable release highlight number 3] Notable release body number 3 diff --git a/docs/changelog/84250.yaml b/docs/changelog/84250.yaml index 28ce1d9b50d9f..87c9b5161e443 100644 --- a/docs/changelog/84250.yaml +++ b/docs/changelog/84250.yaml @@ -4,3 +4,11 @@ area: Ingest type: enhancement issues: - 84274 +highlight: + title: Ingest performance improvement + body: |- + We have improved the pipeline execution logic for pipelines with processors that are synchronous + by avoiding (deep) recursion. + On our nightly benchmark that simulates a Logging use-case, this resulted in a 10% reduction of + CPU time spent on ingest pipelines and a 3% overall ingestion speedup. + notable: true diff --git a/docs/changelog/86596.yaml b/docs/changelog/86596.yaml new file mode 100644 index 0000000000000..b03671d75b863 --- /dev/null +++ b/docs/changelog/86596.yaml @@ -0,0 +1,34 @@ +pr: 86596 +summary: New geo_grid query to be used with geogrid aggregations +area: Geo +type: feature +issues: + - 85727 +highlight: + title: New geo_grid query + body: |- + With the geo grid query, you can now natively return all the documents that overlap a specific geo tile. + There is no need to reconstruct the geometry or the actual boundaries of the spatial cluster as Elasticsearch + can do this for you, which saves you time and reduces complexity. + This is especially useful when geometries are spread across tiles like on a soccer ball or football. + While hexagon tiles line the sphere, calculating the boundary of each tile is not straightforward. + + ``` + GET /example/_search + { + "query": { + "geo_grid" :{ + "location" : { + "geotile" : "6/32/22" + } + } + } + } + ``` + + Geo grid query can also help determine the single source of truth of containment. + With geo grid query, you can match exactly the intersection-test of Elasticsearch. + As an example, if a client has bounds for a grid-cell at a higher (or lower) precision than what is used + by Elasticsearch when running a corresponding aggregation, the containment-check might be slightly different. + This side-steps any disconnect based on projection/datum difference between client and Elasticsearch. + notable: true diff --git a/docs/reference/release-notes/8.3.0.asciidoc b/docs/reference/release-notes/8.3.0.asciidoc index 193dd95762e94..4cdca2665a03a 100644 --- a/docs/reference/release-notes/8.3.0.asciidoc +++ b/docs/reference/release-notes/8.3.0.asciidoc @@ -12,6 +12,7 @@ Also see <>. Aggregations:: * Allow `serial_diff` under `min_doc_count` aggs {es-pull}86401[#86401] * Allow bucket paths to specify `_count` within a bucket {es-pull}85720[#85720] +* Fix a bug with flattened fields in terms aggregations {es-pull}87392[#87392] * Fix flaky `top_metrics` test {es-pull}86582[#86582] (issue: {es-issue}86377[#86377]) * Fix: check field existence before trying to merge running stats {es-pull}86926[#86926] * Fix: ordering terms aggregation on top metrics null values {es-pull}85774[#85774] @@ -44,6 +45,7 @@ Distributed:: * Enforce external id uniqueness during `DesiredNode` construction {es-pull}84227[#84227] Engine:: +* Fork to WRITE thread before failing shard in `updateCheckPoints` {es-pull}87458[#87458] (issue: {es-issue}87094[#87094]) * Removing Blocking Wait for Close in `RecoverySourceHandler` {es-pull}86127[#86127] (issue: {es-issue}85839[#85839]) Features:: @@ -73,6 +75,9 @@ Infra/Core:: Infra/Logging:: * Temporarily provide `SystemPropertiesPropertySource` {es-pull}87149[#87149] +Infra/Node Lifecycle:: +* Upgrade folders after settings validation {es-pull}87319[#87319] + Infra/Plugins:: * Use Windows newlines when listing plugin information on Windows {es-pull}86408[#86408] (issue: {es-issue}86352[#86352]) @@ -82,6 +87,7 @@ Infra/REST API:: Infra/Scripting:: * Allow to sort by script value using `SemVer` semantics {es-pull}85990[#85990] (issues: {es-issue}85989[#85989], {es-issue}82287[#82287]) * Script: Fix setter shortcut for unbridged setters {es-pull}86868[#86868] +* Script: Load Whitelists as Resource {es-pull}87539[#87539] Infra/Settings:: * Permit removal of archived index settings {es-pull}86107[#86107] @@ -97,16 +103,24 @@ License:: Machine Learning:: * Fix ML task auditor exception early in cluster lifecycle {es-pull}87023[#87023] (issue: {es-issue}87002[#87002]) +* Fix `WordPiece` tokenization of unknown words with known subwords {es-pull}87510[#87510] * Fix distribution change check for `change_point` aggregation {es-pull}86423[#86423] +* Fixes inference timeout handling bug that throws unexpected `NullPointerException` {es-pull}87533[#87533] * Correct logic for restart from failover fine tuning hyperparameters for training classification and regression models {ml-pull}2251[#2251] * Fix possible source of "x = NaN, distribution = class boost::math::normal_distribution<..." log errors training classification and regression models {ml-pull}2249[#2249] * Fix some bugs affecting decision to stop optimizing hyperparameters for training classification and regression models {ml-pull}2259[#2259] * Fix cause of "Must provide points at which to evaluate function" log error training classification and regression models {ml-pull}2268[#2268] * Fix a source of "Discarding sample = nan, weights = ..." log errors for time series anomaly detection {ml-pull}2286[#2286] +Mapping:: +* Don't run `include_in_parent` when in `copy_to` context {es-pull}87123[#87123] (issue: {es-issue}87036[#87036]) + Network:: * Reject `openConnection` attempt while closing {es-pull}86315[#86315] (issue: {es-issue}86249[#86249]) +Recovery:: +* Fail shard if STARTED after master failover {es-pull}87451[#87451] (issue: {es-issue}87367[#87367]) + SQL:: * Fix FORMAT function to comply with Microsoft SQL Server specification {es-pull}86225[#86225] (issue: {es-issue}66560[#66560]) * Implement binary format support for SQL clear cursor {es-pull}84230[#84230] (issue: {es-issue}53359[#53359]) @@ -120,6 +134,7 @@ Search:: Security:: * Make user and role name constraint consistent with max document ID {es-pull}86728[#86728] (issue: {es-issue}66020[#66020]) +* Security plugin close releasable realms {es-pull}87429[#87429] (issue: {es-issue}86286[#86286]) Snapshot/Restore:: * DONE should mean fully processed in snapshot status {es-pull}86414[#86414] @@ -189,6 +204,7 @@ Geo:: Health:: * Add a basic check for tier preference and allocation filter clashing {es-pull}85071[#85071] * Add preflight checks to Health API to ensure health is obtainable {es-pull}86404[#86404] +* Add tier information on health api migrate tiers user actions {es-pull}87486[#87486] * Health api add indicator doc links {es-pull}86904[#86904] (issue: {es-issue}86892[#86892]) * Health api copy editing {es-pull}87010[#87010] * Return a default user action if no actions could be determined {es-pull}87079[#87079] @@ -276,6 +292,9 @@ Transform:: Authorization:: * Has privileges API for profiles {es-pull}85898[#85898] +Geo:: +* New geo_grid query to be used with geogrid aggregations {es-pull}86596[#86596] (issue: {es-issue}85727[#85727]) + Health:: * Add support for `impact_areas` to health impacts {es-pull}85830[#85830] (issue: {es-issue}85829[#85829]) * Add troubleshooting guides to shards allocation actions {es-pull}87078[#87078] diff --git a/docs/reference/release-notes/highlights.asciidoc b/docs/reference/release-notes/highlights.asciidoc index 8deb96bd98601..6d5bdecd3c722 100644 --- a/docs/reference/release-notes/highlights.asciidoc +++ b/docs/reference/release-notes/highlights.asciidoc @@ -19,19 +19,49 @@ Other versions: // tag::notable-highlights[] [discrete] -[[reading_indices_from_older_elasticsearch_versions]] -=== Reading indices from older Elasticsearch versions -Elasticsearch has full query and write support for indices created in the previous major -version. If you have indices created in Elasticsearch versions 5 or 6, you can now use -the archive functionality to import and query these indices as well. -The archive functionality provides slower read-only access to older data, -for compliance or regulatory reasons, the occasional lookback or investigation, -or to rehydrate parts of it. Access to the data is expected to be infrequent, -and can therefore happen with limited performance and query capabilities. +[[ingest_performance_improvement]] +=== {es-pull}84250[Ingest performance improvement] +We have improved the pipeline execution logic for pipelines with processors that are synchronous +by avoiding (deep) recursion. +On our nightly benchmark that simulates a Logging use-case, this resulted in a 10% reduction of +CPU time spent on ingest pipelines and a 3% overall ingestion speedup. + +[discrete] +[[execute_self_reference_checks_once_per_pipeline]] +=== {es-pull}85926[Execute self-reference checks once per pipeline] +Ingest pipelines have a mechanism to prevent circular references in the records +they process, so that they are serializable. Prior to this change, this check was +performed after each `script` processor execution, and was ignorable. + +Because of this check, a script processor configured with +``` +"source": """ + def x = ctx; + ctx.x = x; +""" +``` + +...would error with `"type" : "illegal_argument_exception", "reason" : "Iterable +object is self-referencing itself (ingest script)"`. + +If the script processor also had +``` +"ignore_failure" true +``` + +...then the handling thread would actually crash from an unrecoverable +StackOverflowError when trying to serialize the resulting event. + +Now, this check is performed once per pipeline, remediating the potential for +a StackOverflowError. There are some side effects also: + +- The resulting error message specifies which pipeline is causing the issue. +- There is a slight speed up for pipelines with multiple scripts, and a slight +slowdown for pipelines without scripts. [discrete] [[add_support_for_dots_in_field_names_for_metrics_usecases]] -=== Add support for dots in field names for metrics usecases +=== {es-pull}86166[Add support for dots in field names for metrics usecases] Metrics data can often be made of several fields with dots in their names, sharing common prefixes, like in the following example: @@ -71,37 +101,43 @@ without expanding dots in field names to the corresponding object structure. That makes it possible to store the metrics document above. [discrete] -[[execute_self_reference_checks_once_per_pipeline]] -=== Execute self-reference checks once per pipeline -Ingest pipelines have a mechanism to prevent circular references in the records -they process, so that they are serializable. Prior to this change, this check was -performed after each `script` processor execution, and was ignorable. - -Because of this check, a script processor configured with -``` -"source": """ - def x = ctx; - ctx.x = x; -""" -``` +[[reading_indices_from_older_elasticsearch_versions]] +=== {es-pull}86261[Reading indices from older Elasticsearch versions] +Elasticsearch has full query and write support for indices created in the previous major +version. If you have indices created in Elasticsearch versions 5 or 6, you can now use +the archive functionality to import and query these indices as well. +The archive functionality provides slower read-only access to older data, +for compliance or regulatory reasons, the occasional lookback or investigation, +or to rehydrate parts of it. Access to the data is expected to be infrequent, +and can therefore happen with limited performance and query capabilities. -...would error with `"type" : "illegal_argument_exception", "reason" : "Iterable -object is self-referencing itself (ingest script)"`. +[discrete] +[[new_geo_grid_query]] +=== {es-pull}86596[New geo_grid query] +With the geo grid query, you can now natively return all the documents that overlap a specific geo tile. +There is no need to reconstruct the geometry or the actual boundaries of the spatial cluster as Elasticsearch +can do this for you, which saves you time and reduces complexity. +This is especially useful when geometries are spread across tiles like on a soccer ball or football. +While hexagon tiles line the sphere, calculating the boundary of each tile is not straightforward. -If the script processor also had ``` -"ignore_failure" true +GET /example/_search +{ + "query": { + "geo_grid" :{ + "location" : { + "geotile" : "6/32/22" + } + } + } +} ``` -...then the handling thread would actually crash from an unrecoverable -StackOverflowError when trying to serialize the resulting event. - -Now, this check is performed once per pipeline, remediating the potential for -a StackOverflowError. There are some side effects also: - -- The resulting error message specifies which pipeline is causing the issue. -- There is a slight speed up for pipelines with multiple scripts, and a slight -slowdown for pipelines without scripts. +Geo grid query can also help determine the single source of truth of containment. +With geo grid query, you can match exactly the intersection-test of Elasticsearch. +As an example, if a client has bounds for a grid-cell at a higher (or lower) precision than what is used +by Elasticsearch when running a corresponding aggregation, the containment-check might be slightly different. +This side-steps any disconnect based on projection/datum difference between client and Elasticsearch. // end::notable-highlights[]