Merge branch 'main' into drop-closet-awaits

Author: smalyshev

.buildkite/pipelines/intake.yml

@@ -65,7 +65,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["8.17.10", "8.18.5", "8.19.1", "9.0.5", "9.1.1", "9.2.0"]
+            BWC_VERSION: ["8.17.10", "8.18.5", "8.19.2", "9.0.5", "9.1.2", "9.2.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2404

.buildkite/pipelines/periodic-packaging.yml

@@ -334,8 +334,8 @@ steps:
         env:
           BWC_VERSION: 8.18.5
 
-      - label: "{{matrix.image}} / 8.19.1 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.19.1
+      - label: "{{matrix.image}} / 8.19.2 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.19.2
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -348,7 +348,7 @@ steps:
           machineType: custom-16-32768
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 8.19.1
+          BWC_VERSION: 8.19.2
 
       - label: "{{matrix.image}} / 9.0.5 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v9.0.5
@@ -366,8 +366,8 @@ steps:
         env:
           BWC_VERSION: 9.0.5
 
-      - label: "{{matrix.image}} / 9.1.1 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v9.1.1
+      - label: "{{matrix.image}} / 9.1.2 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v9.1.2
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -380,7 +380,7 @@ steps:
           machineType: custom-16-32768
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 9.1.1
+          BWC_VERSION: 9.1.2
 
       - label: "{{matrix.image}} / 9.2.0 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v9.2.0

.buildkite/pipelines/periodic.template.yml

@@ -246,7 +246,7 @@ steps:
       image: family/elasticsearch-ubuntu-2404
       machineType: n2-standard-8
       buildDirectory: /dev/shm/bk
-    if: build.branch == "main" || build.branch == "8.19" || build.branch == "7.17"
+    if: build.branch == "main" || build.branch == "9.1" || build.branch == "9.0" || build.branch == "8.19" || build.branch == "8.18" || build.branch == "7.17"
   - label: check-branch-consistency
     command: .ci/scripts/run-gradle.sh branchConsistency
     timeout_in_minutes: 15

.buildkite/pipelines/periodic.yml

@@ -363,8 +363,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 8.19.1 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.19.1#bwcTest
+      - label: 8.19.2 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.19.2#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -373,7 +373,7 @@ steps:
           buildDirectory: /dev/shm/bk
           preemptible: true
         env:
-          BWC_VERSION: 8.19.1
+          BWC_VERSION: 8.19.2
         retry:
           automatic:
             - exit_status: "-1"
@@ -401,8 +401,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 9.1.1 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v9.1.1#bwcTest
+      - label: 9.1.2 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v9.1.2#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -411,7 +411,7 @@ steps:
           buildDirectory: /dev/shm/bk
           preemptible: true
         env:
-          BWC_VERSION: 9.1.1
+          BWC_VERSION: 9.1.2
         retry:
           automatic:
             - exit_status: "-1"
@@ -506,7 +506,7 @@ steps:
           setup:
             ES_RUNTIME_JAVA:
               - openjdk21
-            BWC_VERSION: ["8.17.10", "8.18.5", "8.19.1", "9.0.5", "9.1.1", "9.2.0"]
+            BWC_VERSION: ["8.17.10", "8.18.5", "8.19.2", "9.0.5", "9.1.2", "9.2.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2404
@@ -550,7 +550,7 @@ steps:
             ES_RUNTIME_JAVA:
               - openjdk21
               - openjdk23
-            BWC_VERSION: ["8.17.10", "8.18.5", "8.19.1", "9.0.5", "9.1.1", "9.2.0"]
+            BWC_VERSION: ["8.17.10", "8.18.5", "8.19.2", "9.0.5", "9.1.2", "9.2.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2404
@@ -684,7 +684,7 @@ steps:
       image: family/elasticsearch-ubuntu-2404
       machineType: n2-standard-8
       buildDirectory: /dev/shm/bk
-    if: build.branch == "main" || build.branch == "8.19" || build.branch == "7.17"
+    if: build.branch == "main" || build.branch == "9.1" || build.branch == "9.0" || build.branch == "8.19" || build.branch == "8.18" || build.branch == "7.17"
   - label: check-branch-consistency
     command: .ci/scripts/run-gradle.sh branchConsistency
     timeout_in_minutes: 15

.ci/bwcVersions

@@ -18,7 +18,7 @@ BWC_VERSION:
   - "8.16.6"
   - "8.17.10"
   - "8.18.5"
-  - "8.19.1"
+  - "8.19.2"
   - "9.0.5"
-  - "9.1.1"
+  - "9.1.2"
   - "9.2.0"

.ci/snapshotBwcVersions

@@ -1,7 +1,7 @@
 BWC_VERSION:
   - "8.17.10"
   - "8.18.5"
-  - "8.19.1"
+  - "8.19.2"
   - "9.0.5"
-  - "9.1.1"
+  - "9.1.2"
   - "9.2.0"

benchmarks/build.gradle

@@ -49,6 +49,7 @@ dependencies {
   api(project(':x-pack:plugin:esql:compute'))
   implementation project(path: ':libs:native')
   implementation project(path: ':libs:simdvec')
+  implementation project(path: ':libs:exponential-histogram')
   expression(project(path: ':modules:lang-expression', configuration: 'zip'))
   painless(project(path: ':modules:lang-painless', configuration: 'zip'))
   nativeLib(project(':libs:native'))

benchmarks/src/main/java/org/elasticsearch/benchmark/exponentialhistogram/ExponentialHistogramGenerationBench.java

@@ -0,0 +1,98 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.benchmark.exponentialhistogram;
+
+import org.elasticsearch.exponentialhistogram.ExponentialHistogramGenerator;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Threads;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.profile.GCProfiler;
+import org.openjdk.jmh.profile.StackProfiler;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
+
+import java.util.Random;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
+import java.util.function.DoubleSupplier;
+
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.NANOSECONDS)
+@Warmup(iterations = 3, time = 3, timeUnit = TimeUnit.SECONDS)
+@Measurement(iterations = 5, time = 2, timeUnit = TimeUnit.SECONDS)
+@Fork(1)
+@Threads(1)
+@State(Scope.Thread)
+public class ExponentialHistogramGenerationBench {
+
+    @Param({ "100", "500", "1000", "5000", "10000", "20000" })
+    int bucketCount;
+
+    @Param({ "NORMAL", "GAUSSIAN" })
+    String distribution;
+
+    Random random;
+    ExponentialHistogramGenerator histoGenerator;
+
+    double[] data = new double[1000000];
+
+    int index;
+
+    @Setup
+    public void setUp() {
+        random = ThreadLocalRandom.current();
+        histoGenerator = new ExponentialHistogramGenerator(bucketCount);
+
+        DoubleSupplier nextRandom = () -> distribution.equals("GAUSSIAN") ? random.nextGaussian() : random.nextDouble();
+
+        // Make sure that we start with a non-empty histogram, as this distorts initial additions
+        for (int i = 0; i < 10000; ++i) {
+            histoGenerator.add(nextRandom.getAsDouble());
+        }
+
+        for (int i = 0; i < data.length; ++i) {
+            data[i] = nextRandom.getAsDouble();
+        }
+
+        index = 0;
+    }
+
+    @Benchmark
+    @BenchmarkMode(Mode.AverageTime)
+    @OutputTimeUnit(TimeUnit.MICROSECONDS)
+    public void add() {
+        if (index >= data.length) {
+            index = 0;
+        }
+        histoGenerator.add(data[index++]);
+    }
+
+    public static void main(String[] args) throws RunnerException {
+        Options opt = new OptionsBuilder().include(".*" + ExponentialHistogramGenerationBench.class.getSimpleName() + ".*")
+            .warmupIterations(5)
+            .measurementIterations(5)
+            .addProfiler(GCProfiler.class)
+            .addProfiler(StackProfiler.class)
+            .build();
+
+        new Runner(opt).run();
+    }
+}

benchmarks/src/main/java/org/elasticsearch/benchmark/exponentialhistogram/ExponentialHistogramMergeBench.java

@@ -0,0 +1,112 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.benchmark.exponentialhistogram;
+
+import org.elasticsearch.exponentialhistogram.BucketIterator;
+import org.elasticsearch.exponentialhistogram.ExponentialHistogram;
+import org.elasticsearch.exponentialhistogram.ExponentialHistogramGenerator;
+import org.elasticsearch.exponentialhistogram.ExponentialHistogramMerger;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Threads;
+import org.openjdk.jmh.annotations.Warmup;
+
+import java.util.List;
+import java.util.Random;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
+
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.NANOSECONDS)
+@Warmup(iterations = 3, time = 3, timeUnit = TimeUnit.SECONDS)
+@Measurement(iterations = 5, time = 2, timeUnit = TimeUnit.SECONDS)
+@Fork(1)
+@Threads(1)
+@State(Scope.Thread)
+public class ExponentialHistogramMergeBench {
+
+    @Param({ "1000", "2000", "5000" })
+    int bucketCount;
+
+    @Param({ "0.01", "0.1", "0.25", "0.5", "1.0", "2.0" })
+    double mergedHistoSizeFactor;
+
+    Random random;
+    ExponentialHistogramMerger histoMerger;
+
+    ExponentialHistogram[] toMerge = new ExponentialHistogram[10_000];
+
+    int index;
+
+    @Setup
+    public void setUp() {
+        random = ThreadLocalRandom.current();
+        histoMerger = new ExponentialHistogramMerger(bucketCount);
+
+        ExponentialHistogramGenerator initial = new ExponentialHistogramGenerator(bucketCount);
+        for (int j = 0; j < bucketCount; j++) {
+            initial.add(Math.pow(1.001, j));
+        }
+        ExponentialHistogram initialHisto = initial.get();
+        int cnt = getBucketCount(initialHisto);
+        if (cnt < bucketCount) {
+            throw new IllegalArgumentException("Expected bucket count to be " + bucketCount + ", but was " + cnt);
+        }
+        histoMerger.add(initialHisto);
+
+        int dataPointSize = (int) Math.round(bucketCount * mergedHistoSizeFactor);
+
+        for (int i = 0; i < toMerge.length; i++) {
+            ExponentialHistogramGenerator generator = new ExponentialHistogramGenerator(dataPointSize);
+
+            int bucketIndex = 0;
+            for (int j = 0; j < dataPointSize; j++) {
+                bucketIndex += 1 + random.nextInt(bucketCount) % (Math.max(1, bucketCount / dataPointSize));
+                generator.add(Math.pow(1.001, bucketIndex));
+            }
+            toMerge[i] = generator.get();
+            cnt = getBucketCount(toMerge[i]);
+            if (cnt < dataPointSize) {
+                throw new IllegalArgumentException("Expected bucket count to be " + dataPointSize + ", but was " + cnt);
+            }
+        }
+
+        index = 0;
+    }
+
+    private static int getBucketCount(ExponentialHistogram histo) {
+        int cnt = 0;
+        for (BucketIterator it : List.of(histo.negativeBuckets().iterator(), histo.positiveBuckets().iterator())) {
+            while (it.hasNext()) {
+                cnt++;
+                it.advance();
+            }
+        }
+        return cnt;
+    }
+
+    @Benchmark
+    @BenchmarkMode(Mode.AverageTime)
+    @OutputTimeUnit(TimeUnit.MICROSECONDS)
+    public void add() {
+        if (index >= toMerge.length) {
+            index = 0;
+        }
+        histoMerger.add(toMerge[index++]);
+    }
+}