Fork TDigest library

benchmarks/src/main/java/org/elasticsearch/benchmark/tdigest/SortBench.java

@@ -0,0 +1,79 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ * This project is based on a modification of https://github.com/tdunning/t-digest which is licensed under the Apache 2.0 License.
+ */
+
+package org.elasticsearch.benchmark.tdigest;
+
+import org.elasticsearch.tdigest.Sort;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Threads;
+import org.openjdk.jmh.annotations.Warmup;
+
+import java.util.Arrays;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+
+/** Explores the performance of Sort on pathological input data. */
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
+@Warmup(iterations = 10, time = 3, timeUnit = TimeUnit.SECONDS)
+@Measurement(iterations = 20, time = 2, timeUnit = TimeUnit.SECONDS)
+@Fork(1)
+@Threads(1)
+@State(Scope.Thread)
+public class SortBench {
+    private final int size = 100000;
+    private final double[] values = new double[size];
+
+    @Param({ "0", "1", "-1" })
+    public int sortDirection;
+
+    @Setup
+    public void setup() {
+        Random prng = new Random(999983);
+        for (int i = 0; i < size; i++) {
+            values[i] = prng.nextDouble();
+        }
+        if (sortDirection > 0) {
+            Arrays.sort(values);
+        } else if (sortDirection < 0) {
+            Arrays.sort(values);
+            Sort.reverse(values, 0, values.length);
+        }
+    }
+
+    @Benchmark
+    public void quicksort() {
+        int[] order = new int[size];
+        for (int i = 0; i < size; i++) {
+            order[i] = i;
+        }
+        Sort.sort(order, values, null, values.length);
+    }
+}

benchmarks/src/main/java/org/elasticsearch/benchmark/tdigest/TDigestBench.java

@@ -0,0 +1,131 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ * This project is based on a modification of https://github.com/tdunning/t-digest which is licensed under the Apache 2.0 License.
+ */
+
+package org.elasticsearch.benchmark.tdigest;
+
+import org.elasticsearch.tdigest.AVLTreeDigest;
+import org.elasticsearch.tdigest.MergingDigest;
+import org.elasticsearch.tdigest.TDigest;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Threads;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.profile.GCProfiler;
+import org.openjdk.jmh.profile.StackProfiler;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
+
+import java.util.Random;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Supplier;
+
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.NANOSECONDS)
+@Warmup(iterations = 3, time = 3, timeUnit = TimeUnit.SECONDS)
+@Measurement(iterations = 5, time = 2, timeUnit = TimeUnit.SECONDS)
+@Fork(1)
+@Threads(1)
+@State(Scope.Thread)
+public class TDigestBench {
+
+    public enum TDigestFactory {
+        MERGE {
+            @Override
+            TDigest create(double compression) {
+                return new MergingDigest(compression, (int) (10 * compression));
+            }
+        },
+        AVL_TREE {
+            @Override
+            TDigest create(double compression) {
+                return new AVLTreeDigest(compression);
+            }
+        };
+
+        abstract TDigest create(double compression);
+    }
+
+    @Param({ "100", "300" })
+    double compression;
+
+    @Param({ "MERGE", "AVL_TREE" })
+    TDigestFactory tdigestFactory;
+
+    @Param({ "NORMAL", "GAUSSIAN" })
+    String distribution;
+
+    Random random;
+    TDigest tdigest;
+
+    double[] data = new double[1000000];
+
+    @Setup
+    public void setUp() {
+        random = ThreadLocalRandom.current();
+        tdigest = tdigestFactory.create(compression);
+
+        Supplier<Double> nextRandom = () -> distribution.equals("GAUSSIAN") ? random.nextGaussian() : random.nextDouble();
+        for (int i = 0; i < 10000; ++i) {
+            tdigest.add(nextRandom.get());
+        }
+
+        for (int i = 0; i < data.length; ++i) {
+            data[i] = nextRandom.get();
+        }
+    }
+
+    @State(Scope.Thread)
+    public static class ThreadState {
+        int index = 0;
+    }
+
+    @Benchmark
+    @BenchmarkMode(Mode.AverageTime)
+    @OutputTimeUnit(TimeUnit.MICROSECONDS)
+    public void add(ThreadState state) {
+        if (state.index >= data.length) {
+            state.index = 0;
+        }
+        tdigest.add(data[state.index++]);
+    }
+
+    public static void main(String[] args) throws RunnerException {
+        Options opt = new OptionsBuilder().include(".*" + TDigestBench.class.getSimpleName() + ".*")
+            .warmupIterations(5)
+            .measurementIterations(5)
+            .addProfiler(GCProfiler.class)
+            .addProfiler(StackProfiler.class)
+            .build();
+
+        new Runner(opt).run();
+    }
+}

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionModuleCheckTaskProvider.java

@@ -61,6 +61,7 @@ public class InternalDistributionModuleCheckTaskProvider {
         "org.elasticsearch.preallocate",
         "org.elasticsearch.securesm",
         "org.elasticsearch.server",
+        "org.elasticsearch.tdigest",
         "org.elasticsearch.xcontent"
     );
 
@@ -75,7 +76,7 @@ public class InternalDistributionModuleCheckTaskProvider {
 
     private static final Function<ModuleReference, String> toName = mref -> mref.descriptor().name();
 
-    private InternalDistributionModuleCheckTaskProvider() {};
+    private InternalDistributionModuleCheckTaskProvider() {}
 
     /** Registers the checkModules tasks, which contains all checks relevant to ES Java Modules. */
     static TaskProvider<Task> registerCheckModulesTask(Project project, TaskProvider<Copy> checkExtraction) {

docs/reference/aggregations/metrics/percentile-rank-aggregation.asciidoc

@@ -53,16 +53,16 @@ The response will look like this:
  "aggregations": {
     "load_time_ranks": {
       "values": {
-        "500.0": 90.01,
-        "600.0": 100.0
+        "500.0": 55.0,
+        "600.0": 64.0
       }
     }
   }
 }
 --------------------------------------------------
 // TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,"hits": $body.hits,/]
-// TESTRESPONSE[s/"500.0": 90.01/"500.0": 55.00000000000001/]
-// TESTRESPONSE[s/"600.0": 100.0/"600.0": 64.0/]
+// TESTRESPONSE[s/"500.0": 55.0/"500.0": 55.00000000000001/]
+// TESTRESPONSE[s/"600.0": 64.0/"600.0": 64.0/]
 
 From this information you can determine you are hitting the 99% load time target but not quite
 hitting the 95% load time target
@@ -101,20 +101,20 @@ Response:
       "values": [
         {
           "key": 500.0,
-          "value": 90.01
+          "value": 55.0
         },
         {
           "key": 600.0,
-          "value": 100.0
+          "value": 64.0
         }
       ]
     }
   }
 }
 --------------------------------------------------
 // TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,"hits": $body.hits,/]
-// TESTRESPONSE[s/"value": 90.01/"value": 55.00000000000001/]
-// TESTRESPONSE[s/"value": 100.0/"value": 64.0/]
+// TESTRESPONSE[s/"value": 55.0/"value": 55.00000000000001/]
+// TESTRESPONSE[s/"value": 64.0/"value": 64.0/]
 
 
 ==== Script

server/licenses/t-digest-LICENSE.txt → libs/tdigest/LICENSE.txt

@@ -1,4 +1,4 @@
-Apache License
+                                Apache License
                            Version 2.0, January 2004
                         http://www.apache.org/licenses/
 

libs/tdigest/NOTICES.txt

@@ -0,0 +1,28 @@
+Elastic-t-digest
+
+Copyright 2023 Elasticsearch B.V.
+
+--
+This project is based on a modification of https://github.com/tdunning/t-digest which is licensed under the Apache 2.0 License.
+
+Licensed to Elasticsearch B.V. under one or more contributor
+license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright
+ownership. Elasticsearch B.V. licenses this file to you under
+the Apache License, Version 2.0 (the "License"); you may
+not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ 	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+
+--
+The code for the t-digest was originally authored by Ted Dunning
+
+Adrien Grand contributed the heart of the AVLTreeDigest (https://github.com/jpountz)

libs/tdigest/build.gradle

@@ -0,0 +1,41 @@
+import org.elasticsearch.gradle.internal.conventions.precommit.LicenseHeadersTask
+
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+apply plugin: 'elasticsearch.build'
+apply plugin: 'elasticsearch.publish'
+
+dependencies {
+  testImplementation(project(":test:framework")) {
+    exclude group: 'org.elasticsearch', module: 'elasticsearch-tdigest'
+  }
+  testImplementation 'org.junit.jupiter:junit-jupiter:5.8.1'
+}
+
+tasks.named('forbiddenApisMain').configure {
+  // t-digest does not depend on core, so only jdk signatures should be checked
+  replaceSignatureFiles 'jdk-signatures'
+}
+
+ext.projectLicenses.set(['The Apache Software License, Version 2.0': 'http://www.apache.org/licenses/LICENSE-2.0'])
+licenseFile.set(rootProject.file('licenses/APACHE-LICENSE-2.0.txt'))
+
+tasks.withType(LicenseHeadersTask.class).configureEach {
+  approvedLicenses = ['Apache', 'Generated', 'Vendored']
+}

libs/tdigest/src/main/java/module-info.java

@@ -0,0 +1,22 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+module org.elasticsearch.tdigest {
+    exports org.elasticsearch.tdigest;
+}