diff --git a/.ci/bwcVersions b/.ci/bwcVersions index 3abfe35e5ea63..ef936fe88588a 100644 --- a/.ci/bwcVersions +++ b/.ci/bwcVersions @@ -28,7 +28,7 @@ BWC_VERSION: - "7.10.0" - "7.10.1" - "7.10.2" - - "7.10.3" - "7.11.0" + - "7.11.1" - "7.12.0" - "8.0.0" diff --git a/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/DistributionDownloadPluginFuncTest.groovy b/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/DistributionDownloadPluginFuncTest.groovy index 7ee368fee7687..c7d775dec85a3 100644 --- a/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/DistributionDownloadPluginFuncTest.groovy +++ b/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/DistributionDownloadPluginFuncTest.groovy @@ -51,19 +51,19 @@ class DistributionDownloadPluginFuncTest extends AbstractGradleFuncTest { when: def guh = new File(testProjectDir.getRoot(), "gradle-user-home").absolutePath; def runner = gradleRunner('clean', 'setupDistro', '-i', '-g', guh) + def unpackingMessage = "Unpacking elasticsearch-${version}-linux-${Architecture.current().classifier}.tar.gz " + + "using SymbolicLinkPreservingUntarTransform" def result = withMockedDistributionDownload(version, platform, runner) { // initial run def firstRun = build() - assertOutputContains(firstRun.output, "Unpacking elasticsearch-${version}-linux-x86_64.tar.gz " + - "using SymbolicLinkPreservingUntarTransform") + assertOutputContains(firstRun.output, unpackingMessage) // 2nd invocation build() } then: result.task(":setupDistro").outcome == TaskOutcome.SUCCESS - assertOutputMissing(result.output, "Unpacking elasticsearch-${version}-linux-x86_64.tar.gz " + - "using SymbolicLinkPreservingUntarTransform") + assertOutputMissing(result.output, unpackingMessage) } def "transforms are reused across projects"() { @@ -100,7 +100,7 @@ class DistributionDownloadPluginFuncTest extends AbstractGradleFuncTest { then: result.tasks.size() == 3 - result.output.count("Unpacking elasticsearch-${version}-linux-x86_64.tar.gz " + + result.output.count("Unpacking elasticsearch-${version}-linux-${Architecture.current().classifier}.tar.gz " + "using SymbolicLinkPreservingUntarTransform") == 1 } diff --git a/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/JdkDownloadPluginFuncTest.groovy b/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/JdkDownloadPluginFuncTest.groovy index c5184a79c7214..e80c9095f69c5 100644 --- a/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/JdkDownloadPluginFuncTest.groovy +++ b/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/JdkDownloadPluginFuncTest.groovy @@ -29,6 +29,8 @@ class JdkDownloadPluginFuncTest extends AbstractGradleFuncTest { private static final String OPENJDK_VERSION_OLD = "1+99" private static final String ADOPT_JDK_VERSION = "12.0.2+10" + private static final String ADOPT_JDK_VERSION_11 = "11.0.10+9" + private static final String ADOPT_JDK_VERSION_15 = "15.0.2+7" private static final String OPEN_JDK_VERSION = "12.0.1+99@123456789123456789123456789abcde" private static final String AZUL_AARCH_VERSION = "15.0.1+99@123456789123456789123456789abcde" private static final Pattern JDK_HOME_LOGLINE = Pattern.compile("JDK HOME: (.*)"); @@ -36,7 +38,7 @@ class JdkDownloadPluginFuncTest extends AbstractGradleFuncTest { @Unroll def "jdk #jdkVendor for #platform#suffix are downloaded and extracted"() { given: - def mockRepoUrl = urlPath(jdkVendor, jdkVersion, platform); + def mockRepoUrl = urlPath(jdkVendor, jdkVersion, platform, arch); def mockedContent = filebytes(jdkVendor, platform) buildFile.text = """ plugins { @@ -70,20 +72,22 @@ class JdkDownloadPluginFuncTest extends AbstractGradleFuncTest { assertExtraction(result.output, expectedJavaBin); where: - platform | arch | jdkVendor | jdkVersion | expectedJavaBin | suffix - "linux" | "x64" | VENDOR_ADOPTOPENJDK | ADOPT_JDK_VERSION | "bin/java" | "" - "linux" | "x64" | VENDOR_OPENJDK | OPEN_JDK_VERSION | "bin/java" | "" - "linux" | "x64" | VENDOR_OPENJDK | OPENJDK_VERSION_OLD | "bin/java" | "(old version)" - "windows" | "x64" | VENDOR_ADOPTOPENJDK | ADOPT_JDK_VERSION | "bin/java" | "" - "windows" | "x64" | VENDOR_OPENJDK | OPEN_JDK_VERSION | "bin/java" | "" - "windows" | "x64" | VENDOR_OPENJDK | OPENJDK_VERSION_OLD | "bin/java" | "(old version)" - "darwin" | "x64" | VENDOR_ADOPTOPENJDK | ADOPT_JDK_VERSION | "Contents/Home/bin/java" | "" - "darwin" | "x64" | VENDOR_OPENJDK | OPEN_JDK_VERSION | "Contents/Home/bin/java" | "" - "darwin" | "x64" | VENDOR_OPENJDK | OPENJDK_VERSION_OLD | "Contents/Home/bin/java" | "(old version)" - "mac" | "x64" | VENDOR_OPENJDK | OPEN_JDK_VERSION | "Contents/Home/bin/java" | "" - "mac" | "x64" | VENDOR_OPENJDK | OPENJDK_VERSION_OLD | "Contents/Home/bin/java" | "(old version)" - "darwin" | "aarch64" | VENDOR_AZUL | AZUL_AARCH_VERSION | "Contents/Home/bin/java" | "" - "linux" | "aarch64" | VENDOR_AZUL | AZUL_AARCH_VERSION | "bin/java" | "" + platform | arch | jdkVendor | jdkVersion | expectedJavaBin | suffix + "linux" | "x64" | VENDOR_ADOPTOPENJDK | ADOPT_JDK_VERSION | "bin/java" | "" + "linux" | "x64" | VENDOR_OPENJDK | OPEN_JDK_VERSION | "bin/java" | "" + "linux" | "x64" | VENDOR_OPENJDK | OPENJDK_VERSION_OLD | "bin/java" | "(old version)" + "windows" | "x64" | VENDOR_ADOPTOPENJDK | ADOPT_JDK_VERSION | "bin/java" | "" + "windows" | "x64" | VENDOR_OPENJDK | OPEN_JDK_VERSION | "bin/java" | "" + "windows" | "x64" | VENDOR_OPENJDK | OPENJDK_VERSION_OLD | "bin/java" | "(old version)" + "darwin" | "x64" | VENDOR_ADOPTOPENJDK | ADOPT_JDK_VERSION | "Contents/Home/bin/java" | "" + "darwin" | "x64" | VENDOR_OPENJDK | OPEN_JDK_VERSION | "Contents/Home/bin/java" | "" + "darwin" | "x64" | VENDOR_OPENJDK | OPENJDK_VERSION_OLD | "Contents/Home/bin/java" | "(old version)" + "mac" | "x64" | VENDOR_OPENJDK | OPEN_JDK_VERSION | "Contents/Home/bin/java" | "" + "mac" | "x64" | VENDOR_OPENJDK | OPENJDK_VERSION_OLD | "Contents/Home/bin/java" | "(old version)" + "darwin" | "aarch64" | VENDOR_AZUL | AZUL_AARCH_VERSION | "Contents/Home/bin/java" | "" + "linux" | "aarch64" | VENDOR_AZUL | AZUL_AARCH_VERSION | "bin/java" | "" + "linux" | "aarch64" | VENDOR_ADOPTOPENJDK | ADOPT_JDK_VERSION_11 | "bin/java" | "(jdk 11)" + "linux" | "aarch64" | VENDOR_ADOPTOPENJDK | ADOPT_JDK_VERSION_15 | "bin/java" | "(jdk 15)" } def "transforms are reused across projects"() { @@ -195,10 +199,13 @@ class JdkDownloadPluginFuncTest extends AbstractGradleFuncTest { true } - private static String urlPath(final String vendor, final String version, final String platform) { + private static String urlPath(final String vendor, + final String version, + final String platform, + final String arch = 'x64') { if (vendor.equals(VENDOR_ADOPTOPENJDK)) { final String module = isMac(platform) ? "mac" : platform; - return "/jdk-12.0.2+10/" + module + "/x64/jdk/hotspot/normal/adoptopenjdk"; + return "/jdk-" + version + "/" + module + "/${arch}/jdk/hotspot/normal/adoptopenjdk"; } else if (vendor.equals(VENDOR_OPENJDK)) { final String effectivePlatform = isMac(platform) ? "osx" : platform; final boolean isOld = version.equals(OPENJDK_VERSION_OLD); @@ -208,7 +215,7 @@ class JdkDownloadPluginFuncTest extends AbstractGradleFuncTest { } else if (vendor.equals(VENDOR_AZUL)) { final String module = isMac(platform) ? "macosx" : platform; // we only test zulu 15 darwin aarch64 for now - return "/zulu${module.equals('linux') ? '-embedded' : ''}/bin/zulu15.29.15-ca-jdk15.0.2-${module}_aarch64.tar.gz"; + return "/zulu${module.equals('linux') ? '-embedded' : ''}/bin/zulu15.29.15-ca-jdk15.0.2-${module}_${arch}.tar.gz"; } } diff --git a/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/fixtures/DistributionDownloadFixture.groovy b/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/fixtures/DistributionDownloadFixture.groovy index df6dab6ca1b3d..c9afcff1394c8 100644 --- a/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/fixtures/DistributionDownloadFixture.groovy +++ b/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/fixtures/DistributionDownloadFixture.groovy @@ -44,8 +44,7 @@ class DistributionDownloadFixture { private static String urlPath(String version,ElasticsearchDistribution.Platform platform) { String fileType = ((platform == ElasticsearchDistribution.Platform.LINUX || platform == ElasticsearchDistribution.Platform.DARWIN)) ? "tar.gz" : "zip" - String arch = Architecture.current() == Architecture.AARCH64 ? "aarch64" : "x86_64" - "/downloads/elasticsearch/elasticsearch-${version}-${platform}-${arch}.$fileType" + "/downloads/elasticsearch/elasticsearch-${version}-${platform}-${Architecture.current().classifier}.$fileType" } private static byte[] filebytes(String urlPath) throws IOException { diff --git a/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionDownloadPluginFuncTest.groovy b/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionDownloadPluginFuncTest.groovy index 22c8b53213545..8d0c00fddf01d 100644 --- a/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionDownloadPluginFuncTest.groovy +++ b/buildSrc/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionDownloadPluginFuncTest.groovy @@ -8,14 +8,11 @@ package org.elasticsearch.gradle.internal +import org.elasticsearch.gradle.Architecture import org.elasticsearch.gradle.VersionProperties import org.elasticsearch.gradle.fixtures.AbstractGradleFuncTest -import org.gradle.testkit.runner.GradleRunner import org.gradle.testkit.runner.TaskOutcome -import org.junit.Rule -import org.junit.rules.TemporaryFolder -import java.lang.management.ManagementFactory class InternalDistributionDownloadPluginFuncTest extends AbstractGradleFuncTest { @@ -61,7 +58,7 @@ class InternalDistributionDownloadPluginFuncTest extends AbstractGradleFuncTest def result = gradleRunner("setupDistro", '-g', testProjectDir.newFolder('GUH').path).build() then: - result.task(":distribution:archives:linux-tar:buildExpanded").outcome == TaskOutcome.SUCCESS + result.task(":distribution:archives:${testArchiveProjectName}:buildExpanded").outcome == TaskOutcome.SUCCESS result.task(":setupDistro").outcome == TaskOutcome.SUCCESS assertExtractedDistroIsCreated("build/distro", 'current-marker.txt') } @@ -133,24 +130,24 @@ class InternalDistributionDownloadPluginFuncTest extends AbstractGradleFuncTest apply plugin:'base' // packed distro - configurations.create("linux-tar") + configurations.create("${testArchiveProjectName}") tasks.register("buildBwcTask", Tar) { from('bwc-marker.txt') archiveExtension = "tar.gz" compression = Compression.GZIP } artifacts { - it.add("linux-tar", buildBwcTask) + it.add("${testArchiveProjectName}", buildBwcTask) } // expanded distro - configurations.create("expanded-linux-tar") + configurations.create("expanded-${testArchiveProjectName}") def expandedTask = tasks.register("buildBwcExpandedTask", Copy) { from('bwc-marker.txt') into('build/install/elastic-distro') } artifacts { - it.add("expanded-linux-tar", file('build/install')) { + it.add("expanded-${testArchiveProjectName}", file('build/install')) { builtBy expandedTask type = 'directory' } @@ -160,9 +157,9 @@ class InternalDistributionDownloadPluginFuncTest extends AbstractGradleFuncTest private void localDistroSetup() { settingsFile << """ - include ":distribution:archives:linux-tar" + include ":distribution:archives:${testArchiveProjectName}" """ - def bwcSubProjectFolder = testProjectDir.newFolder("distribution", "archives", "linux-tar") + def bwcSubProjectFolder = testProjectDir.newFolder("distribution", "archives", testArchiveProjectName) new File(bwcSubProjectFolder, 'current-marker.txt') << "current" new File(bwcSubProjectFolder, 'build.gradle') << """ import org.gradle.api.internal.artifacts.ArtifactAttributes; @@ -190,10 +187,12 @@ class InternalDistributionDownloadPluginFuncTest extends AbstractGradleFuncTest it.add("extracted", buildExpanded) } """ - buildFile << """ - """ } + String getTestArchiveProjectName() { + def archSuffix = Architecture.current() == Architecture.AARCH64 ? '-aarch64' : '' + return "linux${archSuffix}-tar" + } boolean assertExtractedDistroIsCreated(String relativeDistroPath, String markerFileName) { File extractedFolder = new File(testProjectDir.root, relativeDistroPath) assert extractedFolder.exists() diff --git a/buildSrc/src/integTest/resources/org/elasticsearch/gradle/internal/fake_git/remote/.ci/java-versions.properties b/buildSrc/src/integTest/resources/org/elasticsearch/gradle/internal/fake_git/remote/.ci/java-versions.properties index 0ccc6bab285a6..4f728deb42e0c 100644 --- a/buildSrc/src/integTest/resources/org/elasticsearch/gradle/internal/fake_git/remote/.ci/java-versions.properties +++ b/buildSrc/src/integTest/resources/org/elasticsearch/gradle/internal/fake_git/remote/.ci/java-versions.properties @@ -5,6 +5,6 @@ # in compliance with, at your election, the Elastic License 2.0 or the Server # Side Public License, v 1. # -ES_BUILD_JAVA=openjdk12 -ES_RUNTIME_JAVA=openjdk12 +ES_BUILD_JAVA=openjdk11 +ES_RUNTIME_JAVA=openjdk11 GRADLE_TASK=build diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/Architecture.java b/buildSrc/src/main/java/org/elasticsearch/gradle/Architecture.java index c9a7e1eeb8edd..ccc81954aabcb 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/Architecture.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/Architecture.java @@ -10,8 +10,14 @@ public enum Architecture { - X64, - AARCH64; + X64("x86_64"), + AARCH64("aarch64"); + + public final String classifier; + + Architecture(String classifier) { + this.classifier = classifier; + } public static Architecture current() { final String architecture = System.getProperty("os.arch", ""); diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java index 32cdfc5ecef96..5772a29d20ec7 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java @@ -166,14 +166,11 @@ private String dependencyNotation(ElasticsearchDistribution distribution) { Version distroVersion = Version.fromString(distribution.getVersion()); String extension = distribution.getType().toString(); - String classifier = ":" + (Architecture.current() == Architecture.AARCH64 ? "aarch64" : "x86_64"); + String classifier = ":" + Architecture.current().classifier; if (distribution.getType() == Type.ARCHIVE) { extension = distribution.getPlatform() == Platform.WINDOWS ? "zip" : "tar.gz"; if (distroVersion.onOrAfter("7.0.0")) { - classifier = ":" - + distribution.getPlatform() - + "-" - + (Architecture.current() == Architecture.AARCH64 ? "aarch64" : "x86_64"); + classifier = ":" + distribution.getPlatform() + "-" + Architecture.current().classifier; } else { classifier = ""; } diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/BwcVersionsTests.java b/buildSrc/src/test/java/org/elasticsearch/gradle/BwcVersionsTests.java index a9db70d365dfa..7e340a1682729 100644 --- a/buildSrc/src/test/java/org/elasticsearch/gradle/BwcVersionsTests.java +++ b/buildSrc/src/test/java/org/elasticsearch/gradle/BwcVersionsTests.java @@ -1,6 +1,8 @@ package org.elasticsearch.gradle; import org.elasticsearch.gradle.test.GradleUnitTestCase; +import org.junit.Assume; +import org.junit.BeforeClass; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; @@ -279,6 +281,11 @@ public class BwcVersionsTests extends GradleUnitTestCase { sampleVersions.put("7.1.0", asList("7_1_0", "7_0_0", "6_7_0", "6_6_1", "6_6_0")); } + @BeforeClass + public static void setupAll() { + Assume.assumeFalse(Architecture.current() == Architecture.AARCH64); + } + @Test(expected = IllegalArgumentException.class) public void testExceptionOnEmpty() { new BwcVersions(asList("foo", "bar"), Version.fromString("7.0.0")); diff --git a/docs/reference/analysis/token-graphs.asciidoc b/docs/reference/analysis/token-graphs.asciidoc index 20f91891aed5b..dfd700176c99c 100644 --- a/docs/reference/analysis/token-graphs.asciidoc +++ b/docs/reference/analysis/token-graphs.asciidoc @@ -39,6 +39,10 @@ record the `positionLength` for multi-position tokens. This filters include: * <> * <> +Some tokenizers, such as the +{plugin}/analysis-nori-tokenizer.html[`nori_tokenizer`], also accurately +decompose compound tokens into multi-position tokens. + In the following graph, `domain name system` and its synonym, `dns`, both have a position of `0`. However, `dns` has a `positionLength` of `3`. Other tokens in the graph have a default `positionLength` of `1`. diff --git a/docs/reference/query-dsl/simple-query-string-query.asciidoc b/docs/reference/query-dsl/simple-query-string-query.asciidoc index 9596191f24d9d..7927f36367dac 100644 --- a/docs/reference/query-dsl/simple-query-string-query.asciidoc +++ b/docs/reference/query-dsl/simple-query-string-query.asciidoc @@ -86,9 +86,10 @@ query string into tokens. Defaults to the `default_field`. If no analyzer is mapped, the index's default analyzer is used. `auto_generate_synonyms_phrase_query`:: -(Optional, Boolean) If `true`, <> -queries are automatically created for multi-term synonyms. Defaults to `true`. -See <> for an example. +(Optional, Boolean) If `true`, the parser creates a +<> query for each +<>. Defaults to `true`. +For examples, see <>. `flags`:: (Optional, string) List of enabled operators for the @@ -273,33 +274,36 @@ GET /_search <1> The `subject` field is three times as important as the `message` field. [[simple-query-string-synonyms]] -===== Synonyms +===== Multi-position tokens -The `simple_query_string` query supports multi-terms synonym expansion with the <> token filter. When this filter is used, the parser creates a phrase query for each multi-terms synonyms. -For example, the following synonym: `"ny, new york"` would produce: +By default, the `simple_query_string` query parser creates a +<> query for each +<> in the query string. +For example, the parser creates a `match_phrase` query for the multi-word +synonym `ny, new york`: `(ny OR ("new york"))` -It is also possible to match multi terms synonyms with conjunctions instead: +To match multi-position tokens with an `AND` conjunction instead, set +`auto_generate_synonyms_phrase_query` to `false`: [source,console] --------------------------------------------------- +---- GET /_search { - "query": { - "simple_query_string" : { - "query" : "ny city", - "auto_generate_synonyms_phrase_query" : false - } - } + "query": { + "simple_query_string": { + "query": "ny city", + "auto_generate_synonyms_phrase_query": false + } + } } --------------------------------------------------- +---- -The example above creates a boolean query: +For the above example, the parser creates the following +<> query: `(ny OR (new AND york)) city)` -that matches documents with the term `ny` or the conjunction `new AND york`. -By default the parameter `auto_generate_synonyms_phrase_query` is set to `true`. - +This `bool` query matches documents with the term `ny` or the conjunction +`new AND york`. diff --git a/docs/reference/sql/endpoints/translate.asciidoc b/docs/reference/sql/endpoints/translate.asciidoc index fdccbf00956b4..086efe1e1de9c 100644 --- a/docs/reference/sql/endpoints/translate.asciidoc +++ b/docs/reference/sql/endpoints/translate.asciidoc @@ -22,20 +22,22 @@ Which returns: -------------------------------------------------- { "size": 10, - "docvalue_fields": [ + "_source": false, + "fields": [ + { + "field": "author" + }, + { + "field": "name" + }, + { + "field": "page_count" + }, { "field": "release_date", "format": "epoch_millis" } ], - "_source": { - "includes": [ - "author", - "name", - "page_count" - ], - "excludes": [] - }, "sort": [ { "page_count": { diff --git a/libs/core/src/main/java/org/elasticsearch/common/compatibility/RestApiCompatibleVersion.java b/libs/core/src/main/java/org/elasticsearch/common/compatibility/RestApiCompatibleVersion.java new file mode 100644 index 0000000000000..f78661aef1079 --- /dev/null +++ b/libs/core/src/main/java/org/elasticsearch/common/compatibility/RestApiCompatibleVersion.java @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.common.compatibility; + +/** + * A enum representing versions which are used by a REST Compatible API. + * A CURRENT instance, represents a major Version.CURRENT from server module. + * + * Only major versions are supported. + */ +public enum RestApiCompatibleVersion { + + V_8(8), + V_7(7); + + public byte major; + private static RestApiCompatibleVersion CURRENT = V_8; + + RestApiCompatibleVersion(int major) { + this.major = (byte) major; + } + + public RestApiCompatibleVersion previousMajor() { + return fromMajorVersion(major - 1); + } + + public static RestApiCompatibleVersion fromMajorVersion(int majorVersion) { + return valueOf("V_" + majorVersion); + } + + public static RestApiCompatibleVersion minimumSupported() { + return currentVersion().previousMajor(); + } + + public static RestApiCompatibleVersion currentVersion() { + return CURRENT; + }; +} diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/NamedXContentRegistry.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/NamedXContentRegistry.java index 0d7a454ac1bab..6a7964883f05f 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/NamedXContentRegistry.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/NamedXContentRegistry.java @@ -10,6 +10,7 @@ import org.elasticsearch.common.CheckedFunction; import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.compatibility.RestApiCompatibleVersion; import java.io.IOException; import java.util.ArrayList; @@ -138,7 +139,8 @@ private Map, Map> getRegistry(List entries){ */ public T parseNamedObject(Class categoryClass, String name, XContentParser parser, C context) throws IOException { - Map parsers = parser.useCompatibility() ? compatibleRegistry.get(categoryClass) : registry.get(categoryClass); + Map parsers = parser.getRestApiCompatibleVersion() == RestApiCompatibleVersion.minimumSupported() ? + compatibleRegistry.get(categoryClass) : registry.get(categoryClass); if (parsers == null) { if (registry.isEmpty()) { // The "empty" registry will never work so we throw a better exception as a hint. diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContent.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContent.java index 49dbb5a0a99dc..839a9b13a84b1 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContent.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContent.java @@ -8,6 +8,8 @@ package org.elasticsearch.common.xcontent; +import org.elasticsearch.common.compatibility.RestApiCompatibleVersion; + import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -76,9 +78,10 @@ XContentParser createParser(NamedXContentRegistry xContentRegistry, DeprecationH /** * Creates a parser over the provided input stream and with the indication that a request is using REST compatible API. - * Parses XContent using the N-1 compatible logic. + * Depending on restApiCompatibleVersionParses + * @param restApiCompatibleVersion - indicates if the N-1 or N compatible XContent parsing logic will be used. */ XContentParser createParserForCompatibility(NamedXContentRegistry xContentRegistry, DeprecationHandler deprecationHandler, - InputStream is) throws IOException; + InputStream is, RestApiCompatibleVersion restApiCompatibleVersion) throws IOException; } diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java index f3ddff5b58746..708f42df752df 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java @@ -8,6 +8,8 @@ package org.elasticsearch.common.xcontent; +import org.elasticsearch.common.compatibility.RestApiCompatibleVersion; + import java.io.ByteArrayOutputStream; import java.io.Closeable; import java.io.Flushable; @@ -155,7 +157,7 @@ public interface HumanReadableTransformer { */ private boolean humanReadable = false; - private byte compatibleMajorVersion; + private RestApiCompatibleVersion restApiCompatibilityVersion; private ParsedMediaType responseContentType; @@ -1006,21 +1008,21 @@ public XContentBuilder copyCurrentStructure(XContentParser parser) throws IOExce /** * Sets a version used for serialising a response compatible with a previous version. + * @param restApiCompatibleVersion - indicates requested a version of API that the builder will be creating */ - public XContentBuilder withCompatibleMajorVersion(byte compatibleMajorVersion) { - assert this.compatibleMajorVersion == 0 : "Compatible version has already been set"; - if (compatibleMajorVersion == 0) { - throw new IllegalArgumentException("Compatible major version must not be equal to 0"); - } - this.compatibleMajorVersion = compatibleMajorVersion; + public XContentBuilder withCompatibleVersion(RestApiCompatibleVersion restApiCompatibleVersion) { + assert this.restApiCompatibilityVersion == null : "restApiCompatibleVersion has already been set"; + Objects.requireNonNull(restApiCompatibleVersion, "restApiCompatibleVersion cannot be null"); + this.restApiCompatibilityVersion = restApiCompatibleVersion; return this; } /** - * Returns a version used for serialising a response compatible with a previous version. + * Returns a version used for serialising a response. + * @return a compatible version */ - public byte getCompatibleMajorVersion() { - return compatibleMajorVersion; + public RestApiCompatibleVersion getRestApiCompatibilityVersion() { + return restApiCompatibilityVersion; } @Override diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentParser.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentParser.java index d653f8becfe3f..b7dea17ee5d70 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentParser.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentParser.java @@ -9,6 +9,7 @@ package org.elasticsearch.common.xcontent; import org.elasticsearch.common.CheckedFunction; +import org.elasticsearch.common.compatibility.RestApiCompatibleVersion; import java.io.Closeable; import java.io.IOException; @@ -251,7 +252,7 @@ Map map( boolean isClosed(); - boolean useCompatibility(); + RestApiCompatibleVersion getRestApiCompatibleVersion(); /** * The callback to notify when parsing encounters a deprecated field. diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentSubParser.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentSubParser.java index c9c214470426c..b318bc5ad343d 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentSubParser.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentSubParser.java @@ -9,6 +9,7 @@ package org.elasticsearch.common.xcontent; import org.elasticsearch.common.CheckedFunction; +import org.elasticsearch.common.compatibility.RestApiCompatibleVersion; import java.io.IOException; import java.nio.CharBuffer; @@ -259,8 +260,8 @@ public boolean isClosed() { } @Override - public boolean useCompatibility() { - return parser.useCompatibility(); + public RestApiCompatibleVersion getRestApiCompatibleVersion() { + return parser.getRestApiCompatibleVersion(); } @Override diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContent.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContent.java index 864e3089f2279..bad16abe7d216 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContent.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContent.java @@ -12,6 +12,7 @@ import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.dataformat.cbor.CBORFactory; +import org.elasticsearch.common.compatibility.RestApiCompatibleVersion; import org.elasticsearch.common.xcontent.DeprecationHandler; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContent; @@ -98,9 +99,10 @@ public XContentParser createParser(NamedXContentRegistry xContentRegistry, @Override public XContentParser createParserForCompatibility(NamedXContentRegistry xContentRegistry, - DeprecationHandler deprecationHandler, InputStream is) + DeprecationHandler deprecationHandler, InputStream is, + RestApiCompatibleVersion restApiCompatibleVersion) throws IOException { - return new CborXContentParser(xContentRegistry, deprecationHandler, cborFactory.createParser(is), true); + return new CborXContentParser(xContentRegistry, deprecationHandler, cborFactory.createParser(is), restApiCompatibleVersion); } } diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContentParser.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContentParser.java index 67e848cc5c368..a12b50efc7aac 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContentParser.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/cbor/CborXContentParser.java @@ -9,6 +9,7 @@ package org.elasticsearch.common.xcontent.cbor; import com.fasterxml.jackson.core.JsonParser; +import org.elasticsearch.common.compatibility.RestApiCompatibleVersion; import org.elasticsearch.common.xcontent.DeprecationHandler; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContentType; @@ -22,8 +23,9 @@ public CborXContentParser(NamedXContentRegistry xContentRegistry, } public CborXContentParser(NamedXContentRegistry xContentRegistry, - DeprecationHandler deprecationHandler, JsonParser parser, boolean useCompatibility) { - super(xContentRegistry, deprecationHandler, parser, useCompatibility); + DeprecationHandler deprecationHandler, JsonParser parser, + RestApiCompatibleVersion restApiCompatibleVersion) { + super(xContentRegistry, deprecationHandler, parser, restApiCompatibleVersion); } @Override diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContent.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContent.java index c237169b7abe5..21fb8e9c3e4e5 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContent.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContent.java @@ -12,6 +12,7 @@ import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.core.JsonParser; +import org.elasticsearch.common.compatibility.RestApiCompatibleVersion; import org.elasticsearch.common.xcontent.DeprecationHandler; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContent; @@ -99,8 +100,9 @@ public XContentParser createParser(NamedXContentRegistry xContentRegistry, @Override public XContentParser createParserForCompatibility(NamedXContentRegistry xContentRegistry, - DeprecationHandler deprecationHandler, InputStream is) throws IOException { - return new JsonXContentParser(xContentRegistry, deprecationHandler, jsonFactory.createParser(is), true); + DeprecationHandler deprecationHandler, InputStream is, + RestApiCompatibleVersion restApiCompatibleVersion) throws IOException { + return new JsonXContentParser(xContentRegistry, deprecationHandler, jsonFactory.createParser(is), restApiCompatibleVersion); } } diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentParser.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentParser.java index 801c6058f3da4..20038b1dc3c1c 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentParser.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentParser.java @@ -11,6 +11,7 @@ import com.fasterxml.jackson.core.JsonLocation; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.core.JsonToken; +import org.elasticsearch.common.compatibility.RestApiCompatibleVersion; import org.elasticsearch.common.xcontent.DeprecationHandler; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContentLocation; @@ -27,13 +28,14 @@ public class JsonXContentParser extends AbstractXContentParser { public JsonXContentParser(NamedXContentRegistry xContentRegistry, DeprecationHandler deprecationHandler, JsonParser parser) { - super(xContentRegistry, deprecationHandler, false); + super(xContentRegistry, deprecationHandler, RestApiCompatibleVersion.currentVersion()); this.parser = parser; } public JsonXContentParser(NamedXContentRegistry xContentRegistry, - DeprecationHandler deprecationHandler, JsonParser parser, boolean useCompatibility) { - super(xContentRegistry, deprecationHandler, useCompatibility); + DeprecationHandler deprecationHandler, JsonParser parser, + RestApiCompatibleVersion restApiCompatibleVersion) { + super(xContentRegistry, deprecationHandler, restApiCompatibleVersion); this.parser = parser; } diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/smile/SmileXContent.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/smile/SmileXContent.java index 0b240b49a79cf..c077ba41a4eb8 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/smile/SmileXContent.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/smile/SmileXContent.java @@ -13,6 +13,7 @@ import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.dataformat.smile.SmileFactory; import com.fasterxml.jackson.dataformat.smile.SmileGenerator; +import org.elasticsearch.common.compatibility.RestApiCompatibleVersion; import org.elasticsearch.common.xcontent.DeprecationHandler; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContent; @@ -100,7 +101,8 @@ public XContentParser createParser(NamedXContentRegistry xContentRegistry, @Override public XContentParser createParserForCompatibility(NamedXContentRegistry xContentRegistry, - DeprecationHandler deprecationHandler, InputStream is) throws IOException { - return new SmileXContentParser(xContentRegistry, deprecationHandler, smileFactory.createParser(is), true); + DeprecationHandler deprecationHandler, InputStream is, + RestApiCompatibleVersion restApiCompatibleVersion) throws IOException { + return new SmileXContentParser(xContentRegistry, deprecationHandler, smileFactory.createParser(is), restApiCompatibleVersion); } } diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/smile/SmileXContentParser.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/smile/SmileXContentParser.java index 68c424b73f890..12f901fb1c3c0 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/smile/SmileXContentParser.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/smile/SmileXContentParser.java @@ -9,6 +9,7 @@ package org.elasticsearch.common.xcontent.smile; import com.fasterxml.jackson.core.JsonParser; +import org.elasticsearch.common.compatibility.RestApiCompatibleVersion; import org.elasticsearch.common.xcontent.DeprecationHandler; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContentType; @@ -22,8 +23,9 @@ public SmileXContentParser(NamedXContentRegistry xContentRegistry, } public SmileXContentParser(NamedXContentRegistry xContentRegistry, - DeprecationHandler deprecationHandler, JsonParser parser, boolean useCompatibility) { - super(xContentRegistry, deprecationHandler, parser, useCompatibility); + DeprecationHandler deprecationHandler, JsonParser parser, + RestApiCompatibleVersion restApiCompatibleVersion) { + super(xContentRegistry, deprecationHandler, parser, restApiCompatibleVersion); } @Override diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/support/AbstractXContentParser.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/support/AbstractXContentParser.java index 43a150f72db55..c2ae840dcc12e 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/support/AbstractXContentParser.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/support/AbstractXContentParser.java @@ -10,6 +10,7 @@ import org.elasticsearch.common.Booleans; import org.elasticsearch.common.CheckedFunction; +import org.elasticsearch.common.compatibility.RestApiCompatibleVersion; import org.elasticsearch.common.xcontent.DeprecationHandler; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContentParseException; @@ -46,16 +47,17 @@ private static void checkCoerceString(boolean coerce, Class cl private final NamedXContentRegistry xContentRegistry; private final DeprecationHandler deprecationHandler; - private final boolean useCompatibility; + private final RestApiCompatibleVersion restApiCompatibleVersion; - public AbstractXContentParser(NamedXContentRegistry xContentRegistry, DeprecationHandler deprecationHandler, boolean useCompatibility) { + public AbstractXContentParser(NamedXContentRegistry xContentRegistry, DeprecationHandler deprecationHandler, + RestApiCompatibleVersion restApiCompatibleVersion) { this.xContentRegistry = xContentRegistry; this.deprecationHandler = deprecationHandler; - this.useCompatibility = useCompatibility; + this.restApiCompatibleVersion = restApiCompatibleVersion; } public AbstractXContentParser(NamedXContentRegistry xContentRegistry, DeprecationHandler deprecationHandler) { - this(xContentRegistry, deprecationHandler, false); + this(xContentRegistry, deprecationHandler, RestApiCompatibleVersion.currentVersion()); } // The 3rd party parsers we rely on are known to silently truncate fractions: see @@ -413,8 +415,8 @@ public NamedXContentRegistry getXContentRegistry() { public abstract boolean isClosed(); @Override - public boolean useCompatibility() { - return useCompatibility; + public RestApiCompatibleVersion getRestApiCompatibleVersion() { + return restApiCompatibleVersion; } @Override diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/yaml/YamlXContent.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/yaml/YamlXContent.java index f271851698aa2..8ce0c4144118e 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/yaml/YamlXContent.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/yaml/YamlXContent.java @@ -11,6 +11,7 @@ import com.fasterxml.jackson.core.JsonEncoding; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import org.elasticsearch.common.compatibility.RestApiCompatibleVersion; import org.elasticsearch.common.xcontent.DeprecationHandler; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContent; @@ -93,8 +94,9 @@ public XContentParser createParser(NamedXContentRegistry xContentRegistry, @Override public XContentParser createParserForCompatibility(NamedXContentRegistry xContentRegistry, - DeprecationHandler deprecationHandler, InputStream is) throws IOException { - return new YamlXContentParser(xContentRegistry, deprecationHandler, yamlFactory.createParser(is), true); + DeprecationHandler deprecationHandler, InputStream is, + RestApiCompatibleVersion restApiCompatibleVersion) throws IOException { + return new YamlXContentParser(xContentRegistry, deprecationHandler, yamlFactory.createParser(is), restApiCompatibleVersion); } } diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/yaml/YamlXContentParser.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/yaml/YamlXContentParser.java index 02f9b3b3f5907..35d85dca59fbb 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/yaml/YamlXContentParser.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/yaml/YamlXContentParser.java @@ -9,6 +9,7 @@ package org.elasticsearch.common.xcontent.yaml; import com.fasterxml.jackson.core.JsonParser; +import org.elasticsearch.common.compatibility.RestApiCompatibleVersion; import org.elasticsearch.common.xcontent.DeprecationHandler; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContentType; @@ -22,8 +23,9 @@ public YamlXContentParser(NamedXContentRegistry xContentRegistry, } public YamlXContentParser(NamedXContentRegistry xContentRegistry, - DeprecationHandler deprecationHandler, JsonParser parser, boolean useCompatibility) { - super(xContentRegistry, deprecationHandler, parser, useCompatibility); + DeprecationHandler deprecationHandler, JsonParser parser, + RestApiCompatibleVersion restApiCompatibleVersion) { + super(xContentRegistry, deprecationHandler, parser, restApiCompatibleVersion); } @Override diff --git a/modules/lang-painless/build.gradle b/modules/lang-painless/build.gradle index 1cad5898d2042..38ec117401490 100644 --- a/modules/lang-painless/build.gradle +++ b/modules/lang-painless/build.gradle @@ -77,6 +77,7 @@ dependencies { docImplementation project(':server') docImplementation project(':modules:lang-painless') docImplementation 'com.github.javaparser:javaparser-core:3.18.0' + docImplementation 'org.jsoup:jsoup:1.13.1' if (isEclipse) { /* * Eclipse isn't quite "with it" enough to understand the different @@ -84,6 +85,7 @@ dependencies { * can compile the doc java files. */ implementation 'com.github.javaparser:javaparser-core:3.18.0' + implementation 'org.jsoup:jsoup:1.13.1' } } diff --git a/modules/lang-painless/src/doc/java/org/elasticsearch/painless/JavadocExtractor.java b/modules/lang-painless/src/doc/java/org/elasticsearch/painless/JavadocExtractor.java index 2b27c0cedb247..4e572ae65fb2b 100644 --- a/modules/lang-painless/src/doc/java/org/elasticsearch/painless/JavadocExtractor.java +++ b/modules/lang-painless/src/doc/java/org/elasticsearch/painless/JavadocExtractor.java @@ -18,9 +18,19 @@ import com.github.javaparser.ast.comments.Comment; import com.github.javaparser.ast.visitor.VoidVisitorAdapter; import com.github.javaparser.javadoc.Javadoc; +import com.github.javaparser.javadoc.JavadocBlockTag; +import com.github.javaparser.javadoc.description.JavadocDescription; +import com.github.javaparser.javadoc.description.JavadocDescriptionElement; +import com.github.javaparser.javadoc.description.JavadocInlineTag; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.jsoup.Jsoup; +import org.jsoup.safety.Whitelist; import java.io.IOException; import java.io.InputStream; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -94,7 +104,7 @@ public void putMethod(MethodDeclaration declaration) { methods.put( MethodSignature.fromDeclaration(declaration), new ParsedMethod( - declaration.getJavadoc().map(Javadoc::toText).orElse(""), + declaration.getJavadoc().map(JavadocExtractor::clean).orElse(null), declaration.getParameters() .stream() .map(p -> p.getName().asString()) @@ -110,7 +120,7 @@ public void putConstructor(ConstructorDeclaration declaration) { constructors.put( declaration.getParameters().stream().map(p -> stripTypeParameters(p.getType().asString())).collect(Collectors.toList()), new ParsedMethod( - declaration.getJavadoc().map(Javadoc::toText).orElse(""), + declaration.getJavadoc().map(JavadocExtractor::clean).orElse(null), declaration.getParameters() .stream() .map(p -> p.getName().asString()) @@ -152,7 +162,7 @@ public void putField(FieldDeclaration declaration) { return; } for (VariableDeclarator var : declaration.getVariables()) { - fields.put(var.getNameAsString(), declaration.getJavadoc().map(Javadoc::toText).orElse("")); + fields.put(var.getNameAsString(), declaration.getJavadoc().map(v -> JavadocExtractor.clean(v).description).orElse("")); } } } @@ -192,15 +202,103 @@ public int hashCode() { } public static class ParsedMethod { - public final String javadoc; + public final ParsedJavadoc javadoc; public final List parameterNames; - public ParsedMethod(String javadoc, List parameterNames) { + public ParsedMethod(ParsedJavadoc javadoc, List parameterNames) { this.javadoc = javadoc; this.parameterNames = parameterNames; } } + public static ParsedJavadoc clean(Javadoc javadoc) { + JavadocDescription description = javadoc.getDescription(); + List tags = javadoc.getBlockTags(); + List cleaned = new ArrayList<>(description.getElements().size() + tags.size()); + cleaned.addAll(stripInlineTags(description)); + ParsedJavadoc parsed = new ParsedJavadoc(cleaned(cleaned)); + for (JavadocBlockTag tag: tags) { + String tagName = tag.getTagName(); + // https://docs.oracle.com/en/java/javase/14/docs/specs/javadoc/doc-comment-spec.html#standard-tags + // ignore author, deprecated, hidden, provides, uses, see, serial*, since and version. + if ("param".equals(tagName)) { + tag.getName().ifPresent(t -> parsed.param.put(t, cleaned(stripInlineTags(tag.getContent())))); + } else if ("return".equals(tagName)) { + parsed.returns = cleaned(stripInlineTags(tag.getContent())); + } else if ("exception".equals(tagName) || "throws".equals(tagName)) { + if (tag.getName().isPresent() == false) { + throw new IllegalStateException("Missing tag " + tag.toText()); + } + parsed.thrws.add(List.of(tag.getName().get(), cleaned(stripInlineTags(tag.getContent())))); + } + } + return parsed; + } + + private static String cleaned(List segments) { + return Jsoup.clean(String.join("", segments), Whitelist.none()).replaceAll("[\n\\s]*\n[\n\\s]*", " "); + } + + private static List stripInlineTags(JavadocDescription description) { + List elements = description.getElements(); + List stripped = new ArrayList<>(elements.size()); + for (JavadocDescriptionElement element: elements) { + if (element instanceof JavadocInlineTag) { + stripped.add(((JavadocInlineTag)element).getContent()); + } else { + stripped.add(element.toText()); + } + } + return stripped; + } + + public static class ParsedJavadoc implements ToXContent { + public final Map param = new HashMap<>(); + public String returns; + public String description; + public List> thrws = new ArrayList<>(); + + public static final ParseField PARAMETERS = new ParseField("parameters"); + public static final ParseField RETURN = new ParseField("return"); + public static final ParseField THROWS = new ParseField("throws"); + public static final ParseField DESCRIPTION = new ParseField("description"); + + public ParsedJavadoc(String description) { + this.description = description; + } + + public boolean isEmpty() { + return param.size() == 0 && + (description == null || description.isEmpty()) && + (returns == null || returns.isEmpty()) && + thrws.size() == 0; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + if (description != null && description.isEmpty() == false) { + builder.field(DESCRIPTION.getPreferredName(), description); + } + if (param.isEmpty() == false) { + builder.field(PARAMETERS.getPreferredName(), param); + } + if (returns != null && returns.isEmpty() == false) { + builder.field(RETURN.getPreferredName(), returns); + } + if (thrws.isEmpty() == false) { + builder.field(THROWS.getPreferredName(), thrws); + } + builder.endObject(); + return builder; + } + + @Override + public boolean isFragment() { + return true; + } + } + private static class ClassFileVisitor extends VoidVisitorAdapter { @Override public void visit(CompilationUnit compilationUnit, ParsedJavaClass parsed) { diff --git a/modules/lang-painless/src/doc/java/org/elasticsearch/painless/PainlessInfoJson.java b/modules/lang-painless/src/doc/java/org/elasticsearch/painless/PainlessInfoJson.java index 0838d5dc4b7fe..eb98d256c8de5 100644 --- a/modules/lang-painless/src/doc/java/org/elasticsearch/painless/PainlessInfoJson.java +++ b/modules/lang-painless/src/doc/java/org/elasticsearch/painless/PainlessInfoJson.java @@ -173,13 +173,20 @@ public static class Method implements ToXContentObject { private final String declaring; private final String name; private final String rtn; - private final String javadoc; + private final JavadocExtractor.ParsedJavadoc javadoc; private final List parameters; private final List parameterNames; public static final ParseField PARAMETER_NAMES = new ParseField("parameter_names"); public static final ParseField JAVADOC = new ParseField("javadoc"); - private Method(String declaring, String name, String rtn, String javadoc, List parameters, List parameterNames) { + private Method( + String declaring, + String name, + String rtn, + JavadocExtractor.ParsedJavadoc javadoc, + List parameters, + List parameterNames + ) { this.declaring = declaring; this.name = name; this.rtn = rtn; @@ -210,11 +217,11 @@ public static List fromInfos( ) { List methods = new ArrayList<>(infos.size()); for (PainlessContextMethodInfo info: infos) { - String javadoc = null; + JavadocExtractor.ParsedJavadoc javadoc = null; List parameterNames = null; String name = info.getName(); - List parameterTypes = info.getParameters(); + List parameterTypes = toDisplayParameterTypes(info.getParameters(), javaNamesToDisplayNames); JavadocExtractor.ParsedMethod parsedMethod = parsed.getMethod(name, parameterTypes); if (parsedMethod != null) { @@ -227,7 +234,7 @@ public static List fromInfos( name, ContextGeneratorCommon.getType(javaNamesToDisplayNames, info.getRtn()), javadoc, - toDisplayParameterTypes(parameterTypes, javaNamesToDisplayNames), + parameterTypes, parameterNames )); } @@ -240,7 +247,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(PainlessContextMethodInfo.DECLARING.getPreferredName(), declaring); builder.field(PainlessContextMethodInfo.NAME.getPreferredName(), name); builder.field(PainlessContextMethodInfo.RTN.getPreferredName(), rtn); - if (javadoc != null && "".equals(javadoc) == false) { + if (javadoc != null && javadoc.isEmpty() == false) { builder.field(JAVADOC.getPreferredName(), javadoc); } builder.field(PainlessContextMethodInfo.PARAMETERS.getPreferredName(), parameters); @@ -257,12 +264,17 @@ public static class Constructor implements ToXContentObject { private final String declaring; private final List parameters; private final List parameterNames; - private final String javadoc; + private final JavadocExtractor.ParsedJavadoc javadoc; public static final ParseField JAVADOC = new ParseField("javadoc"); public static final ParseField PARAMETER_NAMES = new ParseField("parameter_names"); - private Constructor(String declaring, List parameters, List parameterNames, String javadoc) { + private Constructor( + String declaring, + List parameters, + List parameterNames, + JavadocExtractor.ParsedJavadoc javadoc + ) { this.declaring = declaring; this.parameters = parameters; this.parameterNames = parameterNames; @@ -289,9 +301,9 @@ private static List fromInfos( ) { List constructors = new ArrayList<>(infos.size()); for (PainlessContextConstructorInfo info: infos) { - List parameterTypes = info.getParameters(); + List parameterTypes = toDisplayParameterTypes(info.getParameters(), javaNamesToDisplayNames); List parameterNames = null; - String javadoc = null; + JavadocExtractor.ParsedJavadoc javadoc = null; JavadocExtractor.ParsedMethod parsed = pj.getConstructor(parameterTypes); if (parsed != null) { @@ -301,7 +313,7 @@ private static List fromInfos( constructors.add(new Constructor( javaNamesToDisplayNames.get(info.getDeclaring()), - toDisplayParameterTypes(parameterTypes, javaNamesToDisplayNames), + parameterTypes, parameterNames, javadoc )); @@ -317,7 +329,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (parameterNames != null && parameterNames.size() > 0) { builder.field(PARAMETER_NAMES.getPreferredName(), parameterNames); } - if (javadoc != null && "".equals(javadoc) == false) { + if (javadoc != null && javadoc.isEmpty() == false) { builder.field(JAVADOC.getPreferredName(), javadoc); } builder.endObject(); diff --git a/qa/smoke-test-http/src/test/java/org/elasticsearch/http/ClusterStatsRestCancellationIT.java b/qa/smoke-test-http/src/test/java/org/elasticsearch/http/ClusterStatsRestCancellationIT.java new file mode 100644 index 0000000000000..0faf470658301 --- /dev/null +++ b/qa/smoke-test-http/src/test/java/org/elasticsearch/http/ClusterStatsRestCancellationIT.java @@ -0,0 +1,189 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.http; + +import org.apache.http.client.methods.HttpGet; +import org.elasticsearch.action.admin.cluster.stats.ClusterStatsAction; +import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.client.Cancellable; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.ResponseListener; +import org.elasticsearch.cluster.routing.allocation.DiskThresholdSettings; +import org.elasticsearch.common.lease.Releasable; +import org.elasticsearch.common.lease.Releasables; +import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.CollectionUtils; +import org.elasticsearch.index.IndexService; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.engine.Engine; +import org.elasticsearch.index.engine.EngineConfig; +import org.elasticsearch.index.engine.EngineFactory; +import org.elasticsearch.index.engine.ReadOnlyEngine; +import org.elasticsearch.index.seqno.SeqNoStats; +import org.elasticsearch.index.shard.IndexShard; +import org.elasticsearch.index.shard.IndexShardTestCase; +import org.elasticsearch.index.translog.TranslogStats; +import org.elasticsearch.indices.IndicesService; +import org.elasticsearch.plugins.EnginePlugin; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.tasks.CancellableTask; +import org.elasticsearch.tasks.TaskInfo; +import org.elasticsearch.transport.TransportService; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.CancellationException; +import java.util.concurrent.Semaphore; +import java.util.function.Function; + +import static java.util.Collections.singletonList; +import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.not; + +public class ClusterStatsRestCancellationIT extends HttpSmokeTestCase { + + public static final Setting BLOCK_STATS_SETTING = Setting.boolSetting("index.block_stats", false, Setting.Property.IndexScope); + + @Override + protected Collection> nodePlugins() { + return CollectionUtils.appendToCopy(super.nodePlugins(), ClusterStatsRestCancellationIT.StatsBlockingPlugin.class); + } + + @Override + protected boolean addMockInternalEngine() { + return false; + } + + @Override + protected Settings nodeSettings(int nodeOrdinal) { + return Settings.builder() + .put(super.nodeSettings(nodeOrdinal)) + // disable internal cluster info service to avoid internal cluster stats calls + .put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_THRESHOLD_ENABLED_SETTING.getKey(), false) + .build(); + } + + public void testClusterStateRestCancellation() throws Exception { + + createIndex("test", Settings.builder().put(BLOCK_STATS_SETTING.getKey(), true).build()); + ensureGreen("test"); + + final List statsBlocks = new ArrayList<>(); + for (final IndicesService indicesService : internalCluster().getInstances(IndicesService.class)) { + for (final IndexService indexService : indicesService) { + for (final IndexShard indexShard : indexService) { + final Engine engine = IndexShardTestCase.getEngine(indexShard); + if (engine instanceof StatsBlockingEngine) { + statsBlocks.add(((StatsBlockingEngine) engine).statsBlock); + } + } + } + } + assertThat(statsBlocks, not(empty())); + + final List releasables = new ArrayList<>(); + try { + for (final Semaphore statsBlock : statsBlocks) { + statsBlock.acquire(); + releasables.add(statsBlock::release); + } + + final Request clusterStatsRequest = new Request(HttpGet.METHOD_NAME, "/_cluster/stats"); + + final PlainActionFuture future = new PlainActionFuture<>(); + logger.info("--> sending cluster state request"); + final Cancellable cancellable = getRestClient().performRequestAsync(clusterStatsRequest, new ResponseListener() { + @Override + public void onSuccess(Response response) { + future.onResponse(null); + } + + @Override + public void onFailure(Exception exception) { + future.onFailure(exception); + } + }); + + logger.info("--> waiting for task to start"); + assertBusy(() -> { + final List tasks = client().admin().cluster().prepareListTasks().get().getTasks(); + assertTrue(tasks.toString(), tasks.stream().anyMatch(t -> t.getAction().startsWith(ClusterStatsAction.NAME))); + }); + + logger.info("--> waiting for at least one task to hit a block"); + assertBusy(() -> assertTrue(statsBlocks.stream().anyMatch(Semaphore::hasQueuedThreads))); + + logger.info("--> cancelling cluster stats request"); + cancellable.cancel(); + expectThrows(CancellationException.class, future::actionGet); + + logger.info("--> checking that all cluster stats tasks are marked as cancelled"); + assertBusy(() -> { + boolean foundTask = false; + for (TransportService transportService : internalCluster().getInstances(TransportService.class)) { + for (CancellableTask cancellableTask : transportService.getTaskManager().getCancellableTasks().values()) { + if (cancellableTask.getAction().startsWith(ClusterStatsAction.NAME)) { + foundTask = true; + assertTrue(cancellableTask.isCancelled()); + } + } + } + assertTrue(foundTask); + }); + } finally { + Releasables.close(releasables); + } + + logger.info("--> checking that all cluster stats tasks have finished"); + assertBusy(() -> { + final List tasks = client().admin().cluster().prepareListTasks().get().getTasks(); + assertTrue(tasks.toString(), tasks.stream().noneMatch(t -> t.getAction().startsWith(ClusterStatsAction.NAME))); + }); + } + + public static class StatsBlockingPlugin extends Plugin implements EnginePlugin { + + @Override + public Optional getEngineFactory(IndexSettings indexSettings) { + if (BLOCK_STATS_SETTING.get(indexSettings.getSettings())) { + return Optional.of(StatsBlockingEngine::new); + } + return Optional.empty(); + } + + @Override + public List> getSettings() { + return singletonList(BLOCK_STATS_SETTING); + } + } + + private static class StatsBlockingEngine extends ReadOnlyEngine { + + final Semaphore statsBlock = new Semaphore(1); + + StatsBlockingEngine(EngineConfig config) { + super(config, null, new TranslogStats(), true, Function.identity(), true); + } + + @Override + public SeqNoStats getSeqNoStats(long globalCheckpoint) { + try { + statsBlock.acquire(); + } catch (InterruptedException e) { + throw new AssertionError(e); + } + statsBlock.release(); + return super.getSeqNoStats(globalCheckpoint); + } + } +} diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/ConcurrentSnapshotsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/ConcurrentSnapshotsIT.java index 179d303a47520..8aab2b11c3adf 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/ConcurrentSnapshotsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/ConcurrentSnapshotsIT.java @@ -142,7 +142,7 @@ public void testDeletesAreBatched() throws Exception { final PlainActionFuture> allDeletesDone = new PlainActionFuture<>(); final ActionListener deletesListener = new GroupedActionListener<>(allDeletesDone, deleteFutures.size()); for (StepListener deleteFuture : deleteFutures) { - deleteFuture.whenComplete(deletesListener::onResponse, deletesListener::onFailure); + deleteFuture.addListener(deletesListener); } allDeletesDone.get(); diff --git a/server/src/main/java/org/elasticsearch/Version.java b/server/src/main/java/org/elasticsearch/Version.java index 792fa066ac161..447cc954ae6a3 100644 --- a/server/src/main/java/org/elasticsearch/Version.java +++ b/server/src/main/java/org/elasticsearch/Version.java @@ -12,6 +12,7 @@ import org.elasticsearch.common.SuppressForbidden; import org.elasticsearch.common.collect.ImmutableOpenIntMap; import org.elasticsearch.common.collect.ImmutableOpenMap; +import org.elasticsearch.common.compatibility.RestApiCompatibleVersion; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.ToXContentFragment; @@ -74,8 +75,8 @@ public class Version implements Comparable, ToXContentFragment { public static final Version V_7_10_0 = new Version(7100099, org.apache.lucene.util.Version.LUCENE_8_7_0); public static final Version V_7_10_1 = new Version(7100199, org.apache.lucene.util.Version.LUCENE_8_7_0); public static final Version V_7_10_2 = new Version(7100299, org.apache.lucene.util.Version.LUCENE_8_7_0); - public static final Version V_7_10_3 = new Version(7100399, org.apache.lucene.util.Version.LUCENE_8_7_0); public static final Version V_7_11_0 = new Version(7110099, org.apache.lucene.util.Version.LUCENE_8_7_0); + public static final Version V_7_11_1 = new Version(7110199, org.apache.lucene.util.Version.LUCENE_8_7_0); public static final Version V_7_12_0 = new Version(7120099, org.apache.lucene.util.Version.LUCENE_8_8_0); public static final Version V_8_0_0 = new Version(8000099, org.apache.lucene.util.Version.LUCENE_8_8_0); public static final Version CURRENT = V_8_0_0; @@ -117,7 +118,9 @@ public class Version implements Comparable, ToXContentFragment { } assert CURRENT.luceneVersion.equals(org.apache.lucene.util.Version.LATEST) : "Version must be upgraded to [" + org.apache.lucene.util.Version.LATEST + "] is still set to [" + CURRENT.luceneVersion + "]"; - + assert RestApiCompatibleVersion.currentVersion().major == CURRENT.major : "RestApiCompatibleVersion must be upgraded " + + "to reflect major from Version.CURRENT [" + CURRENT.major + "]" + + " but is still set to [" + RestApiCompatibleVersion.currentVersion().major + "]"; builder.put(V_EMPTY_ID, V_EMPTY); builderByString.put(V_EMPTY.toString(), V_EMPTY); idToVersion = builder.build(); @@ -364,12 +367,6 @@ public boolean isCompatible(Version version) { return compatible; } - /** - * Returns the minimum version that can be used for compatible REST API - */ - public Version minimumRestCompatibilityVersion() { - return this.previousMajor(); - } /** * Returns a first major version previous to the version stored in this object. diff --git a/server/src/main/java/org/elasticsearch/action/StepListener.java b/server/src/main/java/org/elasticsearch/action/StepListener.java index e6e56e610d761..e3328ef073373 100644 --- a/server/src/main/java/org/elasticsearch/action/StepListener.java +++ b/server/src/main/java/org/elasticsearch/action/StepListener.java @@ -66,7 +66,7 @@ protected void innerOnFailure(Exception e) { * @param onFailure is called when this step is completed with a failure */ public void whenComplete(CheckedConsumer onResponse, Consumer onFailure) { - delegate.addListener(ActionListener.wrap(onResponse, onFailure), EsExecutors.newDirectExecutorService(), null); + addListener(ActionListener.wrap(onResponse, onFailure)); } /** @@ -100,4 +100,12 @@ public Response result() { } return FutureUtils.get(delegate, 0L, TimeUnit.NANOSECONDS); // this future is done already - use a non-blocking method. } + + /** + * Registers the given listener to be notified with the result of this step. + */ + public void addListener(ActionListener listener) { + delegate.addListener(listener, EsExecutors.newDirectExecutorService()); + } + } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/AnalysisStats.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/AnalysisStats.java index ba5d3bc3261f3..94cee3f6f475a 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/AnalysisStats.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/AnalysisStats.java @@ -41,7 +41,7 @@ public final class AnalysisStats implements ToXContentFragment, Writeable { /** * Create {@link AnalysisStats} from the given cluster state. */ - public static AnalysisStats of(Metadata metadata) { + public static AnalysisStats of(Metadata metadata, Runnable ensureNotCancelled) { final Map usedCharFilterTypes = new HashMap<>(); final Map usedTokenizerTypes = new HashMap<>(); final Map usedTokenFilterTypes = new HashMap<>(); @@ -52,6 +52,7 @@ public static AnalysisStats of(Metadata metadata) { final Map usedBuiltInAnalyzers = new HashMap<>(); for (IndexMetadata indexMetadata : metadata) { + ensureNotCancelled.run(); if (indexMetadata.isSystem()) { // Don't include system indices in statistics about analysis, // we care about the user's indices. diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsRequest.java index cbe6539fff186..ca2ec4e5607e3 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsRequest.java @@ -11,8 +11,12 @@ import org.elasticsearch.action.support.nodes.BaseNodesRequest; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.tasks.CancellableTask; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.tasks.TaskId; import java.io.IOException; +import java.util.Map; /** * A request to get cluster level stats. @@ -31,6 +35,11 @@ public ClusterStatsRequest(String... nodesIds) { super(nodesIds); } + @Override + public Task createTask(long id, String type, String action, TaskId parentTaskId, Map headers) { + return new CancellableTask(id, type, action, "", parentTaskId, headers); + } + @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/MappingStats.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/MappingStats.java index 4faecdd537f6b..0f9d5d85946a2 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/MappingStats.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/MappingStats.java @@ -38,9 +38,10 @@ public final class MappingStats implements ToXContentFragment, Writeable { /** * Create {@link MappingStats} from the given cluster state. */ - public static MappingStats of(Metadata metadata) { + public static MappingStats of(Metadata metadata, Runnable ensureNotCancelled) { Map fieldTypes = new HashMap<>(); for (IndexMetadata indexMetadata : metadata) { + ensureNotCancelled.run(); if (indexMetadata.isSystem()) { // Don't include system indices in statistics about mappings, // we care about the user's indices. diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java index 6760e0219bc66..21d4747fa9f4e 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java @@ -8,10 +8,10 @@ package org.elasticsearch.action.admin.cluster.stats; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import org.apache.lucene.store.AlreadyClosedException; +import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.FailedNodeException; +import org.elasticsearch.action.StepListener; import org.elasticsearch.action.admin.cluster.node.info.NodeInfo; import org.elasticsearch.action.admin.cluster.node.stats.NodeStats; import org.elasticsearch.action.admin.indices.stats.CommonStats; @@ -27,6 +27,7 @@ import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.util.CancellableSingleObjectCache; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.engine.CommitStats; import org.elasticsearch.index.seqno.RetentionLeaseStats; @@ -34,7 +35,10 @@ import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.node.NodeService; +import org.elasticsearch.tasks.CancellableTask; import org.elasticsearch.tasks.Task; +import org.elasticsearch.tasks.TaskCancelledException; +import org.elasticsearch.tasks.TaskId; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportRequest; import org.elasticsearch.transport.TransportService; @@ -43,12 +47,12 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.Map; +import java.util.function.BiFunction; public class TransportClusterStatsAction extends TransportNodesAction { - private static final Logger logger = LogManager.getLogger(TransportClusterStatsAction.class); - private static final CommonStatsFlags SHARD_STATS_FLAGS = new CommonStatsFlags(CommonStatsFlags.Flag.Docs, CommonStatsFlags.Flag.Store, CommonStatsFlags.Flag.FieldData, CommonStatsFlags.Flag.QueryCache, CommonStatsFlags.Flag.Completion, CommonStatsFlags.Flag.Segments); @@ -56,12 +60,8 @@ public class TransportClusterStatsAction extends TransportNodesAction mappingStatsCache = new MetadataStatsCache<>(MappingStats::of); + private final MetadataStatsCache analysisStatsCache = new MetadataStatsCache<>(AnalysisStats::of); @Inject public TransportClusterStatsAction(ThreadPool threadPool, ClusterService clusterService, TransportService transportService, @@ -73,49 +73,45 @@ public TransportClusterStatsAction(ThreadPool threadPool, ClusterService cluster } @Override - protected ClusterStatsResponse newResponse(ClusterStatsRequest request, - List responses, List failures) { - assert Transports.assertNotTransportThread("Constructor of ClusterStatsResponse runs expensive computations on mappings found in" + - " the cluster state that are too slow for a transport thread"); + protected void newResponseAsync( + final Task task, + final ClusterStatsRequest request, + final List responses, + final List failures, + final ActionListener listener) { + assert Transports.assertNotTransportThread("Computation of mapping/analysis stats runs expensive computations on mappings found in " + + "the cluster state that are too slow for a transport thread"); + assert Thread.currentThread().getName().contains("[" + ThreadPool.Names.MANAGEMENT + "]") : Thread.currentThread().getName(); + assert task instanceof CancellableTask; + final CancellableTask cancellableTask = (CancellableTask) task; final ClusterState state = clusterService.state(); final Metadata metadata = state.metadata(); - MappingStats currentMappingStats = null; - AnalysisStats currentAnalysisStats = null; - // check if we already served a stats request for the current metadata version and have the stats cached - synchronized (statsMutex) { - if (metadata.version() == metaVersion) { - logger.trace("Found cached mapping and analysis stats for metadata version [{}]", metadata.version()); - currentMappingStats = this.mappingStats; - currentAnalysisStats = this.analysisStats; - } - } - if (currentMappingStats == null) { - // we didn't find any cached stats so we recompute them outside the mutex since the computation might be expensive for larger - // cluster states - logger.trace("Computing mapping and analysis stats for metadata version [{}]", metadata.version()); - currentMappingStats = MappingStats.of(metadata); - currentAnalysisStats = AnalysisStats.of(metadata); - synchronized (statsMutex) { - // cache the computed stats unless they became outdated because of a concurrent cluster state update and a concurrent - // stats request has already cached a newer version - if (metadata.version() > metaVersion) { - logger.trace("Caching mapping and analysis stats for metadata version [{}]", metadata.version()); - metaVersion = metadata.version(); - this.mappingStats = currentMappingStats; - this.analysisStats = currentAnalysisStats; - } - } - } - VersionStats versionStats = VersionStats.of(metadata, responses); - return new ClusterStatsResponse( - System.currentTimeMillis(), - state.metadata().clusterUUID(), - clusterService.getClusterName(), - responses, - failures, - currentMappingStats, - currentAnalysisStats, - versionStats); + + final StepListener mappingStatsStep = new StepListener<>(); + final StepListener analysisStatsStep = new StepListener<>(); + mappingStatsCache.get(metadata, cancellableTask::isCancelled, mappingStatsStep); + analysisStatsCache.get(metadata, cancellableTask::isCancelled, analysisStatsStep); + mappingStatsStep.whenComplete(mappingStats -> analysisStatsStep.whenComplete(analysisStats -> ActionListener.completeWith( + listener, + () -> new ClusterStatsResponse( + System.currentTimeMillis(), + metadata.clusterUUID(), + clusterService.getClusterName(), + responses, + failures, + mappingStats, + analysisStats, + VersionStats.of(metadata, responses)) + ), listener::onFailure), listener::onFailure); + } + + @Override + protected ClusterStatsResponse newResponse( + ClusterStatsRequest request, + List responses, + List failures) { + assert false; + throw new UnsupportedOperationException("use newResponseAsync instead"); } @Override @@ -130,12 +126,17 @@ protected ClusterStatsNodeResponse newNodeResponse(StreamInput in) throws IOExce @Override protected ClusterStatsNodeResponse nodeOperation(ClusterStatsNodeRequest nodeRequest, Task task) { + assert task instanceof CancellableTask; + final CancellableTask cancellableTask = (CancellableTask) task; NodeInfo nodeInfo = nodeService.info(true, true, false, true, false, true, false, true, false, false, false); NodeStats nodeStats = nodeService.stats(CommonStatsFlags.NONE, true, true, true, false, true, false, false, false, false, false, true, false, false, false); List shardsStats = new ArrayList<>(); for (IndexService indexService : indicesService) { for (IndexShard indexShard : indexService) { + if (cancellableTask.isCancelled()) { + throw new TaskCancelledException("task cancelled"); + } if (indexShard.routingEntry() != null && indexShard.routingEntry().active()) { // only report on fully started shards CommitStats commitStats; @@ -186,10 +187,38 @@ public ClusterStatsNodeRequest(StreamInput in) throws IOException { this.request = request; } + @Override + public Task createTask(long id, String type, String action, TaskId parentTaskId, Map headers) { + return new CancellableTask(id, type, action, "", parentTaskId, headers); + } + @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); request.writeTo(out); } } + + private static class MetadataStatsCache extends CancellableSingleObjectCache { + private final BiFunction function; + + MetadataStatsCache(BiFunction function) { + this.function = function; + } + + @Override + protected void refresh(Metadata metadata, Runnable ensureNotCancelled, ActionListener listener) { + ActionListener.completeWith(listener, () -> function.apply(metadata, ensureNotCancelled)); + } + + @Override + protected Long getKey(Metadata indexMetadata) { + return indexMetadata.version(); + } + + @Override + protected boolean isFresh(Long currentKey, Long newKey) { + return newKey <= currentKey; + } + } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeRequest.java index 2b6de1bb47900..46e3a5fdc3e60 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeRequest.java @@ -226,6 +226,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } } builder.endObject(); + if (maxSinglePrimarySize != null) { + builder.field(MAX_SINGLE_PRIMARY_SIZE.getPreferredName(), maxSinglePrimarySize); + } } builder.endObject(); return builder; diff --git a/server/src/main/java/org/elasticsearch/action/support/ListenableActionFuture.java b/server/src/main/java/org/elasticsearch/action/support/ListenableActionFuture.java index 36b56fd7cb1a1..553b7b988ff7e 100644 --- a/server/src/main/java/org/elasticsearch/action/support/ListenableActionFuture.java +++ b/server/src/main/java/org/elasticsearch/action/support/ListenableActionFuture.java @@ -23,7 +23,7 @@ public class ListenableActionFuture extends AdapterActionFuture { private boolean executedListeners = false; /** - * Registers an {@link ActionListener} to be notified when this future is completed. If the future is already completed then the + * Registers an {@link ActionListener} to be notified when this future is completed. If the future is already completed then the * listener is notified immediately, on the calling thread. If not, the listener is notified on the thread that completes the listener. */ @SuppressWarnings("unchecked,rawtypes") diff --git a/server/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java b/server/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java index 95422969710df..7f83785178331 100644 --- a/server/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/nodes/TransportNodesAction.java @@ -10,7 +10,6 @@ import org.apache.logging.log4j.message.ParameterizedMessage; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.ActionRunnable; import org.elasticsearch.action.FailedNodeException; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.HandledTransportAction; @@ -101,15 +100,22 @@ protected void doExecute(Task task, NodesRequest request, ActionListener nodesResponses) { + // exposed for tests + void newResponse(Task task, NodesRequest request, AtomicReferenceArray nodesResponses, ActionListener listener) { + + if (nodesResponses == null) { + listener.onFailure(new NullPointerException("nodesResponses")); + return; + } + final List responses = new ArrayList<>(); final List failures = new ArrayList<>(); @@ -123,7 +129,7 @@ protected NodesResponse newResponse(NodesRequest request, AtomicReferenceArray responses, List failures); + /** + * Create a new {@link NodesResponse}, possibly asynchronously. The default implementation is synchronous and calls + * {@link #newResponse(BaseNodesRequest, List, List)} + */ + protected void newResponseAsync( + Task task, + NodesRequest request, + List responses, + List failures, + ActionListener listener) { + ActionListener.completeWith(listener, () -> newResponse(request, responses, failures)); + } + protected abstract NodeRequest newNodeRequest(NodesRequest request); protected abstract NodeResponse newNodeResponse(StreamInput in) throws IOException; @@ -181,8 +200,9 @@ class AsyncAction { void start() { final DiscoveryNode[] nodes = request.concreteNodes(); if (nodes.length == 0) { - // nothing to notify - threadPool.generic().execute(() -> listener.onResponse(newResponse(request, responses))); + // nothing to notify, so respond immediately, but always fork even if finalExecutor == SAME + final String executor = finalExecutor.equals(ThreadPool.Names.SAME) ? ThreadPool.Names.GENERIC : finalExecutor; + threadPool.executor(executor).execute(() -> newResponse(task, request, responses, listener)); return; } final TransportRequestOptions transportRequestOptions = TransportRequestOptions.timeout(request.timeout()); @@ -237,7 +257,7 @@ private void onFailure(int idx, String nodeId, Throwable t) { } private void finishHim() { - threadPool.executor(finalExecutor).execute(ActionRunnable.supply(listener, () -> newResponse(request, responses))); + threadPool.executor(finalExecutor).execute(() -> newResponse(task, request, responses, listener)); } } diff --git a/server/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java b/server/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java index a7a12f880b49a..3e27ce8460907 100644 --- a/server/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java +++ b/server/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java @@ -40,6 +40,7 @@ import org.elasticsearch.node.InternalSettingsPreparer; import org.elasticsearch.node.Node; import org.elasticsearch.node.NodeValidationException; +import org.elasticsearch.snapshots.SnapshotsService; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -168,6 +169,18 @@ private void setup(boolean addShutdownHook, Environment environment) throws Boot BootstrapSettings.SYSTEM_CALL_FILTER_SETTING.get(settings), BootstrapSettings.CTRLHANDLER_SETTING.get(settings)); + final long cacheSize = SnapshotsService.SNAPSHOT_CACHE_SIZE_SETTING.get(settings).getBytes(); + final long regionSize = SnapshotsService.SNAPSHOT_CACHE_REGION_SIZE_SETTING.get(settings).getBytes(); + final int numRegions = Math.toIntExact(cacheSize / regionSize); + final long fileSize = numRegions * regionSize; + if (fileSize > 0) { + try { + Natives.tryCreateCacheFile(environment, fileSize); + } catch (Exception e) { + throw new BootstrapException(e); + } + } + // initialize probes before the security manager is installed initializeProbes(); diff --git a/server/src/main/java/org/elasticsearch/bootstrap/JNACLibrary.java b/server/src/main/java/org/elasticsearch/bootstrap/JNACLibrary.java index 52dcd5438ed58..1b8ca51815aa2 100644 --- a/server/src/main/java/org/elasticsearch/bootstrap/JNACLibrary.java +++ b/server/src/main/java/org/elasticsearch/bootstrap/JNACLibrary.java @@ -61,6 +61,9 @@ protected List getFieldOrder() { static native String strerror(int errno); + // TODO: Bind POSIX fallocate as well to support non-Linux? (this would only apply to OSX in practice?) + static native int fallocate(int fd, int mode, long offset, long length); + private JNACLibrary() { } } diff --git a/server/src/main/java/org/elasticsearch/bootstrap/JNANatives.java b/server/src/main/java/org/elasticsearch/bootstrap/JNANatives.java index 90d576b29fdd8..ed402e642a98f 100644 --- a/server/src/main/java/org/elasticsearch/bootstrap/JNANatives.java +++ b/server/src/main/java/org/elasticsearch/bootstrap/JNANatives.java @@ -14,9 +14,17 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.message.ParameterizedMessage; import org.apache.lucene.util.Constants; +import org.elasticsearch.common.SuppressForbidden; +import org.elasticsearch.env.Environment; import org.elasticsearch.monitor.jvm.JvmInfo; +import org.elasticsearch.snapshots.SnapshotUtils; +import java.io.FileOutputStream; +import java.io.IOException; +import java.lang.reflect.Field; +import java.nio.file.Files; import java.nio.file.Path; import static org.elasticsearch.bootstrap.JNAKernel32Library.SizeT; @@ -260,4 +268,39 @@ static void tryInstallSystemCallFilter(Path tmpFile) { logger.warn("unable to install syscall filter: ", e); } } + + @SuppressForbidden(reason = "need access to fd on FileOutputStream") + static void fallocateSnapshotCacheFile(Environment environment, long fileSize) throws IOException { + if (Constants.LINUX == false) { + logger.debug("not trying to create a shared cache file using fallocate on non-Linux platform"); + return; + } + Path cacheFile = SnapshotUtils.findCacheSnapshotCacheFilePath(environment, fileSize); + if (cacheFile == null) { + throw new IOException("could not find a directory with adequate free space for cache file"); + } + boolean success = false; + try (FileOutputStream fileChannel = new FileOutputStream(cacheFile.toFile())) { + long currentSize = fileChannel.getChannel().size(); + if (currentSize < fileSize) { + final Field field = fileChannel.getFD().getClass().getDeclaredField("fd"); + field.setAccessible(true); + final int result = JNACLibrary.fallocate((int) field.get(fileChannel.getFD()), 0, currentSize, fileSize - currentSize); + final int errno = result == 0 ? 0 : Native.getLastError(); + if (errno == 0) { + success = true; + logger.info("allocated cache file [{}] using fallocate", cacheFile); + } else { + logger.warn("failed to initialize cache file [{}] using fallocate errno [{}]", cacheFile, errno); + } + } + } catch (Exception e) { + logger.warn(new ParameterizedMessage("failed to initialize cache file [{}] using fallocate", cacheFile), e); + } finally { + if (success == false) { + // if anything goes wrong, delete the potentially created file to not waste disk space + Files.deleteIfExists(cacheFile); + } + } + } } diff --git a/server/src/main/java/org/elasticsearch/bootstrap/Natives.java b/server/src/main/java/org/elasticsearch/bootstrap/Natives.java index b9f072a52e817..e91116a5508ec 100644 --- a/server/src/main/java/org/elasticsearch/bootstrap/Natives.java +++ b/server/src/main/java/org/elasticsearch/bootstrap/Natives.java @@ -10,7 +10,9 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.elasticsearch.env.Environment; +import java.io.IOException; import java.nio.file.Path; /** @@ -132,4 +134,20 @@ static boolean isSystemCallFilterInstalled() { } return JNANatives.LOCAL_SYSTEM_CALL_FILTER; } + + /** + * On Linux, this method tries to create the searchable snapshot frozen cache file using fallocate if JNA is available. This enables + * a much faster creation of the file than the fallback mechanism in the searchable snapshots plugin that will pre-allocate the cache + * file by writing zeros to the file. + * + * @throws IOException on failure to determine free disk space for a data path + */ + public static void tryCreateCacheFile(Environment environment, long fileSize) throws IOException { + if (JNA_AVAILABLE == false) { + logger.warn("cannot use fallocate to create cache file because JNA is not available"); + return; + } + JNANatives.fallocateSnapshotCacheFile(environment, fileSize); + } + } diff --git a/server/src/main/java/org/elasticsearch/common/util/CancellableSingleObjectCache.java b/server/src/main/java/org/elasticsearch/common/util/CancellableSingleObjectCache.java new file mode 100644 index 0000000000000..e994f3e531bd0 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/common/util/CancellableSingleObjectCache.java @@ -0,0 +1,283 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.common.util; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.ListenableActionFuture; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.util.concurrent.AbstractRefCounted; +import org.elasticsearch.tasks.TaskCancelledException; + +import java.util.ArrayList; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.BooleanSupplier; + +/** + * A cache of a single object whose refresh process can be cancelled. The cached value is computed lazily on the first retrieval, and + * associated with a key which is used to determine its freshness for subsequent retrievals. + *

+ * This is useful for things like computing stats over cluster metadata: the first time stats are requested they are computed, but + * subsequent calls re-use the computed value as long as they pertain to the same metadata version. If stats are requested for a different + * metadata version then the cached value is dropped and a new one is computed. + *

+ * Retrievals happen via the async {@link #get} method. If a retrieval is cancelled (e.g. the channel on which to return the stats is + * closed) then the computation carries on running in case another retrieval for the same key arrives in future. However if all of the + * retrievals for a key are cancelled and a retrieval occurs for a fresher key then the computation itself is cancelled. + *

+ * Cancellation is based on polling: the {@link #refresh} method checks whether it should abort whenever it is convenient to do so, which in + * turn checks all the pending retrievals to see whether they have been cancelled. + * + * @param The type of the input to the computation of the cached value. + * @param The key type. The cached value is associated with a key, and subsequent {@link #get} calls compare keys of the given input + * value to determine whether the cached value is fresh or not. See {@link #isFresh}. + * @param The type of the cached value. + */ +public abstract class CancellableSingleObjectCache { + + private final AtomicReference currentCachedItemRef = new AtomicReference<>(); + + /** + * Compute a new value for the cache. + *

+ * If an exception is thrown, or passed to the {@code listener}, then it is passed on to all waiting listeners but it is not cached so + * that subsequent retrievals will trigger subsequent calls to this method. + *

+ * Implementations of this method should poll for cancellation by running {@code ensureNotCancelled} whenever appropriate. The + * computation is cancelled if all of the corresponding retrievals have been cancelled and a retrieval has since happened for a + * fresher key. + * + * @param input The input to this computation, which will be converted to a key and used to determine whether it is + * suitably fresh for future requests too. + * @param ensureNotCancelled A {@link Runnable} which throws a {@link TaskCancelledException} if the result of the computation is no + * longer needed. On cancellation, notifying the {@code listener} is optional. + * @param listener A {@link ActionListener} which should be notified when the computation completes. If the computation fails + * by calling {@link ActionListener#onFailure} then the result is returned to the pending listeners but is not + * cached. + */ + protected abstract void refresh(Input input, Runnable ensureNotCancelled, ActionListener listener); + + /** + * Compute the key for the given input value. + */ + protected abstract Key getKey(Input input); + + /** + * Compute whether the {@code currentKey} is fresh enough for a retrieval associated with {@code newKey}. + * + * @param currentKey The key of the current (cached or pending) value. + * @param newKey The key associated with a new retrival. + * @return {@code true} if a value computed for {@code currentKey} is fresh enough to satisfy a retrieval for {@code newKey}. + */ + protected boolean isFresh(Key currentKey, Key newKey) { + return currentKey.equals(newKey); + } + + /** + * Start a retrieval for the value associated with the given {@code input}, and pass it to the given {@code listener}. + *

+ * If a fresh-enough result is available when this method is called then the {@code listener} is notified immediately, on this thread. + * If a fresh-enough result is already being computed then the {@code listener} is captured and will be notified when the result becomes + * available, on the thread on which the refresh completes. If no fresh-enough result is either pending or available then this method + * starts to compute one by calling {@link #refresh} on this thread. + * + * @param input The input to compute the desired value, converted to a {@link Key} to determine if the value that's currently + * cached or pending is fresh enough. + * @param isCancelled Returns {@code true} if the listener no longer requires the value being computed. + * @param listener The listener to notify when the desired value becomes available. + */ + public final void get(Input input, BooleanSupplier isCancelled, ActionListener listener) { + + final Key key = getKey(input); + + CachedItem newCachedItem = null; + + do { + if (isCancelled.getAsBoolean()) { + listener.onFailure(new TaskCancelledException("task cancelled")); + return; + } + + final CachedItem currentCachedItem = currentCachedItemRef.get(); + if (currentCachedItem != null && isFresh(currentCachedItem.getKey(), key)) { + final boolean listenerAdded = currentCachedItem.addListener(listener, isCancelled); + if (listenerAdded) { + return; + } + + assert currentCachedItem.refCount() == 0 : currentCachedItem.refCount(); + assert currentCachedItemRef.get() != currentCachedItem; + + // Our item was only just released, possibly cancelled, by another get() with a fresher key. We don't simply retry + // since that would evict the new item. Instead let's see if it was cancelled or whether it completed properly. + if (currentCachedItem.getFuture().isDone()) { + try { + listener.onResponse(currentCachedItem.getFuture().actionGet(0L)); + return; + } catch (TaskCancelledException e) { + // previous task was cancelled before completion, therefore we must perform our own one-shot refresh + } catch (Exception e) { + // either the refresh completed exceptionally or the listener threw an exception; call onFailure() either way + listener.onFailure(e); + return; + } + } // else it's just about to be cancelled, so we can just retry knowing that it will be removed very soon + + continue; + } + + if (newCachedItem == null) { + newCachedItem = new CachedItem(key); + } + + if (currentCachedItemRef.compareAndSet(currentCachedItem, newCachedItem)) { + if (currentCachedItem != null) { + currentCachedItem.decRef(); + } + startRefresh(input, newCachedItem); + final boolean listenerAdded = newCachedItem.addListener(listener, isCancelled); + assert listenerAdded; + newCachedItem.decRef(); + return; + } + // else the CAS failed because we lost a race to a concurrent retrieval; try again from the top since we expect the race winner + // to be fresh enough for us and therefore we can just wait for its result. + } while (true); + } + + private void startRefresh(Input input, CachedItem cachedItem) { + try { + refresh(input, cachedItem::ensureNotCancelled, cachedItem.getFuture()); + } catch (Exception e) { + cachedItem.getFuture().onFailure(e); + } + } + + /** + * An item in the cache, representing a single invocation of {@link #refresh}. + *

+ * This item is ref-counted so that it can be cancelled if it becomes irrelevant. References are held by: + *

    + *
  • Every listener that is waiting for the result, released on cancellation. There's no need to release on completion because + * there's nothing to cancel once the refresh has completed.
    • + *
  • The cache itself, released once this item is no longer the current one in the cache, either because it failed or because a + * fresher computation was started.
    • + *
  • The process that adds the first listener, released once the first listener is added.
    • + *
+ */ + private final class CachedItem extends AbstractRefCounted { + + private final Key key; + private final ListenableActionFuture future = new ListenableActionFuture<>(); + private final CancellationChecks cancellationChecks = new CancellationChecks(); + + CachedItem(Key key) { + super("cached item"); + this.key = key; + incRef(); // start with a refcount of 2 so we're not closed while adding the first listener + this.future.addListener(new ActionListener<>() { + @Override + public void onResponse(Value value) { + cancellationChecks.clear(); + } + + @Override + public void onFailure(Exception e) { + cancellationChecks.clear(); + // Do not cache this failure + if (currentCachedItemRef.compareAndSet(CachedItem.this, null)) { + // Release reference held by the cache, so that concurrent calls to addListener() fail and retry. Not totally + // necessary, we could also fail those listeners as if they'd been added slightly sooner, but it makes the ref + // counting easier to document. + decRef(); + } + } + }); + } + + Key getKey() { + return key; + } + + ListenableActionFuture getFuture() { + return future; + } + + boolean addListener(ActionListener listener, BooleanSupplier isCancelled) { + if (tryIncRef()) { + if (future.isDone()) { + // No need to bother with ref counting & cancellation any more, just complete the listener. + // We know it wasn't cancelled because there are still references. + ActionListener.completeWith(listener, () -> future.actionGet(0L)); + } else { + // Refresh is still pending; it's not cancelled because there are still references. + future.addListener(listener); + final AtomicBoolean released = new AtomicBoolean(); + cancellationChecks.add(() -> { + if (released.get() == false && isCancelled.getAsBoolean() && released.compareAndSet(false, true)) { + decRef(); + } + }); + } + return true; + } else { + return false; + } + } + + void ensureNotCancelled() { + cancellationChecks.runAll(); + if (refCount() == 0) { + throw new TaskCancelledException("task cancelled"); + } + } + + @Override + protected void closeInternal() { + // Complete the future (and hence all its listeners) with an exception if it hasn't already been completed. + future.onFailure(new TaskCancelledException("task cancelled")); + } + } + + private static final class CancellationChecks { + @Nullable // if cleared + private ArrayList checks = new ArrayList<>(); + + synchronized void clear() { + checks = null; + } + + synchronized void add(Runnable check) { + if (checks != null) { + checks.add(check); + } + } + + void runAll() { + // It's ok not to run all the checks so there's no need for a completely synchronized iteration. + final int count; + synchronized (this) { + if (checks == null) { + return; + } + count = checks.size(); + } + for (int i = 0; i < count; i++) { + final Runnable cancellationCheck; + synchronized (this) { + if (checks == null) { + return; + } + cancellationCheck = checks.get(i); + } + cancellationCheck.run(); + } + } + } +} diff --git a/server/src/main/java/org/elasticsearch/env/Environment.java b/server/src/main/java/org/elasticsearch/env/Environment.java index 18b509c6f58bf..8bf2a5fbbd241 100644 --- a/server/src/main/java/org/elasticsearch/env/Environment.java +++ b/server/src/main/java/org/elasticsearch/env/Environment.java @@ -300,6 +300,16 @@ public static FileStore getFileStore(final Path path) throws IOException { return new ESFileStore(Files.getFileStore(path)); } + public static long getUsableSpace(Path path) throws IOException { + long freeSpaceInBytes = Environment.getFileStore(path).getUsableSpace(); + + /* See: https://bugs.openjdk.java.net/browse/JDK-8162520 */ + if (freeSpaceInBytes < 0) { + freeSpaceInBytes = Long.MAX_VALUE; + } + return freeSpaceInBytes; + } + /** * asserts that the two environments are equivalent for all things the environment cares about (i.e., all but the setting * object which may contain different setting) diff --git a/server/src/main/java/org/elasticsearch/index/store/Store.java b/server/src/main/java/org/elasticsearch/index/store/Store.java index c43dd162bebc1..66757375a636b 100644 --- a/server/src/main/java/org/elasticsearch/index/store/Store.java +++ b/server/src/main/java/org/elasticsearch/index/store/Store.java @@ -1360,7 +1360,7 @@ public void markStoreCorrupted(IOException exception) throws IOException { BytesRef ref = bytes.toBytesRef(); output.writeBytes(ref.bytes, ref.offset, ref.length); CodecUtil.writeFooter(output); - } catch (IOException ex) { + } catch (IOException | ImmutableDirectoryException ex) { logger.warn("Can't mark store as corrupted", ex); } directory().sync(Collections.singleton(corruptionMarkerName)); diff --git a/server/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java b/server/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java index 48ddbbce1d999..6bf132a5b0323 100644 --- a/server/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java +++ b/server/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java @@ -583,7 +583,7 @@ void createRetentionLease(final long startingSeqNo, ActionListener(logger, shard.getThreadPool(), ThreadPool.Names.GENERIC, cloneRetentionLeaseStep, false)); logger.trace("cloned primary's retention lease as [{}]", clonedLease); - cloneRetentionLeaseStep.whenComplete(rr -> listener.onResponse(clonedLease), listener::onFailure); + cloneRetentionLeaseStep.addListener(listener.map(rr -> clonedLease)); } catch (RetentionLeaseNotFoundException e) { // it's possible that the primary has no retention lease yet if we are doing a rolling upgrade from a version before // 7.4, and in that case we just create a lease using the local checkpoint of the safe commit which we're using for @@ -595,7 +595,7 @@ void createRetentionLease(final long startingSeqNo, ActionListener(logger, shard.getThreadPool(), ThreadPool.Names.GENERIC, addRetentionLeaseStep, false)); - addRetentionLeaseStep.whenComplete(rr -> listener.onResponse(newLease), listener::onFailure); + addRetentionLeaseStep.addListener(listener.map(rr -> newLease)); logger.trace("created retention lease with estimated checkpoint of [{}]", estimatedGlobalCheckpoint); } }, shardId + " establishing retention lease for [" + request.targetAllocationId() + "]", diff --git a/server/src/main/java/org/elasticsearch/node/Node.java b/server/src/main/java/org/elasticsearch/node/Node.java index db2b89104cba3..f0ecdf761a8d6 100644 --- a/server/src/main/java/org/elasticsearch/node/Node.java +++ b/server/src/main/java/org/elasticsearch/node/Node.java @@ -454,8 +454,7 @@ protected Node(final Environment initialEnvironment, .flatMap(p -> p.getNamedXContent().stream()), ClusterModule.getNamedXWriteables().stream()) .flatMap(Function.identity()).collect(toList()), - pluginsService.filterPlugins(Plugin.class).stream() - .flatMap(p -> p.getNamedXContentForCompatibility().stream()).collect(toList()) + getCompatibleNamedXContents() ); final MetaStateService metaStateService = new MetaStateService(nodeEnvironment, xContentRegistry); final PersistedClusterStateService lucenePersistedStateFactory @@ -730,6 +729,12 @@ protected Node(final Environment initialEnvironment, } } + // package scope for testing + List getCompatibleNamedXContents() { + return pluginsService.filterPlugins(Plugin.class).stream() + .flatMap(p -> p.getNamedXContentForCompatibility().stream()).collect(toList()); + } + protected TransportService newTransportService(Settings settings, Transport transport, ThreadPool threadPool, TransportInterceptor interceptor, Function localNodeFactory, diff --git a/server/src/main/java/org/elasticsearch/repositories/RepositoriesService.java b/server/src/main/java/org/elasticsearch/repositories/RepositoriesService.java index 13aea18b8206e..fb1b511fa240c 100644 --- a/server/src/main/java/org/elasticsearch/repositories/RepositoriesService.java +++ b/server/src/main/java/org/elasticsearch/repositories/RepositoriesService.java @@ -158,11 +158,11 @@ public void registerRepository(final PutRepositoryRequest request, final ActionL // Finally respond to the outer listener with the response from the original cluster state update updateRepoUuidStep.whenComplete( - ignored -> acknowledgementStep.whenComplete(listener::onResponse, listener::onFailure), + ignored -> acknowledgementStep.addListener(listener), listener::onFailure); } else { - acknowledgementStep.whenComplete(listener::onResponse, listener::onFailure); + acknowledgementStep.addListener(listener); } clusterService.submitStateUpdateTask("put_repository [" + request.name() + "]", diff --git a/server/src/main/java/org/elasticsearch/rest/MethodHandlers.java b/server/src/main/java/org/elasticsearch/rest/MethodHandlers.java index e893c5fa639c6..aa18c8ce43c1e 100644 --- a/server/src/main/java/org/elasticsearch/rest/MethodHandlers.java +++ b/server/src/main/java/org/elasticsearch/rest/MethodHandlers.java @@ -8,7 +8,7 @@ package org.elasticsearch.rest; -import org.elasticsearch.Version; +import org.elasticsearch.common.compatibility.RestApiCompatibleVersion; import java.util.HashMap; import java.util.Map; @@ -20,7 +20,7 @@ final class MethodHandlers { private final String path; - private final Map> methodHandlers; + private final Map> methodHandlers; MethodHandlers(String path, RestHandler handler, RestRequest.Method... methods) { this.path = path; @@ -54,13 +54,13 @@ MethodHandlers addMethods(RestHandler handler, RestRequest.Method... methods) { * (as opposed to non-compatible/breaking) * or {@code null} if none exists. */ - RestHandler getHandler(RestRequest.Method method, Version version) { - Map versionToHandlers = methodHandlers.get(method); + RestHandler getHandler(RestRequest.Method method, RestApiCompatibleVersion version) { + Map versionToHandlers = methodHandlers.get(method); if (versionToHandlers == null) { return null; //method not found } final RestHandler handler = versionToHandlers.get(version); - return handler == null ? versionToHandlers.get(Version.CURRENT) : handler; + return handler == null ? versionToHandlers.get(RestApiCompatibleVersion.currentVersion()) : handler; } diff --git a/server/src/main/java/org/elasticsearch/rest/RestCompatibleVersionHelper.java b/server/src/main/java/org/elasticsearch/rest/RestCompatibleVersionHelper.java index fe53ae212939f..24f88cc6ff00d 100644 --- a/server/src/main/java/org/elasticsearch/rest/RestCompatibleVersionHelper.java +++ b/server/src/main/java/org/elasticsearch/rest/RestCompatibleVersionHelper.java @@ -15,8 +15,8 @@ package org.elasticsearch.rest; import org.elasticsearch.ElasticsearchStatusException; -import org.elasticsearch.Version; import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.compatibility.RestApiCompatibleVersion; import org.elasticsearch.common.xcontent.MediaType; import org.elasticsearch.common.xcontent.ParsedMediaType; @@ -27,23 +27,25 @@ */ class RestCompatibleVersionHelper { - static Version getCompatibleVersion( + static RestApiCompatibleVersion getCompatibleVersion( @Nullable ParsedMediaType acceptHeader, @Nullable ParsedMediaType contentTypeHeader, boolean hasContent ) { Byte aVersion = parseVersion(acceptHeader); - byte acceptVersion = aVersion == null ? Version.CURRENT.major : Integer.valueOf(aVersion).byteValue(); + byte acceptVersion = aVersion == null ? RestApiCompatibleVersion.currentVersion().major : Integer.valueOf(aVersion).byteValue(); Byte cVersion = parseVersion(contentTypeHeader); - byte contentTypeVersion = cVersion == null ? Version.CURRENT.major : Integer.valueOf(cVersion).byteValue(); + byte contentTypeVersion = cVersion == null ? + RestApiCompatibleVersion.currentVersion().major : Integer.valueOf(cVersion).byteValue(); // accept version must be current or prior - if (acceptVersion > Version.CURRENT.major || acceptVersion < Version.CURRENT.minimumRestCompatibilityVersion().major) { + if (acceptVersion > RestApiCompatibleVersion.currentVersion().major || + acceptVersion < RestApiCompatibleVersion.minimumSupported().major) { throw new ElasticsearchStatusException( "Accept version must be either version {} or {}, but found {}. Accept={}", RestStatus.BAD_REQUEST, - Version.CURRENT.major, - Version.CURRENT.minimumRestCompatibilityVersion().major, + RestApiCompatibleVersion.currentVersion().major, + RestApiCompatibleVersion.minimumSupported().major, acceptVersion, acceptHeader ); @@ -51,13 +53,13 @@ static Version getCompatibleVersion( if (hasContent) { // content-type version must be current or prior - if (contentTypeVersion > Version.CURRENT.major - || contentTypeVersion < Version.CURRENT.minimumRestCompatibilityVersion().major) { + if (contentTypeVersion > RestApiCompatibleVersion.currentVersion().major + || contentTypeVersion < RestApiCompatibleVersion.minimumSupported().major) { throw new ElasticsearchStatusException( "Content-Type version must be either version {} or {}, but found {}. Content-Type={}", RestStatus.BAD_REQUEST, - Version.CURRENT.major, - Version.CURRENT.minimumRestCompatibilityVersion().major, + RestApiCompatibleVersion.currentVersion().major, + RestApiCompatibleVersion.minimumSupported().major, contentTypeVersion, contentTypeHeader ); @@ -83,16 +85,16 @@ static Version getCompatibleVersion( contentTypeHeader ); } - if (contentTypeVersion < Version.CURRENT.major) { - return Version.CURRENT.previousMajor(); + if (contentTypeVersion < RestApiCompatibleVersion.currentVersion().major) { + return RestApiCompatibleVersion.minimumSupported(); } } - if (acceptVersion < Version.CURRENT.major) { - return Version.CURRENT.previousMajor(); + if (acceptVersion < RestApiCompatibleVersion.currentVersion().major) { + return RestApiCompatibleVersion.minimumSupported(); } - return Version.CURRENT; + return RestApiCompatibleVersion.currentVersion(); } static Byte parseVersion(ParsedMediaType parsedMediaType) { diff --git a/server/src/main/java/org/elasticsearch/rest/RestController.java b/server/src/main/java/org/elasticsearch/rest/RestController.java index b10ab89f6ada0..fa2d533d99f06 100644 --- a/server/src/main/java/org/elasticsearch/rest/RestController.java +++ b/server/src/main/java/org/elasticsearch/rest/RestController.java @@ -12,13 +12,13 @@ import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.Version; import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Strings; import org.elasticsearch.common.breaker.CircuitBreaker; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.compatibility.RestApiCompatibleVersion; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.path.PathTrie; @@ -158,9 +158,9 @@ protected void registerHandler(RestRequest.Method method, String path, RestHandl } private void registerHandlerNoWrap(RestRequest.Method method, String path, RestHandler maybeWrappedHandler) { - final Version version = maybeWrappedHandler.compatibleWithVersion(); - assert Version.CURRENT.minimumRestCompatibilityVersion() == version || Version.CURRENT == version - : "REST API compatibility is only supported for version " + Version.CURRENT.minimumRestCompatibilityVersion().major; + final RestApiCompatibleVersion version = maybeWrappedHandler.compatibleWithVersion(); + assert RestApiCompatibleVersion.minimumSupported() == version || RestApiCompatibleVersion.currentVersion() == version + : "REST API compatibility is only supported for version " + RestApiCompatibleVersion.minimumSupported().major; handlers.insertOrUpdate(path, new MethodHandlers(path, maybeWrappedHandler, method), (mHandlers, newMHandler) -> mHandlers.addMethods(maybeWrappedHandler, method)); @@ -214,7 +214,8 @@ public void dispatchBadRequest(final RestChannel channel, final ThreadContext th } } - private void dispatchRequest(RestRequest request, RestChannel channel, RestHandler handler, Version compatibleVersion) + private void dispatchRequest(RestRequest request, RestChannel channel, RestHandler handler, + RestApiCompatibleVersion restApiCompatibleVersion) throws Exception { final int contentLength = request.contentLength(); if (contentLength > 0) { @@ -239,7 +240,7 @@ private void dispatchRequest(RestRequest request, RestChannel channel, RestHandl inFlightRequestsBreaker(circuitBreakerService).addWithoutBreaking(contentLength); } // iff we could reserve bytes for the request we need to send the response also over this channel - responseChannel = new ResourceHandlingHttpChannel(channel, circuitBreakerService, contentLength, compatibleVersion); + responseChannel = new ResourceHandlingHttpChannel(channel, circuitBreakerService, contentLength, restApiCompatibleVersion); // TODO: Count requests double in the circuit breaker if they need copying? if (handler.allowsUnsafeBuffers() == false) { request.ensureSafeBuffers(); @@ -316,7 +317,7 @@ private void tryAllHandlers(final RestRequest request, final RestChannel channel final String uri = request.uri(); final RestRequest.Method requestMethod; - Version compatibleVersion = request.getCompatibleVersion(); + RestApiCompatibleVersion restApiCompatibleVersion = request.getRestApiCompatibleVersion(); try { // Resolves the HTTP method and fails if the method is invalid requestMethod = request.method(); @@ -328,14 +329,14 @@ private void tryAllHandlers(final RestRequest request, final RestChannel channel if (handlers == null) { handler = null; } else { - handler = handlers.getHandler(requestMethod, compatibleVersion); + handler = handlers.getHandler(requestMethod, restApiCompatibleVersion); } if (handler == null) { if (handleNoHandlerFound(rawPath, requestMethod, uri, channel)) { return; } } else { - dispatchRequest(request, channel, handler, compatibleVersion); + dispatchRequest(request, channel, handler, restApiCompatibleVersion); return; } } @@ -453,40 +454,40 @@ private static final class ResourceHandlingHttpChannel implements RestChannel { private final RestChannel delegate; private final CircuitBreakerService circuitBreakerService; private final int contentLength; - private final Version compatibleVersion; + private final RestApiCompatibleVersion restApiCompatibleVersion; private final AtomicBoolean closed = new AtomicBoolean(); ResourceHandlingHttpChannel(RestChannel delegate, CircuitBreakerService circuitBreakerService, int contentLength, - Version compatibleVersion) { + RestApiCompatibleVersion restApiCompatibleVersion) { this.delegate = delegate; this.circuitBreakerService = circuitBreakerService; this.contentLength = contentLength; - this.compatibleVersion = compatibleVersion; + this.restApiCompatibleVersion = restApiCompatibleVersion; } @Override public XContentBuilder newBuilder() throws IOException { return delegate.newBuilder() - .withCompatibleMajorVersion(compatibleVersion.major); + .withCompatibleVersion(restApiCompatibleVersion); } @Override public XContentBuilder newErrorBuilder() throws IOException { return delegate.newErrorBuilder() - .withCompatibleMajorVersion(compatibleVersion.major); + .withCompatibleVersion(restApiCompatibleVersion); } @Override public XContentBuilder newBuilder(@Nullable XContentType xContentType, boolean useFiltering) throws IOException { return delegate.newBuilder(xContentType, useFiltering) - .withCompatibleMajorVersion(compatibleVersion.major); + .withCompatibleVersion(restApiCompatibleVersion); } @Override public XContentBuilder newBuilder(XContentType xContentType, XContentType responseContentType, boolean useFiltering) throws IOException { return delegate.newBuilder(xContentType, responseContentType, useFiltering) - .withCompatibleMajorVersion(compatibleVersion.major); + .withCompatibleVersion(restApiCompatibleVersion); } @Override diff --git a/server/src/main/java/org/elasticsearch/rest/RestHandler.java b/server/src/main/java/org/elasticsearch/rest/RestHandler.java index 88e271b63e919..9b1902e27a85d 100644 --- a/server/src/main/java/org/elasticsearch/rest/RestHandler.java +++ b/server/src/main/java/org/elasticsearch/rest/RestHandler.java @@ -8,8 +8,8 @@ package org.elasticsearch.rest; -import org.elasticsearch.Version; import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.compatibility.RestApiCompatibleVersion; import org.elasticsearch.common.xcontent.MediaType; import org.elasticsearch.common.xcontent.MediaTypeRegistry; import org.elasticsearch.common.xcontent.XContent; @@ -102,8 +102,8 @@ default MediaTypeRegistry validAcceptMediaTypes() { * If no version is specified, handler is assumed to be compatible with Version.CURRENT * @return a version */ - default Version compatibleWithVersion() { - return Version.CURRENT; + default RestApiCompatibleVersion compatibleWithVersion() { + return RestApiCompatibleVersion.currentVersion(); } class Route { diff --git a/server/src/main/java/org/elasticsearch/rest/RestRequest.java b/server/src/main/java/org/elasticsearch/rest/RestRequest.java index a1caa2d0c7535..8895fcf9ad023 100644 --- a/server/src/main/java/org/elasticsearch/rest/RestRequest.java +++ b/server/src/main/java/org/elasticsearch/rest/RestRequest.java @@ -10,7 +10,6 @@ import org.apache.lucene.util.SetOnce; import org.elasticsearch.ElasticsearchParseException; -import org.elasticsearch.Version; import org.elasticsearch.common.Booleans; import org.elasticsearch.common.CheckedConsumer; import org.elasticsearch.common.Nullable; @@ -18,6 +17,7 @@ import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.collect.Tuple; +import org.elasticsearch.common.compatibility.RestApiCompatibleVersion; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; @@ -62,7 +62,7 @@ public class RestRequest implements ToXContent.Params { private final HttpChannel httpChannel; private final ParsedMediaType parsedAccept; private final ParsedMediaType parsedContentType; - private final Version compatibleVersion; + private final RestApiCompatibleVersion restApiCompatibleVersion; private HttpRequest httpRequest; private boolean contentConsumed = false; @@ -100,7 +100,7 @@ private RestRequest(NamedXContentRegistry xContentRegistry, Map this.rawPath = path; this.headers = Collections.unmodifiableMap(headers); this.requestId = requestId; - this.compatibleVersion = RestCompatibleVersionHelper.getCompatibleVersion(parsedAccept, parsedContentType, hasContent()); + this.restApiCompatibleVersion = RestCompatibleVersionHelper.getCompatibleVersion(parsedAccept, parsedContentType, hasContent()); } private static @Nullable ParsedMediaType parseHeaderWithMediaType(Map> headers, String headerName) { @@ -439,11 +439,9 @@ public NamedXContentRegistry getXContentRegistry() { public final XContentParser contentParser() throws IOException { BytesReference content = requiredContent(); // will throw exception if body or content type missing XContent xContent = xContentType.get().xContent(); - if (compatibleVersion == Version.CURRENT.minimumRestCompatibilityVersion()) { - return xContent.createParserForCompatibility(xContentRegistry, LoggingDeprecationHandler.INSTANCE, content.streamInput()); - } else { - return xContent.createParser(xContentRegistry, LoggingDeprecationHandler.INSTANCE, content.streamInput()); - } + return xContent.createParserForCompatibility(xContentRegistry, LoggingDeprecationHandler.INSTANCE, content.streamInput(), + restApiCompatibleVersion); + } /** @@ -551,8 +549,8 @@ public static XContentType parseContentType(List header) { throw new IllegalArgumentException("empty Content-Type header"); } - public Version getCompatibleVersion() { - return compatibleVersion; + public RestApiCompatibleVersion getRestApiCompatibleVersion() { + return restApiCompatibleVersion; } public static class MediaTypeHeaderException extends RuntimeException { diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterStatsAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterStatsAction.java index bc9221a5c1b69..1fee2db8429e1 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterStatsAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterStatsAction.java @@ -13,6 +13,7 @@ import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.RestActions.NodesResponseRestListener; +import org.elasticsearch.rest.action.RestCancellableNodeClient; import java.io.IOException; import java.util.List; @@ -37,7 +38,8 @@ public String getName() { public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { ClusterStatsRequest clusterStatsRequest = new ClusterStatsRequest().nodesIds(request.paramAsStringArray("nodeId", null)); clusterStatsRequest.timeout(request.param("timeout")); - return channel -> client.admin().cluster().clusterStats(clusterStatsRequest, new NodesResponseRestListener<>(channel)); + return channel -> new RestCancellableNodeClient(client, request.getHttpChannel()) + .admin().cluster().clusterStats(clusterStatsRequest, new NodesResponseRestListener<>(channel)); } @Override diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotUtils.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotUtils.java index 58f41ffd9b306..16d479d0f94f8 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotUtils.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotUtils.java @@ -9,9 +9,14 @@ import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.regex.Regex; +import org.elasticsearch.env.Environment; import org.elasticsearch.index.IndexNotFoundException; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.Arrays; import java.util.HashSet; import java.util.List; @@ -107,4 +112,29 @@ public static List filterIndices(List availableIndices, String[] } return List.copyOf(result); } + + /** + * Tries to find a suitable path to a searchable snapshots shared cache file in the data paths founds in the environment. + * + * @return path for the cache file or {@code null} if none could be found + */ + @Nullable + public static Path findCacheSnapshotCacheFilePath(Environment environment, long fileSize) throws IOException { + Path cacheFile = null; + for (Path path : environment.dataFiles()) { + Files.createDirectories(path); + // TODO: be resilient to this check failing and try next path? + long usableSpace = Environment.getUsableSpace(path); + Path p = path.resolve(SnapshotsService.CACHE_FILE_NAME); + if (Files.exists(p)) { + usableSpace += Files.size(p); + } + // TODO: leave some margin for error here + if (usableSpace > fileSize) { + cacheFile = p; + break; + } + } + return cacheFile; + } } diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java index 7fe2ce8caca49..913e99736323e 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java @@ -37,6 +37,7 @@ import org.elasticsearch.cluster.RestoreInProgress; import org.elasticsearch.cluster.SnapshotDeletionsInProgress; import org.elasticsearch.cluster.SnapshotsInProgress; +import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.util.CollectionUtils; import org.elasticsearch.repositories.RepositoryShardId; import org.elasticsearch.cluster.SnapshotsInProgress.ShardSnapshotStatus; @@ -132,6 +133,26 @@ public class SnapshotsService extends AbstractLifecycleComponent implements Clus public static final String UPDATE_SNAPSHOT_STATUS_ACTION_NAME = "internal:cluster/snapshot/update_snapshot_status"; + public static final String SHARED_CACHE_SETTINGS_PREFIX = "xpack.searchable.snapshot.shared_cache."; + + public static final Setting SHARED_CACHE_RANGE_SIZE_SETTING = Setting.byteSizeSetting( + SHARED_CACHE_SETTINGS_PREFIX + "range_size", + ByteSizeValue.ofMb(16), // default + Setting.Property.NodeScope + ); + public static final Setting SNAPSHOT_CACHE_REGION_SIZE_SETTING = Setting.byteSizeSetting( + SHARED_CACHE_SETTINGS_PREFIX + "region_size", + SHARED_CACHE_RANGE_SIZE_SETTING, + Setting.Property.NodeScope + ); + public static final Setting SNAPSHOT_CACHE_SIZE_SETTING = Setting.byteSizeSetting( + SHARED_CACHE_SETTINGS_PREFIX + "size", + ByteSizeValue.ZERO, + Setting.Property.NodeScope + ); + + public static final String CACHE_FILE_NAME = "shared_snapshot_cache"; + private final ClusterService clusterService; private final IndexNameExpressionResolver indexNameExpressionResolver; diff --git a/server/src/main/java/org/elasticsearch/tasks/TaskCancellationService.java b/server/src/main/java/org/elasticsearch/tasks/TaskCancellationService.java index d3946dd06279a..fe69df544ae7d 100644 --- a/server/src/main/java/org/elasticsearch/tasks/TaskCancellationService.java +++ b/server/src/main/java/org/elasticsearch/tasks/TaskCancellationService.java @@ -98,7 +98,7 @@ void doCancelTaskAndDescendants(CancellableTask task, String reason, boolean wai }); StepListener setBanListener = new StepListener<>(); setBanOnChildConnections(reason, waitForCompletion, task, childConnections, setBanListener); - setBanListener.whenComplete(groupedListener::onResponse, groupedListener::onFailure); + setBanListener.addListener(groupedListener); // If we start unbanning when the last child task completed and that child task executed with a specific user, then unban // requests are denied because internal requests can't run with a user. We need to remove bans with the current thread context. final Runnable removeBansRunnable = transportService.getThreadPool().getThreadContext() @@ -108,9 +108,9 @@ void doCancelTaskAndDescendants(CancellableTask task, String reason, boolean wai // if wait_for_completion is true, then only return when (1) bans are placed on child connections, (2) child tasks are // completed or failed, (3) the main task is cancelled. Otherwise, return after bans are placed on child connections. if (waitForCompletion) { - completedListener.whenComplete(r -> listener.onResponse(null), listener::onFailure); + completedListener.addListener(listener); } else { - setBanListener.whenComplete(r -> listener.onResponse(null), listener::onFailure); + setBanListener.addListener(listener); } } else { logger.trace("task [{}] doesn't have any children that should be cancelled", taskId); diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/AnalysisStatsTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/AnalysisStatsTests.java index e14718c83d378..ed08a51ca5840 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/AnalysisStatsTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/AnalysisStatsTests.java @@ -13,6 +13,7 @@ import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.common.io.stream.Writeable.Reader; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.tasks.TaskCancelledException; import org.elasticsearch.test.AbstractWireSerializingTestCase; import java.io.IOException; @@ -175,7 +176,7 @@ public void testAccountsRegularIndices() { Metadata metadata = new Metadata.Builder() .put(indexMetadata) .build(); - AnalysisStats analysisStats = AnalysisStats.of(metadata); + AnalysisStats analysisStats = AnalysisStats.of(metadata, () -> {}); IndexFeatureStats expectedStats = new IndexFeatureStats("german"); expectedStats.count = 1; expectedStats.indexCount = 1; @@ -198,7 +199,23 @@ public void testIgnoreSystemIndices() { Metadata metadata = new Metadata.Builder() .put(indexMetadata) .build(); - AnalysisStats analysisStats = AnalysisStats.of(metadata); + AnalysisStats analysisStats = AnalysisStats.of(metadata, () -> {}); assertEquals(Collections.emptySet(), analysisStats.getUsedBuiltInAnalyzers()); } + + public void testChecksForCancellation() { + Settings settings = Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 4) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1) + .build(); + IndexMetadata.Builder indexMetadata = new IndexMetadata.Builder("foo") + .settings(settings); + Metadata metadata = new Metadata.Builder() + .put(indexMetadata) + .build(); + expectThrows(TaskCancelledException.class, () -> AnalysisStats.of(metadata, () -> { + throw new TaskCancelledException("task cancelled"); + })); + } } diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/MappingStatsTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/MappingStatsTests.java index f01c85c16f47f..262bd251bd207 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/MappingStatsTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/MappingStatsTests.java @@ -13,6 +13,7 @@ import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.common.io.stream.Writeable.Reader; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.tasks.TaskCancelledException; import org.elasticsearch.test.AbstractWireSerializingTestCase; import java.io.IOException; @@ -75,7 +76,7 @@ public void testAccountsRegularIndices() { Metadata metadata = new Metadata.Builder() .put(indexMetadata) .build(); - MappingStats mappingStats = MappingStats.of(metadata); + MappingStats mappingStats = MappingStats.of(metadata, () -> {}); IndexFeatureStats expectedStats = new IndexFeatureStats("long"); expectedStats.count = 1; expectedStats.indexCount = 1; @@ -98,7 +99,23 @@ public void testIgnoreSystemIndices() { Metadata metadata = new Metadata.Builder() .put(indexMetadata) .build(); - MappingStats mappingStats = MappingStats.of(metadata); + MappingStats mappingStats = MappingStats.of(metadata, () -> {}); assertEquals(Collections.emptySet(), mappingStats.getFieldTypeStats()); } + + public void testChecksForCancellation() { + Settings settings = Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 4) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1) + .build(); + IndexMetadata.Builder indexMetadata = new IndexMetadata.Builder("foo") + .settings(settings); + Metadata metadata = new Metadata.Builder() + .put(indexMetadata) + .build(); + expectThrows(TaskCancelledException.class, () -> MappingStats.of(metadata, () -> { + throw new TaskCancelledException("task cancelled"); + })); + } } diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/shrink/ResizeRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/shrink/ResizeRequestTests.java index c3bf3ed9aa9f2..99218d1545083 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/shrink/ResizeRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/shrink/ResizeRequestTests.java @@ -14,6 +14,8 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.ByteSizeUnit; +import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.RandomCreateIndexGenerator; @@ -55,6 +57,12 @@ public void testToXContent() throws IOException { String actualRequestBody = Strings.toString(request); assertEquals("{\"settings\":{},\"aliases\":{}}", actualRequestBody); } + { + ResizeRequest request = new ResizeRequest("target", "source"); + request.setMaxSinglePrimarySize(new ByteSizeValue(100, ByteSizeUnit.MB)); + String actualRequestBody = Strings.toString(request); + assertEquals("{\"settings\":{},\"aliases\":{},\"max_single_primary_size\":\"100mb\"}", actualRequestBody); + } { ResizeRequest request = new ResizeRequest(); CreateIndexRequest target = new CreateIndexRequest("target"); diff --git a/server/src/test/java/org/elasticsearch/action/support/ListenableActionFutureTests.java b/server/src/test/java/org/elasticsearch/action/support/ListenableActionFutureTests.java index a04035564d536..8954fd6ede9da 100644 --- a/server/src/test/java/org/elasticsearch/action/support/ListenableActionFutureTests.java +++ b/server/src/test/java/org/elasticsearch/action/support/ListenableActionFutureTests.java @@ -68,7 +68,6 @@ protected void doRun() { } } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/68772") public void testListenersNotifiedOnCorrectThreads() throws InterruptedException { final int adderThreads = between(1, 5); @@ -90,7 +89,10 @@ public void testListenersNotifiedOnCorrectThreads() throws InterruptedException awaitSafe(barrier); final AtomicBoolean isComplete = new AtomicBoolean(); - if (postComplete.get()) { + if (completerThreads == 1 && postComplete.get()) { + // If there are multiple completer threads then onResponse might return on one thread, and hence postComplete is + // set, before the other completer thread notifies all the listeners. OTOH with one completer thread we know that + // postComplete indicates that the listeners were already notified. future.addListener(new ActionListener<>() { @Override public void onResponse(Void response) { diff --git a/server/src/test/java/org/elasticsearch/action/support/nodes/TransportNodesActionTests.java b/server/src/test/java/org/elasticsearch/action/support/nodes/TransportNodesActionTests.java index 4764da06eab5a..48f044d509a6d 100644 --- a/server/src/test/java/org/elasticsearch/action/support/nodes/TransportNodesActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/support/nodes/TransportNodesActionTests.java @@ -46,6 +46,7 @@ import java.util.concurrent.atomic.AtomicReferenceArray; import java.util.function.Supplier; +import static java.util.Collections.emptyMap; import static org.elasticsearch.test.ClusterServiceUtils.createClusterService; import static org.elasticsearch.test.ClusterServiceUtils.setState; import static org.mockito.Mockito.mock; @@ -90,8 +91,10 @@ public void testNodesSelectors() { } public void testNewResponseNullArray() { - TransportNodesAction action = getTestTransportNodesAction(); - expectThrows(NullPointerException.class, () -> action.newResponse(new TestNodesRequest(), null)); + TransportNodesAction action = getTestTransportNodesAction(); + final PlainActionFuture future = new PlainActionFuture<>(); + action.newResponse(new Task(1, "test", "test", "", null, emptyMap()), new TestNodesRequest(), null, future); + expectThrows(NullPointerException.class, future::actionGet); } public void testNewResponse() { @@ -99,9 +102,6 @@ public void testNewResponse() { TestNodesRequest request = new TestNodesRequest(); List expectedNodeResponses = mockList(TestNodeResponse::new, randomIntBetween(0, 2)); expectedNodeResponses.add(new TestNodeResponse()); - List nodeResponses = new ArrayList<>(expectedNodeResponses); - // This should be ignored: - nodeResponses.add(new OtherNodeResponse()); List failures = mockList( () -> new FailedNodeException( randomAlphaOfLength(8), @@ -116,7 +116,9 @@ public void testNewResponse() { AtomicReferenceArray atomicArray = new AtomicReferenceArray<>(allResponses.toArray()); - TestNodesResponse response = action.newResponse(request, atomicArray); + final PlainActionFuture future = new PlainActionFuture<>(); + action.newResponse(new Task(1, "test", "test", "", null, emptyMap()), request, atomicArray, future); + TestNodesResponse response = future.actionGet(); assertSame(request, response.request); // note: I shuffled the overall list, so it's not possible to guarantee that it's in the right order diff --git a/server/src/test/java/org/elasticsearch/common/util/CancellableSingleObjectCacheTests.java b/server/src/test/java/org/elasticsearch/common/util/CancellableSingleObjectCacheTests.java new file mode 100644 index 0000000000000..521e03cc8aac6 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/common/util/CancellableSingleObjectCacheTests.java @@ -0,0 +1,292 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.common.util; + +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.StepListener; +import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.common.util.concurrent.ConcurrentCollections; +import org.elasticsearch.tasks.TaskCancelledException; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.threadpool.TestThreadPool; +import org.elasticsearch.threadpool.ThreadPool; + +import java.util.LinkedList; +import java.util.Objects; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Function; + +import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.sameInstance; + +public class CancellableSingleObjectCacheTests extends ESTestCase { + + public void testNoPendingRefreshIfAlreadyCancelled() { + final TestCache testCache = new TestCache(); + final TestFuture future = new TestFuture(); + testCache.get("foo", () -> true, future); + testCache.assertPendingRefreshes(0); + assertTrue(future.isDone()); + expectThrows(ExecutionException.class, TaskCancelledException.class, future::get); + } + + public void testListenersCompletedByRefresh() { + final TestCache testCache = new TestCache(); + + // The first get() calls the refresh function + final TestFuture future0 = new TestFuture(); + testCache.get("foo", () -> false, future0); + testCache.assertPendingRefreshes(1); + + // The second get() with a matching key does not refresh again + final TestFuture future1 = new TestFuture(); + testCache.get("foo", () -> false, future1); + assertFalse(future0.isDone()); + assertFalse(future1.isDone()); + testCache.assertPendingRefreshes(1); + testCache.completeNextRefresh("foo", 1); + assertThat(future0.actionGet(0L), equalTo(1)); + assertThat(future0.actionGet(0L), sameInstance(future1.actionGet(0L))); + + // A further get() call with a matching key re-uses the cached value + final TestFuture future2 = new TestFuture(); + testCache.get("foo", () -> false, future2); + testCache.assertNoPendingRefreshes(); + assertThat(future2.actionGet(0L), sameInstance(future1.actionGet(0L))); + + // A call with a different key triggers another refresh + final TestFuture future3 = new TestFuture(); + testCache.get("bar", () -> false, future3); + assertFalse(future3.isDone()); + testCache.assertPendingRefreshes(1); + testCache.completeNextRefresh("bar", 2); + assertThat(future3.actionGet(0L), equalTo(2)); + } + + public void testListenerCompletedByRefreshEvenIfDiscarded() { + final TestCache testCache = new TestCache(); + + // This computation is discarded before it completes. + final TestFuture future1 = new TestFuture(); + final AtomicBoolean future1Cancelled = new AtomicBoolean(); + testCache.get("foo", future1Cancelled::get, future1); + future1Cancelled.set(true); + testCache.assertPendingRefreshes(1); + assertFalse(future1.isDone()); + + // However the refresh continues and makes its result available to a later get() call for the same value. + final TestFuture future2 = new TestFuture(); + testCache.get("foo", () -> false, future2); + testCache.assertPendingRefreshes(1); + testCache.completeNextRefresh("foo", 1); + assertThat(future2.actionGet(0L), equalTo(1)); + + // ... and the original listener is also completed successfully + assertThat(future1.actionGet(0L), sameInstance(future2.actionGet(0L))); + } + + public void testListenerCompletedWithCancellationExceptionIfRefreshCancelled() { + final TestCache testCache = new TestCache(); + + // This computation is discarded before it completes. + final TestFuture future1 = new TestFuture(); + final AtomicBoolean future1Cancelled = new AtomicBoolean(); + testCache.get("foo", future1Cancelled::get, future1); + future1Cancelled.set(true); + testCache.assertPendingRefreshes(1); + + assertFalse(future1.isDone()); + + // A second get() call with a non-matching key cancels the original refresh and starts another one + final TestFuture future2 = new TestFuture(); + testCache.get("bar", () -> false, future2); + testCache.assertPendingRefreshes(2); + testCache.assertNextRefreshCancelled(); + expectThrows(TaskCancelledException.class, () -> future1.actionGet(0L)); + testCache.completeNextRefresh("bar", 2); + assertThat(future2.actionGet(0L), equalTo(2)); + } + + public void testExceptionCompletesListenersButIsNotCached() { + final TestCache testCache = new TestCache(); + + // If a refresh results in an exception then all the pending get() calls complete exceptionally + final TestFuture future0 = new TestFuture(); + final TestFuture future1 = new TestFuture(); + testCache.get("foo", () -> false, future0); + testCache.get("foo", () -> false, future1); + testCache.assertPendingRefreshes(1); + final ElasticsearchException exception = new ElasticsearchException("simulated"); + testCache.completeNextRefresh(exception); + assertSame(exception, expectThrows(ElasticsearchException.class, () -> future0.actionGet(0L))); + assertSame(exception, expectThrows(ElasticsearchException.class, () -> future1.actionGet(0L))); + + testCache.assertNoPendingRefreshes(); + // The exception is not cached, however, so a subsequent get() call with a matching key performs another refresh + final TestFuture future2 = new TestFuture(); + testCache.get("foo", () -> false, future2); + testCache.assertPendingRefreshes(1); + testCache.completeNextRefresh("foo", 1); + assertThat(future2.actionGet(0L), equalTo(1)); + } + + public void testConcurrentRefreshesAndCancellation() throws InterruptedException { + final ThreadPool threadPool = new TestThreadPool("test"); + try { + final CancellableSingleObjectCache testCache = new CancellableSingleObjectCache<>() { + @Override + protected void refresh(String s, Runnable ensureNotCancelled, ActionListener listener) { + threadPool.generic().execute(() -> ActionListener.completeWith(listener, () -> { + ensureNotCancelled.run(); + if (s.equals("FAIL")) { + throw new ElasticsearchException("simulated"); + } + return s.length(); + })); + } + + @Override + protected String getKey(String s) { + return s; + } + }; + final int count = 1000; + final CountDownLatch startLatch = new CountDownLatch(1); + final CountDownLatch finishLatch = new CountDownLatch(count); + final BlockingQueue queue = ConcurrentCollections.newBlockingQueue(); + + for (int i = 0; i < count; i++) { + final boolean cancel = randomBoolean(); + final String input = randomFrom("FAIL", "foo", "barbaz", "quux", "gruly"); + queue.offer(() -> { + try { + assertTrue(startLatch.await(10, TimeUnit.SECONDS)); + } catch (InterruptedException e) { + throw new AssertionError(e); + } + + final StepListener stepListener = new StepListener<>(); + final AtomicBoolean isComplete = new AtomicBoolean(); + final AtomicBoolean isCancelled = new AtomicBoolean(); + testCache.get(input, isCancelled::get, ActionListener.runBefore(stepListener, + () -> assertTrue(isComplete.compareAndSet(false, true)))); + + final Runnable next = queue.poll(); + if (next != null) { + threadPool.generic().execute(next); + } + + if (cancel) { + isCancelled.set(true); + } + + stepListener.whenComplete(len -> { + finishLatch.countDown(); + assertThat(len, equalTo(input.length())); + assertNotEquals("FAIL", input); + }, e -> { + finishLatch.countDown(); + if (e instanceof TaskCancelledException) { + assertTrue(cancel); + } else { + assertEquals("FAIL", input); + } + }); + }); + } + + for (int i = 0; i < 10; i++) { + threadPool.generic().execute(Objects.requireNonNull(queue.poll())); + } + + startLatch.countDown(); + assertTrue(finishLatch.await(10, TimeUnit.SECONDS)); + } finally { + ThreadPool.terminate(threadPool, 10, TimeUnit.SECONDS); + } + } + + private static class TestCache extends CancellableSingleObjectCache { + + private final LinkedList>> pendingRefreshes = new LinkedList<>(); + + @Override + protected void refresh(String input, Runnable ensureNotCancelled, ActionListener listener) { + final StepListener> stepListener = new StepListener<>(); + pendingRefreshes.offer(stepListener); + stepListener.whenComplete(f -> ActionListener.completeWith(listener, () -> { + ensureNotCancelled.run(); + return f.apply(input); + }), listener::onFailure); + } + + @Override + protected String getKey(String s) { + return s; + } + + void assertPendingRefreshes(int expected) { + assertThat(pendingRefreshes.size(), equalTo(expected)); + } + + void assertNoPendingRefreshes() { + assertThat(pendingRefreshes, empty()); + } + + void completeNextRefresh(String key, int value) { + nextRefresh().onResponse(k -> { + assertThat(k, equalTo(key)); + return value; + }); + } + + void completeNextRefresh(Exception e) { + nextRefresh().onFailure(e); + } + + void assertNextRefreshCancelled() { + nextRefresh().onResponse(k -> { + throw new AssertionError("should not be called"); + }); + } + + private StepListener> nextRefresh() { + assertThat(pendingRefreshes, not(empty())); + final StepListener> nextRefresh = pendingRefreshes.poll(); + assertNotNull(nextRefresh); + return nextRefresh; + } + + } + + private static class TestFuture extends PlainActionFuture { + + private final AtomicBoolean isCompleted = new AtomicBoolean(); + + @Override + public void onResponse(Integer result) { + assertTrue(isCompleted.compareAndSet(false, true)); + super.onResponse(result); + } + + @Override + public void onFailure(Exception e) { + assertTrue(isCompleted.compareAndSet(false, true)); + super.onFailure(e); + } + } + +} diff --git a/server/src/test/java/org/elasticsearch/common/xcontent/CompatibleNamedXContentRegistryTests.java b/server/src/test/java/org/elasticsearch/common/xcontent/CompatibleNamedXContentRegistryTests.java index 5792ddbe95c9b..25ca050a848f0 100644 --- a/server/src/test/java/org/elasticsearch/common/xcontent/CompatibleNamedXContentRegistryTests.java +++ b/server/src/test/java/org/elasticsearch/common/xcontent/CompatibleNamedXContentRegistryTests.java @@ -11,6 +11,7 @@ import org.elasticsearch.Version; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.compatibility.RestApiCompatibleVersion; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.rest.FakeRestRequest; @@ -138,7 +139,7 @@ public void testCompatibleRequest() throws IOException { b.endObject(); String mediaType = XContentType.VND_JSON.toParsedMediaType() .responseContentTypeHeader(Map.of(MediaType.COMPATIBLE_WITH_PARAMETER_NAME, - String.valueOf(Version.CURRENT.minimumRestCompatibilityVersion().major))); + String.valueOf(RestApiCompatibleVersion.minimumSupported().major))); List mediaTypeList = Collections.singletonList(mediaType); RestRequest restRequest2 = new FakeRestRequest.Builder(compatibleRegistry) diff --git a/server/src/test/java/org/elasticsearch/node/NodeTests.java b/server/src/test/java/org/elasticsearch/node/NodeTests.java index fd0865061e377..408d7ba19a0b8 100644 --- a/server/src/test/java/org/elasticsearch/node/NodeTests.java +++ b/server/src/test/java/org/elasticsearch/node/NodeTests.java @@ -12,10 +12,14 @@ import org.elasticsearch.bootstrap.BootstrapCheck; import org.elasticsearch.bootstrap.BootstrapContext; import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.common.ParseField; import org.elasticsearch.common.breaker.CircuitBreaker; +import org.elasticsearch.common.compatibility.RestApiCompatibleVersion; import org.elasticsearch.common.network.NetworkModule; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.BoundTransportAddress; +import org.elasticsearch.common.xcontent.ContextParser; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.env.Environment; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.engine.Engine.Searcher; @@ -29,6 +33,7 @@ import org.elasticsearch.test.InternalTestCluster; import org.elasticsearch.test.MockHttpTransport; import org.elasticsearch.threadpool.ThreadPool; +import org.mockito.Mockito; import java.io.IOException; import java.nio.file.Path; @@ -44,6 +49,7 @@ import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS; import static org.elasticsearch.test.NodeRoles.dataNode; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; @@ -142,10 +148,10 @@ public void testServerNameNodeAttribute() throws IOException { private static Settings.Builder baseSettings() { final Path tempDir = createTempDir(); return Settings.builder() - .put(ClusterName.CLUSTER_NAME_SETTING.getKey(), InternalTestCluster.clusterName("single-node-cluster", randomLong())) - .put(Environment.PATH_HOME_SETTING.getKey(), tempDir) - .put(NetworkModule.TRANSPORT_TYPE_KEY, getTestTransportType()) - .put(dataNode()); + .put(ClusterName.CLUSTER_NAME_SETTING.getKey(), InternalTestCluster.clusterName("single-node-cluster", randomLong())) + .put(Environment.PATH_HOME_SETTING.getKey(), tempDir) + .put(NetworkModule.TRANSPORT_TYPE_KEY, getTestTransportType()) + .put(dataNode()); } public void testCloseOnOutstandingTask() throws Exception { @@ -156,7 +162,7 @@ public void testCloseOnOutstandingTask() throws Exception { final CountDownLatch threadRunning = new CountDownLatch(1); threadpool.executor(ThreadPool.Names.SEARCH).execute(() -> { threadRunning.countDown(); - while (shouldRun.get()); + while (shouldRun.get()) ; }); threadRunning.await(); node.close(); @@ -179,7 +185,7 @@ public void testCloseRaceWithTaskExecution() throws Exception { } try { threadpool.executor(ThreadPool.Names.SEARCH).execute(() -> { - while (shouldRun.get()); + while (shouldRun.get()) ; }); } catch (RejectedExecutionException e) { assertThat(e.getMessage(), containsString("[Terminated,")); @@ -218,7 +224,7 @@ public void testAwaitCloseTimeoutsOnNonInterruptibleTask() throws Exception { final CountDownLatch threadRunning = new CountDownLatch(1); threadpool.executor(ThreadPool.Names.SEARCH).execute(() -> { threadRunning.countDown(); - while (shouldRun.get()); + while (shouldRun.get()) ; }); threadRunning.await(); node.close(); @@ -261,7 +267,7 @@ public void testCloseOnLeakedIndexReaderReference() throws Exception { node.start(); IndicesService indicesService = node.injector().getInstance(IndicesService.class); assertAcked(node.client().admin().indices().prepareCreate("test") - .setSettings(Settings.builder().put(SETTING_NUMBER_OF_SHARDS, 1).put(SETTING_NUMBER_OF_REPLICAS, 0))); + .setSettings(Settings.builder().put(SETTING_NUMBER_OF_SHARDS, 1).put(SETTING_NUMBER_OF_REPLICAS, 0))); IndexService indexService = indicesService.iterator().next(); IndexShard shard = indexService.getShard(0); Searcher searcher = shard.acquireSearcher("test"); @@ -277,7 +283,7 @@ public void testCloseOnLeakedStoreReference() throws Exception { node.start(); IndicesService indicesService = node.injector().getInstance(IndicesService.class); assertAcked(node.client().admin().indices().prepareCreate("test") - .setSettings(Settings.builder().put(SETTING_NUMBER_OF_SHARDS, 1).put(SETTING_NUMBER_OF_REPLICAS, 0))); + .setSettings(Settings.builder().put(SETTING_NUMBER_OF_SHARDS, 1).put(SETTING_NUMBER_OF_REPLICAS, 0))); IndexService indexService = indicesService.iterator().next(); IndexShard shard = indexService.getShard(0); shard.store().incRef(); @@ -300,7 +306,7 @@ public void testCreateWithCircuitBreakerPlugins() throws IOException { CircuitBreakerPlugin breakerPlugin = node.getPluginsService().filterPlugins(CircuitBreakerPlugin.class).get(0); assertTrue(breakerPlugin instanceof MockCircuitBreakerPlugin); assertSame("plugin circuit breaker instance is not the same as breaker service's instance", - ((MockCircuitBreakerPlugin)breakerPlugin).myCircuitBreaker.get(), + ((MockCircuitBreakerPlugin) breakerPlugin).myCircuitBreaker.get(), service.getBreaker("test_breaker")); } } @@ -328,4 +334,73 @@ public void setCircuitBreaker(CircuitBreaker circuitBreaker) { myCircuitBreaker.set(circuitBreaker); } } + + + interface MockRestApiCompatibleVersion { + RestApiCompatibleVersion minimumRestCompatibilityVersion(); + } + + static MockRestApiCompatibleVersion MockCompatibleVersion = Mockito.mock(MockRestApiCompatibleVersion.class); + + static NamedXContentRegistry.Entry v7CompatibleEntries = new NamedXContentRegistry.Entry(Integer.class, + new ParseField("name"), Mockito.mock(ContextParser.class)); + static NamedXContentRegistry.Entry v8CompatibleEntries = new NamedXContentRegistry.Entry(Integer.class, + new ParseField("name2"), Mockito.mock(ContextParser.class)); + + public static class TestRestCompatibility1 extends Plugin { + + @Override + public List getNamedXContentForCompatibility() { + // real plugin will use CompatibleVersion.minimumRestCompatibilityVersion() + if (/*CompatibleVersion.minimumRestCompatibilityVersion()*/ + MockCompatibleVersion.minimumRestCompatibilityVersion().equals(RestApiCompatibleVersion.V_7)) { + //return set of N-1 entries + return List.of(v7CompatibleEntries); + } + // after major release, new compatible apis can be added before the old ones are removed. + if (/*CompatibleVersion.minimumRestCompatibilityVersion()*/ + MockCompatibleVersion.minimumRestCompatibilityVersion().equals(RestApiCompatibleVersion.V_8)) { + return List.of(v8CompatibleEntries); + + } + return super.getNamedXContentForCompatibility(); + } + } + + // This test shows an example on how multiple compatible namedxcontent can be present at the same time. + public void testLoadingMultipleRestCompatibilityPlugins() throws IOException { + + Mockito.when(MockCompatibleVersion.minimumRestCompatibilityVersion()) + .thenReturn(RestApiCompatibleVersion.V_7); + + { + Settings.Builder settings = baseSettings(); + + // throw an exception when two plugins are registered + List> plugins = basePlugins(); + plugins.add(TestRestCompatibility1.class); + + try (Node node = new MockNode(settings.build(), plugins)) { + List compatibleNamedXContents = node.getCompatibleNamedXContents(); + assertThat(compatibleNamedXContents, contains(v7CompatibleEntries)); + } + } + // after version bump CompatibleVersion.minimumRestCompatibilityVersion() will return V_8 + Mockito.when(MockCompatibleVersion.minimumRestCompatibilityVersion()) + .thenReturn(RestApiCompatibleVersion.V_8); + { + Settings.Builder settings = baseSettings(); + + // throw an exception when two plugins are registered + List> plugins = basePlugins(); + plugins.add(TestRestCompatibility1.class); + + try (Node node = new MockNode(settings.build(), plugins)) { + List compatibleNamedXContents = node.getCompatibleNamedXContents(); + assertThat(compatibleNamedXContents, contains(v8CompatibleEntries)); + } + } + } + + } diff --git a/server/src/test/java/org/elasticsearch/rest/MethodHandlersTests.java b/server/src/test/java/org/elasticsearch/rest/MethodHandlersTests.java index 312f26ea272ff..a827eaa523060 100644 --- a/server/src/test/java/org/elasticsearch/rest/MethodHandlersTests.java +++ b/server/src/test/java/org/elasticsearch/rest/MethodHandlersTests.java @@ -8,8 +8,8 @@ package org.elasticsearch.rest; -import org.elasticsearch.Version; import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.compatibility.RestApiCompatibleVersion; import org.elasticsearch.test.ESTestCase; import static org.hamcrest.Matchers.sameInstance; @@ -22,7 +22,7 @@ public void testLookupForDifferentMethodsSameVersion() { MethodHandlers methodHandlers = new MethodHandlers("path", putHandler, RestRequest.Method.PUT); methodHandlers.addMethods(postHandler, RestRequest.Method.POST); - RestHandler handler = methodHandlers.getHandler(RestRequest.Method.PUT, Version.CURRENT); + RestHandler handler = methodHandlers.getHandler(RestRequest.Method.PUT, RestApiCompatibleVersion.currentVersion()); assertThat(handler, sameInstance(putHandler)); } @@ -30,10 +30,10 @@ public void testLookupForHandlerUnderMultipleMethods() { RestHandler handler = new CurrentVersionHandler(); MethodHandlers methodHandlers = new MethodHandlers("path", handler, RestRequest.Method.PUT, RestRequest.Method.POST); - RestHandler handlerFound = methodHandlers.getHandler(RestRequest.Method.PUT, Version.CURRENT); + RestHandler handlerFound = methodHandlers.getHandler(RestRequest.Method.PUT, RestApiCompatibleVersion.currentVersion()); assertThat(handlerFound, sameInstance(handler)); - handlerFound = methodHandlers.getHandler(RestRequest.Method.POST, Version.CURRENT); + handlerFound = methodHandlers.getHandler(RestRequest.Method.POST, RestApiCompatibleVersion.currentVersion()); assertThat(handlerFound, sameInstance(handler)); } @@ -43,10 +43,10 @@ public void testLookupForHandlersUnderDifferentVersions() { MethodHandlers methodHandlers = new MethodHandlers("path", currentVersionHandler, RestRequest.Method.PUT); methodHandlers.addMethods(previousVersionHandler, RestRequest.Method.PUT); - RestHandler handler = methodHandlers.getHandler(RestRequest.Method.PUT, Version.CURRENT); + RestHandler handler = methodHandlers.getHandler(RestRequest.Method.PUT, RestApiCompatibleVersion.currentVersion()); assertThat(handler, sameInstance(currentVersionHandler)); - handler = methodHandlers.getHandler(RestRequest.Method.PUT, Version.CURRENT.previousMajor()); + handler = methodHandlers.getHandler(RestRequest.Method.PUT, RestApiCompatibleVersion.currentVersion().previousMajor()); assertThat(handler, sameInstance(previousVersionHandler)); } @@ -60,14 +60,14 @@ public void testExceptionOnOverride() { public void testMissingCurrentHandler(){ RestHandler previousVersionHandler = new PreviousVersionHandler(); MethodHandlers methodHandlers = new MethodHandlers("path", previousVersionHandler, RestRequest.Method.PUT, RestRequest.Method.POST); - RestHandler handler = methodHandlers.getHandler(RestRequest.Method.PUT, Version.CURRENT); + RestHandler handler = methodHandlers.getHandler(RestRequest.Method.PUT, RestApiCompatibleVersion.currentVersion()); assertNull(handler); } public void testMissingPriorHandlerReturnsCurrentHandler(){ RestHandler currentVersionHandler = new CurrentVersionHandler(); MethodHandlers methodHandlers = new MethodHandlers("path", currentVersionHandler, RestRequest.Method.PUT, RestRequest.Method.POST); - RestHandler handler = methodHandlers.getHandler(RestRequest.Method.PUT, Version.CURRENT.previousMajor()); + RestHandler handler = methodHandlers.getHandler(RestRequest.Method.PUT, RestApiCompatibleVersion.currentVersion().previousMajor()); assertThat(handler, sameInstance(currentVersionHandler)); } @@ -85,8 +85,8 @@ public void handleRequest(RestRequest request, RestChannel channel, NodeClient c } @Override - public Version compatibleWithVersion() { - return Version.CURRENT.previousMajor(); + public RestApiCompatibleVersion compatibleWithVersion() { + return RestApiCompatibleVersion.currentVersion().previousMajor(); } } } diff --git a/server/src/test/java/org/elasticsearch/rest/RestCompatibleVersionHelperTests.java b/server/src/test/java/org/elasticsearch/rest/RestCompatibleVersionHelperTests.java index 2548b2dfd5873..ae19cb7bde33a 100644 --- a/server/src/test/java/org/elasticsearch/rest/RestCompatibleVersionHelperTests.java +++ b/server/src/test/java/org/elasticsearch/rest/RestCompatibleVersionHelperTests.java @@ -8,7 +8,7 @@ package org.elasticsearch.rest; import org.elasticsearch.ElasticsearchStatusException; -import org.elasticsearch.Version; +import org.elasticsearch.common.compatibility.RestApiCompatibleVersion; import org.elasticsearch.common.xcontent.ParsedMediaType; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.hamcrest.ElasticsearchMatchers; @@ -20,9 +20,9 @@ import static org.hamcrest.Matchers.nullValue; public class RestCompatibleVersionHelperTests extends ESTestCase { - int CURRENT_VERSION = Version.CURRENT.major; - int PREVIOUS_VERSION = Version.CURRENT.major - 1; - int OBSOLETE_VERSION = Version.CURRENT.major - 2; + int CURRENT_VERSION = RestApiCompatibleVersion.currentVersion().major; + int PREVIOUS_VERSION = RestApiCompatibleVersion.currentVersion().major - 1; + int OBSOLETE_VERSION = RestApiCompatibleVersion.currentVersion().major - 2; public void testAcceptAndContentTypeCombinations() { assertThat(requestWith(acceptHeader(PREVIOUS_VERSION), contentTypeHeader(PREVIOUS_VERSION), bodyPresent()), isCompatible()); @@ -322,11 +322,11 @@ public void testVersionParsing() { } - private Matcher isCompatible() { + private Matcher isCompatible() { return requestHasVersion(PREVIOUS_VERSION); } - private Matcher requestHasVersion(int version) { + private Matcher requestHasVersion(int version) { return ElasticsearchMatchers.HasPropertyLambdaMatcher.hasProperty(v -> (int) v.major, equalTo(version)); } @@ -361,7 +361,7 @@ private String mediaType(String version) { return null; } - private Version requestWith(String accept, String contentType, String body) { + private RestApiCompatibleVersion requestWith(String accept, String contentType, String body) { ParsedMediaType parsedAccept = ParsedMediaType.parseMediaType(accept); ParsedMediaType parsedContentType = ParsedMediaType.parseMediaType(contentType); return RestCompatibleVersionHelper.getCompatibleVersion(parsedAccept, parsedContentType, body.isEmpty() == false); diff --git a/server/src/test/java/org/elasticsearch/rest/RestControllerTests.java b/server/src/test/java/org/elasticsearch/rest/RestControllerTests.java index fa3d5c37c3802..301132a233f99 100644 --- a/server/src/test/java/org/elasticsearch/rest/RestControllerTests.java +++ b/server/src/test/java/org/elasticsearch/rest/RestControllerTests.java @@ -13,6 +13,7 @@ import org.elasticsearch.common.breaker.CircuitBreaker; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.compatibility.RestApiCompatibleVersion; import org.elasticsearch.common.component.AbstractLifecycleComponent; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; @@ -233,7 +234,7 @@ public void testRegisterSecondMethodWithDifferentNamedWildcard() { private RestHandler v8mockHandler() { RestHandler mock = mock(RestHandler.class); - Mockito.when(mock.compatibleWithVersion()).thenReturn(Version.CURRENT); + Mockito.when(mock.compatibleWithVersion()).thenReturn(RestApiCompatibleVersion.currentVersion()); return mock; } @@ -368,7 +369,7 @@ public void testDispatchWorksWithNewlineDelimitedJson() { restController.registerHandler(RestRequest.Method.GET, "/foo", new RestHandler() { @Override public void handleRequest(RestRequest request, RestChannel channel, NodeClient client) throws Exception { - assertThat(request.contentParser().useCompatibility(), is(false)); + assertThat(request.contentParser().getRestApiCompatibleVersion(), is(RestApiCompatibleVersion.currentVersion())); channel.sendResponse(new BytesRestResponse(RestStatus.OK, BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY)); } @@ -395,7 +396,7 @@ public void testDispatchWithContentStream() { restController.registerHandler(RestRequest.Method.GET, "/foo", new RestHandler() { @Override public void handleRequest(RestRequest request, RestChannel channel, NodeClient client) throws Exception { - assertThat(request.contentParser().useCompatibility(), is(false)); + assertThat(request.contentParser().getRestApiCompatibleVersion(), is(RestApiCompatibleVersion.currentVersion())); channel.sendResponse(new BytesRestResponse(RestStatus.OK, BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY)); } @@ -627,7 +628,7 @@ public void testDispatchCompatibleHandler() { RestController restController = new RestController(Collections.emptySet(), null, client, circuitBreakerService, usageService); - final byte version = Version.CURRENT.minimumRestCompatibilityVersion().major; + final byte version = RestApiCompatibleVersion.minimumSupported().major; final String mediaType = randomCompatibleMediaType(version); FakeRestRequest fakeRestRequest = requestWithContent(mediaType); @@ -636,15 +637,16 @@ public void testDispatchCompatibleHandler() { restController.registerHandler(RestRequest.Method.GET, "/foo", new RestHandler() { @Override public void handleRequest(RestRequest request, RestChannel channel, NodeClient client) throws Exception { - assertThat(request.contentParser().useCompatibility(), is(true)); + // in real use case we will use exact version RestApiCompatibleVersion.V_7 XContentBuilder xContentBuilder = channel.newBuilder(); - assertThat(xContentBuilder.getCompatibleMajorVersion(), equalTo(version)); + assertThat(xContentBuilder.getRestApiCompatibilityVersion(), equalTo(RestApiCompatibleVersion.minimumSupported())); + assertThat(request.contentParser().getRestApiCompatibleVersion(), equalTo(RestApiCompatibleVersion.minimumSupported())); channel.sendResponse(new BytesRestResponse(RestStatus.OK, BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY)); } @Override - public Version compatibleWithVersion() { - return Version.CURRENT.minimumRestCompatibilityVersion(); + public RestApiCompatibleVersion compatibleWithVersion() { + return RestApiCompatibleVersion.minimumSupported(); } }); @@ -657,7 +659,7 @@ public void testDispatchCompatibleRequestToNewlyAddedHandler() { RestController restController = new RestController(Collections.emptySet(), null, client, circuitBreakerService, usageService); - final byte version = Version.CURRENT.minimumRestCompatibilityVersion().major; + final byte version = RestApiCompatibleVersion.minimumSupported().major; final String mediaType = randomCompatibleMediaType(version); FakeRestRequest fakeRestRequest = requestWithContent(mediaType); @@ -667,19 +669,20 @@ public void testDispatchCompatibleRequestToNewlyAddedHandler() { restController.registerHandler(RestRequest.Method.GET, "/foo", new RestHandler() { @Override public void handleRequest(RestRequest request, RestChannel channel, NodeClient client) throws Exception { - assertThat(request.contentParser().useCompatibility(), is(true)); XContentBuilder xContentBuilder = channel.newBuilder(); // even though the handler is CURRENT, the xContentBuilder has the version requested by a client. // This allows to implement the compatible logic within the serialisation without introducing V7 (compatible) handler // when only response shape has changed - assertThat(xContentBuilder.getCompatibleMajorVersion(), equalTo(version)); + assertThat(xContentBuilder.getRestApiCompatibilityVersion(), equalTo(RestApiCompatibleVersion.minimumSupported())); + assertThat(request.contentParser().getRestApiCompatibleVersion(), equalTo(RestApiCompatibleVersion.minimumSupported())); + channel.sendResponse(new BytesRestResponse(RestStatus.OK, BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY)); } @Override - public Version compatibleWithVersion() { - return Version.CURRENT; + public RestApiCompatibleVersion compatibleWithVersion() { + return RestApiCompatibleVersion.currentVersion(); } }); @@ -713,16 +716,16 @@ public void testCurrentVersionVNDMediaTypeIsNotUsingCompatibility() { public void handleRequest(RestRequest request, RestChannel channel, NodeClient client) throws Exception { // the media type is in application/vnd.elasticsearch form but with compatible-with=CURRENT. // Hence compatibility is not used. - assertThat(request.contentParser().useCompatibility(), is(false)); XContentBuilder xContentBuilder = channel.newBuilder(); - assertThat(xContentBuilder.getCompatibleMajorVersion(), equalTo(version)); + assertThat(request.contentParser().getRestApiCompatibleVersion(), equalTo(RestApiCompatibleVersion.currentVersion())); + assertThat(xContentBuilder.getRestApiCompatibilityVersion(), equalTo(RestApiCompatibleVersion.currentVersion())); channel.sendResponse(new BytesRestResponse(RestStatus.OK, BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY)); } @Override - public Version compatibleWithVersion() { - return Version.CURRENT; + public RestApiCompatibleVersion compatibleWithVersion() { + return RestApiCompatibleVersion.currentVersion(); } }); @@ -731,23 +734,6 @@ public Version compatibleWithVersion() { assertTrue(channel.getSendResponseCalled()); } - public void testRegisterIncompatibleVersionHandler() { - //using restController which uses a compatible version function returning always Version.CURRENT - final byte version = (byte) (Version.CURRENT.major - 2); - - expectThrows(AssertionError.class, - () -> restController.registerHandler(RestRequest.Method.GET, "/foo", new RestHandler() { - @Override - public void handleRequest(RestRequest request, RestChannel channel, NodeClient client) throws Exception { - } - - @Override - public Version compatibleWithVersion() { - return Version.fromString(version + ".0.0"); - } - })); - } - private static final class TestHttpServerTransport extends AbstractLifecycleComponent implements HttpServerTransport { diff --git a/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java b/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java index 055c9bf223656..6adb1470a4433 100644 --- a/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java @@ -2744,7 +2744,7 @@ public static Future submitRequest(TransportSer } responseListener.whenComplete(handler::handleResponse, e -> handler.handleException((TransportException) e)); final PlainActionFuture future = PlainActionFuture.newFuture(); - responseListener.whenComplete(future::onResponse, future::onFailure); + responseListener.addListener(future); return future; } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/SearchableSnapshotAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/SearchableSnapshotAction.java index 93d889bfff556..76b56809d2d5a 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/SearchableSnapshotAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/SearchableSnapshotAction.java @@ -101,7 +101,7 @@ public SearchableSnapshotAction(StreamInput in) throws IOException { } else { this.forceMergeIndex = true; } - if (in.getVersion().onOrAfter(Version.V_8_0_0)) { + if (in.getVersion().onOrAfter(Version.V_7_12_0)) { this.storageType = in.readOptionalEnum(MountSearchableSnapshotRequest.Storage.class); } else { this.storageType = null; @@ -333,7 +333,7 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getVersion().onOrAfter(Version.V_7_10_0)) { out.writeBoolean(forceMergeIndex); } - if (out.getVersion().onOrAfter(Version.V_8_0_0)) { + if (out.getVersion().onOrAfter(Version.V_7_12_0)) { out.writeOptionalEnum(storageType); } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/support/xcontent/WatcherXContentParser.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/support/xcontent/WatcherXContentParser.java index c06f3a2af84a4..8d76a1b7b312b 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/support/xcontent/WatcherXContentParser.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/support/xcontent/WatcherXContentParser.java @@ -9,6 +9,7 @@ import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.common.CheckedFunction; import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.compatibility.RestApiCompatibleVersion; import org.elasticsearch.common.xcontent.DeprecationHandler; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContentLocation; @@ -288,8 +289,8 @@ public void close() throws IOException { } @Override - public boolean useCompatibility() { - return false; + public RestApiCompatibleVersion getRestApiCompatibleVersion() { + return RestApiCompatibleVersion.currentVersion(); } @Override diff --git a/x-pack/plugin/eql/qa/mixed-node/build.gradle b/x-pack/plugin/eql/qa/mixed-node/build.gradle new file mode 100644 index 0000000000000..5e041a2aa5f2d --- /dev/null +++ b/x-pack/plugin/eql/qa/mixed-node/build.gradle @@ -0,0 +1,66 @@ +apply plugin: 'elasticsearch.testclusters' +apply plugin: 'elasticsearch.standalone-rest-test' +apply from : "$rootDir/gradle/bwc-test.gradle" +apply plugin: 'elasticsearch.rest-test' + +import org.elasticsearch.gradle.Version +import org.elasticsearch.gradle.VersionProperties +import org.elasticsearch.gradle.info.BuildParams +import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask + +dependencies { + testImplementation project(':x-pack:qa') + testImplementation(project(xpackModule('ql:test'))) + testImplementation project(path: xpackModule('eql'), configuration: 'default') +} + +tasks.named("integTest").configure{ enabled = false} + +for (Version bwcVersion : BuildParams.bwcVersions.wireCompatible.findAll { it.onOrAfter('7.10.0') }) { + if (bwcVersion == VersionProperties.getElasticsearchVersion()) { + // Not really a mixed cluster + continue; + } + + String baseName = "v${bwcVersion}" + + testClusters { + "${baseName}" { + versions = [bwcVersion.toString(), project.version] + numberOfNodes = 3 + testDistribution = 'DEFAULT' + setting 'xpack.security.enabled', 'false' + setting 'xpack.watcher.enabled', 'false' + setting 'xpack.ml.enabled', 'false' + setting 'xpack.eql.enabled', 'true' + setting 'xpack.license.self_generated.type', 'trial' + // for debugging purposes + // setting 'logger.org.elasticsearch.xpack.eql.plugin.TransportEqlSearchAction', 'TRACE' + } + } + + tasks.register("${baseName}#mixedClusterTest", StandaloneRestIntegTestTask) { + useCluster testClusters."${baseName}" + mustRunAfter("precommit") + doFirst { + // Getting the endpoints causes a wait for the cluster + println "Endpoints are: ${-> testClusters."${baseName}".allHttpSocketURI.join(",")}" + println "Upgrading one node to create a mixed cluster" + testClusters."${baseName}".nextNodeToNextVersion() + + println "Upgrade complete, endpoints are: ${-> testClusters."${baseName}".allHttpSocketURI.join(",")}" + nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}") + nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}") + } + onlyIf { project.bwc_tests_enabled } + } + + tasks.register(bwcTaskName(bwcVersion)) { + dependsOn "${baseName}#mixedClusterTest" + } + + // run these bwc tests as part of the "check" task + tasks.named("check").configure { + dependsOn "${baseName}#mixedClusterTest" + } +} diff --git a/x-pack/plugin/eql/qa/mixed-node/src/test/java/org/elasticsearch/xpack/eql/qa/mixed_node/EqlSearchIT.java b/x-pack/plugin/eql/qa/mixed-node/src/test/java/org/elasticsearch/xpack/eql/qa/mixed_node/EqlSearchIT.java new file mode 100644 index 0000000000000..8e3c45f816ffc --- /dev/null +++ b/x-pack/plugin/eql/qa/mixed-node/src/test/java/org/elasticsearch/xpack/eql/qa/mixed_node/EqlSearchIT.java @@ -0,0 +1,239 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.eql.qa.mixed_node; + +import org.apache.http.HttpHost; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.elasticsearch.test.NotEqualMessageBuilder; +import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.xpack.ql.TestNode; +import org.elasticsearch.xpack.ql.TestNodes; +import org.junit.After; +import org.junit.Before; + +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static java.util.Arrays.asList; +import static java.util.Collections.emptyMap; +import static java.util.Collections.singletonList; +import static java.util.Collections.singletonMap; +import static java.util.Collections.unmodifiableList; +import static org.elasticsearch.xpack.ql.TestUtils.buildNodeAndVersions; +import static org.elasticsearch.xpack.ql.TestUtils.readResource; + +/** + * Class testing the behavior of events and sequence queries in a mixed cluster scenario (during rolling upgrade). + * The test is against a three-node cluster where one node is upgraded, the other two are on the old version. + * + */ +public class EqlSearchIT extends ESRestTestCase { + + private static final String index = "test_eql_mixed_versions"; + private static int numShards; + private static int numReplicas = 1; + private static int numDocs; + private static TestNodes nodes; + private static List newNodes; + private static List bwcNodes; + + @Before + public void createIndex() throws IOException { + nodes = buildNodeAndVersions(client()); + numShards = nodes.size(); + numDocs = randomIntBetween(numShards, 15); + newNodes = new ArrayList<>(nodes.getNewNodes()); + bwcNodes = new ArrayList<>(nodes.getBWCNodes()); + + String mappings = readResource(EqlSearchIT.class.getResourceAsStream("/eql_mapping.json")); + createIndex( + index, + Settings.builder() + .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), numShards) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, numReplicas) + .build(), + mappings + ); + } + + @After + public void cleanUpIndex() throws IOException { + if (indexExists(index)) { + deleteIndex(index); + } + } + + public void testEventsWithRequestToOldNodes() throws Exception { + assertEventsQueryOnNodes(bwcNodes); + } + + public void testEventsWithRequestToUpgradedNodes() throws Exception { + assertEventsQueryOnNodes(newNodes); + } + + public void testSequencesWithRequestToOldNodes() throws Exception { + assertSequncesQueryOnNodes(bwcNodes); + } + + public void testSequencesWithRequestToUpgradedNodes() throws Exception { + assertSequncesQueryOnNodes(newNodes); + } + + private void assertEventsQueryOnNodes(List nodesList) throws Exception { + final String event = randomEvent(); + Map expectedResponse = prepareEventsTestData(event); + try ( + RestClient client = buildClient(restClientSettings(), + nodesList.stream().map(TestNode::getPublishAddress).toArray(HttpHost[]::new)) + ) { + // filter only the relevant bits of the response + String filterPath = "filter_path=hits.events._source.@timestamp,hits.events._source.event_type,hits.events._source.sequence"; + + Request request = new Request("POST", index + "/_eql/search?" + filterPath); + request.setJsonEntity("{\"query\":\"" + event + " where true\"}"); + assertBusy(() -> { assertResponse(expectedResponse, runEql(client, request)); }); + } + } + + private void assertSequncesQueryOnNodes(List nodesList) throws Exception { + Map expectedResponse = prepareSequencesTestData(); + try ( + RestClient client = buildClient(restClientSettings(), + nodesList.stream().map(TestNode::getPublishAddress).toArray(HttpHost[]::new)) + ) { + String filterPath = "filter_path=hits.sequences.join_keys,hits.sequences.events._id,hits.sequences.events._source"; + String query = "sequence by `sequence` with maxspan=100ms [success where true] by correlation_success1, correlation_success2 " + + "[failure where true] by correlation_failure1, correlation_failure2"; + String filter = "{\"range\":{\"@timestamp\":{\"gte\":\"1970-05-01\"}}}"; + + Request request = new Request("POST", index + "/_eql/search?" + filterPath); + request.setJsonEntity("{\"query\":\"" + query + "\",\"filter\":" + filter + "}"); + assertBusy(() -> { assertResponse(expectedResponse, runEql(client, request)); }); + } + } + + private String randomEvent() { + return randomFrom("success", "failure"); + } + + private Map prepareEventsTestData(String event) throws IOException { + List> sourceEvents = new ArrayList>(); + Map expectedResponse = singletonMap("hits", singletonMap("events", sourceEvents)); + + for (int i = 0; i < numDocs; i++) { + StringBuilder builder = new StringBuilder(); + final String randomEvent = randomEvent(); + builder.append("{"); + builder.append("\"@timestamp\":" + i + ","); + builder.append("\"event_type\":\"" + randomEvent + "\","); + builder.append("\"sequence\":" + i); + builder.append("}"); + if (randomEvent.equals(event)) { + Map eventSource = new HashMap<>(); + eventSource.put("@timestamp", i); + eventSource.put("event_type", randomEvent); + eventSource.put("sequence", i); + sourceEvents.add(singletonMap("_source", eventSource)); + } + + Request request = new Request("PUT", index + "/_doc/" + i); + request.setJsonEntity(builder.toString()); + assertOK(client().performRequest(request)); + } + if (sourceEvents.isEmpty()) { + return emptyMap(); + } + return expectedResponse; + } + + /* + * Output to compare with looks like this: + * { + * "hits": { + * "sequences": [ + * { + * "join_keys": [ + * 44, + * "C", + * "D" + * ], + * "events": [ + * { + * "_id": "14", + * "_source": { + * ... + * } + * } + * ] + * } + * } + * } + * + */ + private Map prepareSequencesTestData() throws IOException { + Map event14 = new HashMap<>(); + Map event14Source = new HashMap<>(); + event14.put("_id", "14"); + event14.put("_source", event14Source); + event14Source.put("@timestamp", "12345678914"); + event14Source.put("event_type", "success"); + event14Source.put("sequence", 44); + event14Source.put("correlation_success1", "C"); + event14Source.put("correlation_success2", "D"); + + Map event15 = new HashMap<>(); + Map event15Source = new HashMap<>(); + event15.put("_id", "15"); + event15.put("_source", event15Source); + event15Source.put("@timestamp", "12345678999"); + event15Source.put("event_type", "failure"); + event15Source.put("sequence", 44); + event15Source.put("correlation_failure1", "C"); + event15Source.put("correlation_failure2", "D"); + + Map sequence = new HashMap<>(); + List> events = unmodifiableList(asList(event14, event15)); + List> sequences = singletonList(sequence); + Map expectedResponse = singletonMap("hits", singletonMap("sequences", sequences)); + + sequence.put("join_keys", asList(44, "C", "D")); + sequence.put("events", events); + + final String bulkEntries = readResource(EqlSearchIT.class.getResourceAsStream("/eql_data.json")); + Request request = new Request("POST", index + "/_bulk?refresh"); + request.setJsonEntity(bulkEntries); + assertOK(client().performRequest(request)); + + return expectedResponse; + } + + private void assertResponse(Map expected, Map actual) { + if (false == expected.equals(actual)) { + NotEqualMessageBuilder message = new NotEqualMessageBuilder(); + message.compareMaps(actual, expected); + fail("Response does not match:\n" + message.toString()); + } + } + + private Map runEql(RestClient client, Request request) throws IOException { + Response response = client.performRequest(request); + try (InputStream content = response.getEntity().getContent()) { + return XContentHelper.convertToMap(JsonXContent.jsonXContent, content, false); + } + } +} diff --git a/x-pack/plugin/eql/qa/mixed-node/src/test/resources/eql_data.json b/x-pack/plugin/eql/qa/mixed-node/src/test/resources/eql_data.json new file mode 100644 index 0000000000000..3fc6e28834ea8 --- /dev/null +++ b/x-pack/plugin/eql/qa/mixed-node/src/test/resources/eql_data.json @@ -0,0 +1,30 @@ +{"index":{"_id":1}} +{"@timestamp":"1234567891","event_type":"success","sequence":1,"correlation_success1":"A","correlation_success2":"B"} +{"index":{"_id":2}} +{"@timestamp":"1234567892","event_type":"failure","sequence":2,"correlation_failure1":"A","correlation_failure2":"B"} +{"index":{"_id":3}} +{"@timestamp":"1234567893","event_type":"success","sequence":3,"correlation_success1":"A","correlation_success2":"A"} +{"index":{"_id":4}} +{"@timestamp":"1234567894","event_type":"success","sequence":4,"correlation_success1":"C","correlation_success2":"C"} +{"index":{"_id":5}} +{"@timestamp":"1234567895","event_type":"failure","sequence":5,"correlation_failure1":"B","correlation_failure2":"C"} +{"index":{"_id":6}} +{"@timestamp":"1234567896","event_type":"success","sequence":1,"correlation_success1":"A","correlation_success2":"A"} +{"index":{"_id":7}} +{"@timestamp":"1234567897","event_type":"failure","sequence":1,"correlation_failure1":"A","correlation_failure2":"A"} +{"index":{"_id":8}} +{"@timestamp":"1234567898","event_type":"success","sequence":3,"correlation_success1":"A","correlation_success2":"A"} +{"index":{"_id":9}} +{"@timestamp":"1234567899","event_type":"success","sequence":4,"correlation_success1":"C","correlation_success2":"B"} +{"index":{"_id":10}} +{"@timestamp":"12345678910","event_type":"failure","sequence":4,"correlation_failure1":"B","correlation_failure2":"B"} +{"index":{"_id":11}} +{"@timestamp":"12345678911","event_type":"success","sequence":1,"correlation_success1":"A","correlation_success2":"A"} +{"index":{"_id":12}} +{"@timestamp":"12345678912","event_type":"failure","sequence":1,"correlation_failure1":"A","correlation_failure2":"B"} +{"index":{"_id":13}} +{"@timestamp":"12345678913","event_type":"success","sequence":3,"correlation_success1":"A","correlation_success2":"A"} +{"index":{"_id":14}} +{"@timestamp":"12345678914","event_type":"success","sequence":44,"correlation_success1":"C","correlation_success2":"D"} +{"index":{"_id":15}} +{"@timestamp":"12345678999","event_type":"failure","sequence":44,"correlation_failure1":"C","correlation_failure2":"D"} diff --git a/x-pack/plugin/eql/qa/mixed-node/src/test/resources/eql_mapping.json b/x-pack/plugin/eql/qa/mixed-node/src/test/resources/eql_mapping.json new file mode 100644 index 0000000000000..f56dea6722183 --- /dev/null +++ b/x-pack/plugin/eql/qa/mixed-node/src/test/resources/eql_mapping.json @@ -0,0 +1,35 @@ + "properties": { + "@timestamp": { + "type": "date" + }, + "event_type": { + "type": "keyword" + }, + "sequence": { + "type": "long" + }, + "correlation_success1": { + "type": "wildcard" + }, + "correlation_failure1": { + "type": "wildcard" + }, + "correlation_success2": { + "type": "keyword" + }, + "correlation_failure2": { + "type": "keyword" + }, + "event": { + "properties": { + "category": { + "type": "alias", + "path": "event_type" + }, + "sequence": { + "type": "alias", + "path": "sequence" + } + } + } + } diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/BasicQueryClient.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/BasicQueryClient.java index a357491496085..38a2b292cb9c9 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/BasicQueryClient.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/BasicQueryClient.java @@ -57,7 +57,7 @@ public void query(QueryRequest request, ActionListener listener) // set query timeout searchSource.timeout(cfg.requestTimeout()); - SearchRequest search = prepareRequest(client, searchSource, false, indices); + SearchRequest search = prepareRequest(searchSource, false, indices); search(search, searchLogListener(listener, log)); } @@ -138,7 +138,7 @@ public void fetchHits(Iterable> refs, ActionListener createExtractor(List fields, E public static HitExtractor createExtractor(FieldExtraction ref, EqlConfiguration cfg) { if (ref instanceof SearchHitFieldRef) { SearchHitFieldRef f = (SearchHitFieldRef) ref; - return new FieldHitExtractor(f.name(), f.fullFieldName(), f.getDataType(), cfg.zoneId(), f.useDocValue(), f.hitName(), - FAIL_IF_MULTIVALUE); + return new FieldHitExtractor(f.name(), f.getDataType(), cfg.zoneId(), f.hitName(), FAIL_IF_MULTIVALUE); } if (ref instanceof ComputedRef) { @@ -147,16 +146,16 @@ public static HitExtractor createExtractor(FieldExtraction ref, EqlConfiguration } - public static SearchRequest prepareRequest(Client client, - SearchSourceBuilder source, + public static SearchRequest prepareRequest(SearchSourceBuilder source, boolean includeFrozen, String... indices) { - return client.prepareSearch(indices) - .setSource(source) - .setAllowPartialSearchResults(false) - .setIndicesOptions( - includeFrozen ? IndexResolver.FIELD_CAPS_FROZEN_INDICES_OPTIONS : IndexResolver.FIELD_CAPS_INDICES_OPTIONS) - .request(); + SearchRequest searchRequest = new SearchRequest(SWITCH_TO_FIELDS_API_VERSION); + searchRequest.indices(indices); + searchRequest.source(source); + searchRequest.allowPartialSearchResults(false); + searchRequest.indicesOptions( + includeFrozen ? IndexResolver.FIELD_CAPS_FROZEN_INDICES_OPTIONS : IndexResolver.FIELD_CAPS_INDICES_OPTIONS); + return searchRequest; } public static List searchHits(SearchResponse response) { diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/SourceGenerator.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/SourceGenerator.java index adff22ec90bf2..74d59279d26d5 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/SourceGenerator.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/SourceGenerator.java @@ -6,10 +6,8 @@ */ package org.elasticsearch.xpack.eql.execution.search; -import org.elasticsearch.common.util.CollectionUtils; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.elasticsearch.search.fetch.subphase.FetchSourceContext; import org.elasticsearch.search.sort.FieldSortBuilder; import org.elasticsearch.search.sort.NestedSortBuilder; import org.elasticsearch.search.sort.ScriptSortBuilder.ScriptSortType; @@ -60,13 +58,8 @@ public static SearchSourceBuilder sourceBuilder(QueryContainer container, QueryB sorting(container, source); - // disable the source if there are no includes - if (source.fetchSource() == null || CollectionUtils.isEmpty(source.fetchSource().includes())) { - source.fetchSource(FetchSourceContext.DO_NOT_FETCH_SOURCE); - } else { - // use true to fetch only the needed bits from the source - source.fetchSource(true); - } + // disable the source, as we rely on "fields" API + source.fetchSource(false); if (container.limit() != null) { // add size and from diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/extractor/FieldHitExtractor.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/extractor/FieldHitExtractor.java index 3a0bf90cc1f26..fc886efa7268b 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/extractor/FieldHitExtractor.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/extractor/FieldHitExtractor.java @@ -29,9 +29,8 @@ public FieldHitExtractor(StreamInput in) throws IOException { super(in); } - public FieldHitExtractor(String name, String fullFieldName, DataType dataType, ZoneId zoneId, boolean useDocValue, String hitName, - MultiValueHandling multiValueHandling) { - super(name, fullFieldName, dataType, zoneId, useDocValue, hitName, multiValueHandling); + public FieldHitExtractor(String name, DataType dataType, ZoneId zoneId, String hitName, MultiValueHandling multiValueHandling) { + super(name, dataType, zoneId, hitName, multiValueHandling); } @Override diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/extractor/TimestampFieldHitExtractor.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/extractor/TimestampFieldHitExtractor.java index eacae1b6bd654..f0ca24b0e9c08 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/extractor/TimestampFieldHitExtractor.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/extractor/TimestampFieldHitExtractor.java @@ -10,8 +10,7 @@ public class TimestampFieldHitExtractor extends FieldHitExtractor { public TimestampFieldHitExtractor(FieldHitExtractor target) { - super(target.fieldName(), target.fullFieldName(), target.dataType(), target.zoneId(), target.useDocValues(), target.hitName(), - target.multiValueExtraction()); + super(target.fieldName(), target.dataType(), target.zoneId(), target.hitName(), target.multiValueExtraction()); } @Override diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/TransportEqlSearchAction.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/TransportEqlSearchAction.java index 8e7a5133d1117..e454b7c908c4e 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/TransportEqlSearchAction.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/TransportEqlSearchAction.java @@ -6,7 +6,10 @@ */ package org.elasticsearch.xpack.eql.plugin; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionListenerResponseHandler; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.HandledTransportAction; import org.elasticsearch.client.Client; @@ -43,14 +46,17 @@ import static org.elasticsearch.action.ActionListener.wrap; import static org.elasticsearch.xpack.core.ClientHelper.ASYNC_SEARCH_ORIGIN; +import static org.elasticsearch.xpack.ql.plugin.TransportActionUtils.executeRequestWithRetryAttempt; public class TransportEqlSearchAction extends HandledTransportAction implements AsyncTaskManagementService.AsyncOperation { + private static final Logger log = LogManager.getLogger(TransportEqlSearchAction.class); private final SecurityContext securityContext; private final ClusterService clusterService; private final PlanExecutor planExecutor; private final ThreadPool threadPool; + private final TransportService transportService; private final AsyncTaskManagementService asyncTaskManagementService; @Inject @@ -64,6 +70,7 @@ public TransportEqlSearchAction(Settings settings, ClusterService clusterService this.clusterService = clusterService; this.planExecutor = planExecutor; this.threadPool = threadPool; + this.transportService = transportService; this.asyncTaskManagementService = new AsyncTaskManagementService<>(XPackPlugin.ASYNC_RESULTS_INDEX, client, ASYNC_SEARCH_ORIGIN, registry, taskManager, EqlSearchAction.INSTANCE.name(), this, EqlSearchTask.class, clusterService, threadPool); @@ -78,8 +85,7 @@ public EqlSearchTask createTask(EqlSearchRequest request, long id, String type, @Override public void execute(EqlSearchRequest request, EqlSearchTask task, ActionListener listener) { - operation(planExecutor, task, request, username(securityContext), clusterName(clusterService), - clusterService.localNode().getId(), listener); + operation(planExecutor, task, request, username(securityContext), transportService, clusterService, listener); } @Override @@ -99,13 +105,15 @@ protected void doExecute(Task task, EqlSearchRequest request, ActionListener listener) { + TransportService transportService, ClusterService clusterService, + ActionListener listener) { + String nodeId = clusterService.localNode().getId(); + String clusterName = clusterName(clusterService); // TODO: these should be sent by the client ZoneId zoneId = DateUtils.of("Z"); QueryBuilder filter = request.filter(); @@ -122,8 +130,12 @@ public static void operation(PlanExecutor planExecutor, EqlSearchTask task, EqlS EqlConfiguration cfg = new EqlConfiguration(request.indices(), zoneId, username, clusterName, filter, timeout, request.indicesOptions(), request.fetchSize(), clientId, new TaskId(nodeId, task.getId()), task); - planExecutor.eql(cfg, request.query(), params, wrap(r -> listener.onResponse(createResponse(r, task.getExecutionId())), - listener::onFailure)); + executeRequestWithRetryAttempt(clusterService, listener::onFailure, + onFailure -> planExecutor.eql(cfg, request.query(), params, + wrap(r -> listener.onResponse(createResponse(r, task.getExecutionId())), onFailure)), + node -> transportService.sendRequest(node, EqlSearchAction.NAME, request, + new ActionListenerResponseHandler<>(listener, EqlSearchResponse::new, ThreadPool.Names.SAME)), + log); } static EqlSearchResponse createResponse(Results results, AsyncExecutionId id) { diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/querydsl/container/FieldExtractorRegistry.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/querydsl/container/FieldExtractorRegistry.java index 2c4049d1511a5..7726d0d7e3a0d 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/querydsl/container/FieldExtractorRegistry.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/querydsl/container/FieldExtractorRegistry.java @@ -13,7 +13,6 @@ import org.elasticsearch.xpack.ql.expression.Expressions; import org.elasticsearch.xpack.ql.expression.FieldAttribute; import org.elasticsearch.xpack.ql.expression.gen.pipeline.ConstantInput; -import org.elasticsearch.xpack.ql.type.DataTypes; import java.util.HashMap; import java.util.Map; @@ -46,39 +45,6 @@ private FieldExtraction createFieldExtractionFor(Expression expression) { } private FieldExtraction topHitFieldExtractor(FieldAttribute fieldAttr) { - FieldAttribute actualField = fieldAttr; - FieldAttribute rootField = fieldAttr; - StringBuilder fullFieldName = new StringBuilder(fieldAttr.field().getName()); - - // Only if the field is not an alias (in which case it will be taken out from docvalue_fields if it's isAggregatable()), - // go up the tree of parents until a non-object (and non-nested) type of field is found and use that specific parent - // as the field to extract data from, from _source. We do it like this because sub-fields are not in the _source, only - // the root field to which those sub-fields belong to, are. Instead of "text_field.keyword_subfield" for _source extraction, - // we use "text_field", because there is no source for "keyword_subfield". - /* - * "text_field": { - * "type": "text", - * "fields": { - * "keyword_subfield": { - * "type": "keyword" - * } - * } - * } - */ - if (fieldAttr.field().isAlias() == false) { - while (actualField.parent() != null - && actualField.parent().field().getDataType() != DataTypes.OBJECT - && actualField.parent().field().getDataType() != DataTypes.NESTED - && actualField.field().getDataType().hasDocValues() == false) { - actualField = actualField.parent(); - } - } - while (rootField.parent() != null) { - fullFieldName.insert(0, ".").insert(0, rootField.parent().field().getName()); - rootField = rootField.parent(); - } - - return new SearchHitFieldRef(actualField.name(), fullFieldName.toString(), fieldAttr.field().getDataType(), - fieldAttr.field().isAggregatable(), fieldAttr.field().isAlias()); + return new SearchHitFieldRef(fieldAttr.name(), fieldAttr.field().getDataType(), fieldAttr.field().isAlias()); } } diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/querydsl/container/SearchHitFieldRef.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/querydsl/container/SearchHitFieldRef.java index c95d6b2707e4a..9778eecd6a38b 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/querydsl/container/SearchHitFieldRef.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/querydsl/container/SearchHitFieldRef.java @@ -13,7 +13,6 @@ import static org.elasticsearch.xpack.ql.type.DataTypes.DATETIME; import static org.elasticsearch.xpack.ql.type.DataTypes.DATETIME_NANOS; -import static org.elasticsearch.xpack.ql.type.DataTypes.KEYWORD; // NB: this class is taken from SQL - it hasn't been ported over to QL // since at this stage is unclear whether the whole FieldExtraction infrastructure @@ -21,23 +20,16 @@ public class SearchHitFieldRef implements FieldExtraction { private final String name; - private final String fullFieldName; // path included. If field full path is a.b.c, full field name is "a.b.c" and name is "c" private final DataType dataType; - private final boolean docValue; private final String hitName; - public SearchHitFieldRef(String name, String fullFieldName, DataType dataType, boolean useDocValueInsteadOfSource, boolean isAlias) { - this(name, fullFieldName, dataType, useDocValueInsteadOfSource, isAlias, null); + public SearchHitFieldRef(String name, DataType dataType, boolean isAlias) { + this(name, dataType, isAlias, null); } - public SearchHitFieldRef(String name, String fullFieldName, DataType dataType, boolean useDocValueInsteadOfSource, boolean isAlias, - String hitName) { + public SearchHitFieldRef(String name, DataType dataType, boolean isAlias, String hitName) { this.name = name; - this.fullFieldName = fullFieldName; this.dataType = dataType; - // these field types can only be extracted from docvalue_fields (ie, values already computed by Elasticsearch) - // because, for us to be able to extract them from _source, we would need the mapping of those fields (which we don't have) - this.docValue = isAlias ? useDocValueInsteadOfSource : (hasDocValues(dataType) ? useDocValueInsteadOfSource : false); this.hitName = hitName; } @@ -49,29 +41,17 @@ public String name() { return name; } - public String fullFieldName() { - return fullFieldName; - } - public DataType getDataType() { return dataType; } - public boolean useDocValue() { - return docValue; - } - @Override public void collectFields(QlSourceBuilder sourceBuilder) { // nested fields are handled by inner hits if (hitName != null) { return; } - if (docValue) { - sourceBuilder.addDocField(name, format(dataType)); - } else { - sourceBuilder.addSourceField(name); - } + sourceBuilder.addFetchField(name, format(dataType)); } @Override @@ -84,10 +64,6 @@ public String toString() { return name; } - private static boolean hasDocValues(DataType dataType) { - return dataType == KEYWORD || dataType == DATETIME || dataType == DATETIME_NANOS; - } - private static String format(DataType dataType) { if (dataType == DATETIME_NANOS) { return "strict_date_optional_time_nanos"; diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/CancellationTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/CancellationTests.java index e3704ba26540a..d69f522f8bf47 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/CancellationTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/analysis/CancellationTests.java @@ -14,10 +14,14 @@ import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.tasks.TaskCancelledException; import org.elasticsearch.tasks.TaskId; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.eql.action.EqlSearchRequest; import org.elasticsearch.xpack.eql.action.EqlSearchResponse; import org.elasticsearch.xpack.eql.action.EqlSearchTask; @@ -51,12 +55,13 @@ public void testCancellationBeforeFieldCaps() throws InterruptedException { Client client = mock(Client.class); EqlSearchTask task = mock(EqlSearchTask.class); when(task.isCancelled()).thenReturn(true); + ClusterService mockClusterService = mockClusterService(); IndexResolver indexResolver = new IndexResolver(client, randomAlphaOfLength(10), DefaultDataTypeRegistry.INSTANCE); PlanExecutor planExecutor = new PlanExecutor(client, indexResolver, new NamedWriteableRegistry(Collections.emptyList())); CountDownLatch countDownLatch = new CountDownLatch(1); - TransportEqlSearchAction.operation(planExecutor, task, new EqlSearchRequest().query("foo where blah"), "", "", "node_id", - new ActionListener<>() { + TransportEqlSearchAction.operation(planExecutor, task, new EqlSearchRequest().query("foo where blah"), "", + mock(TransportService.class), mockClusterService, new ActionListener<>() { @Override public void onResponse(EqlSearchResponse eqlSearchResponse) { fail("Shouldn't be here"); @@ -96,10 +101,10 @@ public void testCancellationBeforeSearch() throws InterruptedException { AtomicBoolean cancelled = new AtomicBoolean(false); EqlSearchTask task = mock(EqlSearchTask.class); - String nodeId = randomAlphaOfLength(10); long taskId = randomNonNegativeLong(); when(task.isCancelled()).then(invocationOnMock -> cancelled.get()); when(task.getId()).thenReturn(taskId); + ClusterService mockClusterService = mockClusterService(); String[] indices = new String[]{"endgame"}; @@ -119,7 +124,7 @@ public void testCancellationBeforeSearch() throws InterruptedException { PlanExecutor planExecutor = new PlanExecutor(client, indexResolver, new NamedWriteableRegistry(Collections.emptyList())); CountDownLatch countDownLatch = new CountDownLatch(1); TransportEqlSearchAction.operation(planExecutor, task, new EqlSearchRequest().indices("endgame") - .query("process where foo==3"), "", "", nodeId, new ActionListener<>() { + .query("process where foo==3"), "", mock(TransportService.class), mockClusterService, new ActionListener<>() { @Override public void onResponse(EqlSearchResponse eqlSearchResponse) { fail("Shouldn't be here"); @@ -149,6 +154,7 @@ public void testCancellationDuringSearch() throws InterruptedException { long taskId = randomNonNegativeLong(); when(task.isCancelled()).thenReturn(false); when(task.getId()).thenReturn(taskId); + ClusterService mockClusterService = mockClusterService(nodeId); String[] indices = new String[]{"endgame"}; @@ -183,7 +189,7 @@ public void testCancellationDuringSearch() throws InterruptedException { PlanExecutor planExecutor = new PlanExecutor(client, indexResolver, new NamedWriteableRegistry(Collections.emptyList())); CountDownLatch countDownLatch = new CountDownLatch(1); TransportEqlSearchAction.operation(planExecutor, task, new EqlSearchRequest().indices("endgame") - .query("process where foo==3"), "", "", nodeId, new ActionListener<>() { + .query("process where foo==3"), "", mock(TransportService.class), mockClusterService, new ActionListener<>() { @Override public void onResponse(EqlSearchResponse eqlSearchResponse) { fail("Shouldn't be here"); @@ -207,4 +213,18 @@ public void onFailure(Exception e) { verifyNoMoreInteractions(client, task); } + private ClusterService mockClusterService() { + return mockClusterService(null); + } + + private ClusterService mockClusterService(String nodeId) { + final ClusterService mockClusterService = mock(ClusterService.class); + final DiscoveryNode mockNode = mock(DiscoveryNode.class); + final ClusterName mockClusterName = mock(ClusterName.class); + when(mockNode.getId()).thenReturn(nodeId == null ? randomAlphaOfLength(10) : nodeId); + when(mockClusterService.localNode()).thenReturn(mockNode); + when(mockClusterName.value()).thenReturn(randomAlphaOfLength(10)); + when(mockClusterService.getClusterName()).thenReturn(mockClusterName); + return mockClusterService; + } } diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/search/CriterionOrdinalExtractionTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/search/CriterionOrdinalExtractionTests.java index 84d72b1e62830..248b02e083c07 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/search/CriterionOrdinalExtractionTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/execution/search/CriterionOrdinalExtractionTests.java @@ -31,8 +31,8 @@ public class CriterionOrdinalExtractionTests extends ESTestCase { private String tsField = "timestamp"; private String tbField = "tiebreaker"; - private HitExtractor tsExtractor = new FieldHitExtractor(tsField, tsField, DataTypes.LONG, null, true, null, FAIL_IF_MULTIVALUE); - private HitExtractor tbExtractor = new FieldHitExtractor(tbField, tbField, DataTypes.LONG, null, true, null, FAIL_IF_MULTIVALUE); + private HitExtractor tsExtractor = new FieldHitExtractor(tsField, DataTypes.LONG, null, null, FAIL_IF_MULTIVALUE); + private HitExtractor tbExtractor = new FieldHitExtractor(tbField, DataTypes.LONG, null, null, FAIL_IF_MULTIVALUE); public void testTimeOnly() throws Exception { long time = randomLong(); @@ -57,7 +57,7 @@ public void testTimeAndTiebreakerNull() throws Exception { } public void testTimeNotComparable() throws Exception { - HitExtractor badExtractor = new FieldHitExtractor(tsField, tsField, DataTypes.BINARY, null, true, null, FAIL_IF_MULTIVALUE); + HitExtractor badExtractor = new FieldHitExtractor(tsField, DataTypes.BINARY, null, null, FAIL_IF_MULTIVALUE); SearchHit hit = searchHit(randomAlphaOfLength(10), null); Criterion criterion = new Criterion(0, null, emptyList(), badExtractor, null, false); EqlIllegalArgumentException exception = expectThrows(EqlIllegalArgumentException.class, () -> criterion.ordinal(hit)); diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/SearchableSnapshotActionIT.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/SearchableSnapshotActionIT.java index 7798130057137..a3b657e3cb9fb 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/SearchableSnapshotActionIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/SearchableSnapshotActionIT.java @@ -9,8 +9,6 @@ import org.apache.http.entity.ContentType; import org.apache.http.entity.StringEntity; -import org.apache.http.util.EntityUtils; -import org.elasticsearch.client.Client; import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; import org.elasticsearch.cluster.metadata.DataStream; @@ -22,7 +20,6 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.test.client.NoOpClient; import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.xpack.core.ilm.DeleteAction; import org.elasticsearch.xpack.core.ilm.ForceMergeAction; @@ -30,7 +27,6 @@ import org.elasticsearch.xpack.core.ilm.LifecycleAction; import org.elasticsearch.xpack.core.ilm.LifecyclePolicy; import org.elasticsearch.xpack.core.ilm.LifecycleSettings; -import org.elasticsearch.xpack.core.ilm.MountSnapshotStep; import org.elasticsearch.xpack.core.ilm.Phase; import org.elasticsearch.xpack.core.ilm.PhaseCompleteStep; import org.elasticsearch.xpack.core.ilm.RolloverAction; @@ -38,7 +34,6 @@ import org.elasticsearch.xpack.core.ilm.SetPriorityAction; import org.elasticsearch.xpack.core.ilm.ShrinkAction; import org.elasticsearch.xpack.core.ilm.Step; -import org.elasticsearch.xpack.core.ilm.StepKeyTests; import org.elasticsearch.xpack.core.searchablesnapshots.MountSearchableSnapshotRequest; import org.junit.Before; @@ -163,8 +158,8 @@ public void testSearchableSnapshotForceMergesIndexToOneSegment() throws Exceptio TimeUnit.SECONDS); } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/pull/54433") - public void testDeleteActionDeletesSearchableSnapshot() throws Exception { + @SuppressWarnings("unchecked") + public void testDeleteActionDeletesSearchableSnapshot() throws Exception { createSnapshotRepo(client(), snapshotRepo, randomBoolean()); // create policy with cold and delete phases @@ -192,32 +187,29 @@ public void testDeleteActionDeletesSearchableSnapshot() throws Exception { // rolling over the data stream so we can apply the searchable snapshot policy to a backing index that's not the write index rolloverMaxOneDocCondition(client(), dataStream); - String[] snapshotName = new String[1]; String backingIndexName = DataStream.getDefaultBackingIndexName(dataStream, 1L); String restoredIndexName = SearchableSnapshotAction.FULL_RESTORED_INDEX_PREFIX + backingIndexName; - assertTrue(waitUntil(() -> { - try { - Map explainIndex = explainIndex(client(), backingIndexName); - if (explainIndex == null) { - // in case we missed the original index and it was deleted - explainIndex = explainIndex(client(), restoredIndexName); - } - snapshotName[0] = (String) explainIndex.get("snapshot_name"); - return snapshotName[0] != null; - } catch (IOException e) { - return false; - } - }, 30, TimeUnit.SECONDS)); - assertBusy(() -> assertFalse(indexExists(restoredIndexName))); + + // let's wait for ILM to finish + assertBusy(() -> assertFalse(indexExists(backingIndexName)), 60, TimeUnit.SECONDS); + assertBusy(() -> assertFalse(indexExists(restoredIndexName)), 60, TimeUnit.SECONDS); assertTrue("the snapshot we generate in the cold phase should be deleted by the delete phase", waitUntil(() -> { - try { - Request getSnapshotsRequest = new Request("GET", "_snapshot/" + snapshotRepo + "/" + snapshotName[0]); - Response getSnapshotsResponse = client().performRequest(getSnapshotsRequest); - return EntityUtils.toString(getSnapshotsResponse.getEntity()).contains("snapshot_missing_exception"); - } catch (IOException e) { - return false; - } + try { + Request getSnapshotsRequest = new Request("GET", "_snapshot/" + snapshotRepo + "/_all"); + Response getSnapshotsResponse = client().performRequest(getSnapshotsRequest); + + Map responseMap; + try (InputStream is = getSnapshotsResponse.getEntity().getContent()) { + responseMap = XContentHelper.convertToMap(XContentType.JSON.xContent(), is, true); + } + List responses = (List) responseMap.get("responses"); + Object snapshots = ((Map) responses.get(0)).get("snapshots"); + return ((List>) snapshots).size() == 0; + } catch (Exception e) { + logger.error(e.getMessage(), e); + return false; + } }, 30, TimeUnit.SECONDS)); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/process/NativeStorageProvider.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/process/NativeStorageProvider.java index d5e2edf479060..23307d4452856 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/process/NativeStorageProvider.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/process/NativeStorageProvider.java @@ -134,13 +134,8 @@ public ByteSizeValue getMinLocalStorageAvailable() { return minLocalStorageAvailable; } + // non-static indirection to enable mocking in tests long getUsableSpace(Path path) throws IOException { - long freeSpaceInBytes = Environment.getFileStore(path).getUsableSpace(); - - /* See: https://bugs.openjdk.java.net/browse/JDK-8162520 */ - if (freeSpaceInBytes < 0) { - freeSpaceInBytes = Long.MAX_VALUE; - } - return freeSpaceInBytes; + return Environment.getUsableSpace(path); } } diff --git a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporter.java b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporter.java index 1dddcf5a47849..9a1532f60a6bd 100644 --- a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporter.java +++ b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporter.java @@ -429,9 +429,20 @@ private void setupClusterAlertsTasks(ClusterState clusterState, boolean clusterS if (watches != null && watches.allPrimaryShardsActive() == false) { logger.trace("cannot manage cluster alerts because [.watches] index is not allocated"); } else if ((watches == null || indexExists) && watcherSetup.compareAndSet(false, true)) { + logger.trace("installing monitoring watches"); getClusterAlertsInstallationAsyncActions(indexExists, asyncActions, pendingResponses); + } else { + logger.trace("skipping installing monitoring watches, watches=[{}], indexExists=[{}], watcherSetup=[{}]", + watches, indexExists, watcherSetup.get()); } + } else { + logger.trace("watches shouldn't be setup, because state=[{}] and clusterStateChange=[{}]", state.get(), clusterStateChange); } + } else { + logger.trace("watches can't be used, because xpack.watcher.enabled=[{}] and " + + "xpack.monitoring.exporters._local.cluster_alerts.management.enabled=[{}]", + XPackSettings.WATCHER_ENABLED.get(config.settings()), + CLUSTER_ALERTS_MANAGEMENT_SETTING.getConcreteSettingForNamespace(config.name()).get(config.settings())); } } @@ -577,6 +588,7 @@ private void getClusterAlertsInstallationAsyncActions(final boolean indexExists, new ResponseActionListener<>("watch", uniqueWatchId, pendingResponses))); } } else if (addWatch) { + logger.trace("adding monitoring watch [{}]", uniqueWatchId); asyncActions.add(() -> putWatch(client, watchId, uniqueWatchId, pendingResponses)); } } diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/action/TransportMonitoringMigrateAlertsActionTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/action/TransportMonitoringMigrateAlertsActionTests.java index 1ea0eb2d9038f..fb7b8a62802f9 100644 --- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/action/TransportMonitoringMigrateAlertsActionTests.java +++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/action/TransportMonitoringMigrateAlertsActionTests.java @@ -109,7 +109,6 @@ private void stopMonitoring() { )); } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/66586") @TestLogging( value = "org.elasticsearch.xpack.monitoring.exporter.local:trace", reason = "to ensure we log local exporter on trace level") @@ -147,7 +146,6 @@ public void testLocalAlertsRemoval() throws Exception { } } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/66586") @TestLogging( value = "org.elasticsearch.xpack.monitoring.exporter.local:trace", reason = "to ensure we log local exporter on trace level") diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDocTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDocTests.java index 4f8ec94c126c4..95ce27d6c56f1 100644 --- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDocTests.java +++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDocTests.java @@ -336,8 +336,8 @@ public void testToXContent() throws IOException { clusterName, singletonList(mockNodeResponse), emptyList(), - MappingStats.of(metadata), - AnalysisStats.of(metadata), + MappingStats.of(metadata, () -> {}), + AnalysisStats.of(metadata, () -> {}), VersionStats.of(metadata, singletonList(mockNodeResponse))); final MonitoringDoc.Node node = new MonitoringDoc.Node("_uuid", "_host", "_addr", "_ip", "_name", 1504169190855L); diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/execution/search/QlSourceBuilder.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/execution/search/QlSourceBuilder.java index 327a3151110bf..4f0c1f623b772 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/execution/search/QlSourceBuilder.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/execution/search/QlSourceBuilder.java @@ -6,7 +6,7 @@ */ package org.elasticsearch.xpack.ql.execution.search; -import org.elasticsearch.common.Strings; +import org.elasticsearch.Version; import org.elasticsearch.script.Script; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.fetch.subphase.FieldAndFormat; @@ -22,9 +22,9 @@ * the resulting ES document as a field. */ public class QlSourceBuilder { + public static final Version SWITCH_TO_FIELDS_API_VERSION = Version.V_7_10_0; // The LinkedHashMaps preserve the order of the fields in the response - private final Set sourceFields = new LinkedHashSet<>(); - private final Set docFields = new LinkedHashSet<>(); + private final Set fetchFields = new LinkedHashSet<>(); private final Map scriptFields = new LinkedHashMap<>(); boolean trackScores = false; @@ -40,17 +40,10 @@ public void trackScores() { } /** - * Retrieve the requested field from the {@code _source} of the document + * Retrieve the requested field using the "fields" API */ - public void addSourceField(String field) { - sourceFields.add(field); - } - - /** - * Retrieve the requested field from doc values (or fielddata) of the document - */ - public void addDocField(String field, String format) { - docFields.add(new FieldAndFormat(field, format)); + public void addFetchField(String field, String format) { + fetchFields.add(new FieldAndFormat(field, format)); } /** @@ -66,14 +59,7 @@ public void addScriptField(String name, Script script) { */ public void build(SearchSourceBuilder sourceBuilder) { sourceBuilder.trackScores(this.trackScores); - if (sourceFields.isEmpty() == false) { - sourceBuilder.fetchSource(sourceFields.toArray(Strings.EMPTY_ARRAY), null); - } - docFields.forEach(field -> sourceBuilder.docValueField(field.field, field.format)); + fetchFields.forEach(field -> sourceBuilder.fetchField(new FieldAndFormat(field.field, field.format, null))); scriptFields.forEach(sourceBuilder::scriptField); } - - public boolean noSource() { - return sourceFields.isEmpty(); - } } diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/execution/search/extractor/AbstractFieldHitExtractor.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/execution/search/extractor/AbstractFieldHitExtractor.java index 59a0bc8c2a93e..f1a9df250f5e1 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/execution/search/extractor/AbstractFieldHitExtractor.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/execution/search/extractor/AbstractFieldHitExtractor.java @@ -6,35 +6,23 @@ */ package org.elasticsearch.xpack.ql.execution.search.extractor; -import java.io.IOException; -import java.time.ZoneId; -import java.util.ArrayDeque; -import java.util.ArrayList; -import java.util.Deque; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Objects; -import java.util.StringJoiner; - import org.elasticsearch.Version; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.document.DocumentField; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.index.mapper.IgnoredFieldMapper; -import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType; import org.elasticsearch.search.SearchHit; import org.elasticsearch.xpack.ql.QlIllegalArgumentException; import org.elasticsearch.xpack.ql.type.DataType; import org.elasticsearch.xpack.ql.type.DataTypes; +import java.io.IOException; +import java.time.ZoneId; +import java.util.List; +import java.util.Map; +import java.util.Objects; + import static java.util.Collections.singletonList; -import static org.elasticsearch.xpack.ql.type.DataTypes.DATETIME; -import static org.elasticsearch.xpack.ql.type.DataTypes.DATETIME_NANOS; -import static org.elasticsearch.xpack.ql.type.DataTypes.KEYWORD; -import static org.elasticsearch.xpack.ql.type.DataTypes.SCALED_FLOAT; /** * Extractor for ES fields. Works for both 'normal' fields but also nested ones (which require hitName to be set). @@ -42,7 +30,6 @@ */ public abstract class AbstractFieldHitExtractor implements HitExtractor { - private static final Version SWITCHED_FROM_DOCVALUES_TO_SOURCE_EXTRACTION = Version.V_7_4_0; private static final Version INTRODUCED_MULTI_VALUE_EXTRACTION = Version.V_7_12_0; // TODO: update if merging in 7.13.0 public enum MultiValueHandling { @@ -89,29 +76,23 @@ private static String[] sourcePath(String name, boolean useDocValue, String hitN } private final String fieldName, hitName; - private final String fullFieldName; // used to look at the _ignored section of the query response for the actual full field name private final DataType dataType; private final ZoneId zoneId; - private final boolean useDocValue; private final MultiValueHandling multiValueHandling; - private final String[] path; - protected AbstractFieldHitExtractor(String name, DataType dataType, ZoneId zoneId, boolean useDocValue) { - this(name, null, dataType, zoneId, useDocValue, null, MultiValueHandling.FAIL_IF_MULTIVALUE); + protected AbstractFieldHitExtractor(String name, DataType dataType, ZoneId zoneId) { + this(name, dataType, zoneId, null, MultiValueHandling.FAIL_IF_MULTIVALUE); } - protected AbstractFieldHitExtractor(String name, DataType dataType, ZoneId zoneId, boolean useDocValue, - MultiValueHandling multiValueHandling) { - this(name, null, dataType, zoneId, useDocValue, null, multiValueHandling); + protected AbstractFieldHitExtractor(String name, DataType dataType, ZoneId zoneId, MultiValueHandling multiValueHandling) { + this(name, dataType, zoneId, null, multiValueHandling); } - protected AbstractFieldHitExtractor(String name, String fullFieldName, DataType dataType, ZoneId zoneId, boolean useDocValue, - String hitName, MultiValueHandling multiValueHandling) { + protected AbstractFieldHitExtractor(String name, DataType dataType, ZoneId zoneId, String hitName, + MultiValueHandling multiValueHandling) { this.fieldName = name; - this.fullFieldName = fullFieldName; this.dataType = dataType; this.zoneId = zoneId; - this.useDocValue = useDocValue; this.multiValueHandling = multiValueHandling; this.hitName = hitName; @@ -120,27 +101,18 @@ protected AbstractFieldHitExtractor(String name, String fullFieldName, DataType throw new QlIllegalArgumentException("Hitname [{}] specified but not part of the name [{}]", hitName, name); } } - - this.path = sourcePath(fieldName, useDocValue, hitName); } protected AbstractFieldHitExtractor(StreamInput in) throws IOException { fieldName = in.readString(); - if (in.getVersion().onOrAfter(SWITCHED_FROM_DOCVALUES_TO_SOURCE_EXTRACTION)) { - fullFieldName = in.readOptionalString(); - } else { - fullFieldName = null; - } String typeName = in.readOptionalString(); dataType = typeName != null ? loadTypeFromName(typeName) : null; - useDocValue = in.readBoolean(); hitName = in.readOptionalString(); if (in.getVersion().onOrAfter(INTRODUCED_MULTI_VALUE_EXTRACTION)) { multiValueHandling = in.readEnum(MultiValueHandling.class); } else { multiValueHandling = in.readBoolean() ? MultiValueHandling.EXTRACT_ONE : MultiValueHandling.FAIL_IF_MULTIVALUE; } - path = sourcePath(fieldName, useDocValue, hitName); zoneId = readZoneId(in); } @@ -153,11 +125,7 @@ protected DataType loadTypeFromName(String typeName) { @Override public void writeTo(StreamOutput out) throws IOException { out.writeString(fieldName); - if (out.getVersion().onOrAfter(SWITCHED_FROM_DOCVALUES_TO_SOURCE_EXTRACTION)) { - out.writeOptionalString(fullFieldName); - } out.writeOptionalString(dataType == null ? null : dataType.typeName()); - out.writeBoolean(useDocValue); out.writeOptionalString(hitName); if (out.getVersion().onOrAfter(INTRODUCED_MULTI_VALUE_EXTRACTION)) { out.writeEnum(multiValueHandling); @@ -169,56 +137,39 @@ public void writeTo(StreamOutput out) throws IOException { @Override public Object extract(SearchHit hit) { Object value = null; - if (useDocValue) { - DocumentField field = hit.field(fieldName); - if (field != null) { - value = extractMultiValue(field.getValues()); - } + DocumentField field = null; + if (hitName != null) { + // a nested field value is grouped under the nested parent name (ie dep.dep_name lives under "dep":[{dep_name:value}]) + field = hit.field(hitName); } else { - // if the field was ignored because it was malformed and ignore_malformed was turned on - if (fullFieldName != null - && hit.getFields().containsKey(IgnoredFieldMapper.NAME) - && isFromDocValuesOnly(dataType) == false) { - /* - * We check here the presence of the field name (fullFieldName including the parent name) in the list - * of _ignored fields (due to malformed data, which was ignored). - * For example, in the case of a malformed number, a "byte" field with "ignore_malformed: true" - * with a "text" sub-field should return "null" for the "byte" parent field and the actual malformed - * data for the "text" sub-field. - */ - if (hit.getFields().get(IgnoredFieldMapper.NAME).getValues().contains(fullFieldName)) { - return null; - } - } - Map source = hit.getSourceAsMap(); - if (source != null) { - value = extractFromSource(source); - } + field = hit.field(fieldName); + } + if (field != null) { + value = unwrapFieldsMultiValue(field.getValues()); } return value; } - protected Object extractMultiValue(Object values) { - return multiValueHandling.handle(unwrapMultiValue(values), fieldName); - } - - protected Object unwrapMultiValue(Object values) { + protected Object unwrapFieldsMultiValue(Object values) { if (values == null) { return null; } + if (values instanceof Map && hitName != null) { + // extract the sub-field from a nested field (dep.dep_name -> dep_name) + return unwrapFieldsMultiValue(((Map) values).get(fieldName.substring(hitName.length() + 1))); + } if (values instanceof List) { List list = (List) values; - if (isPrimitive(list) == false) { - List unwrappedList = new ArrayList<>(); - for (Object o : list) { - Object unwrapped = unwrapMultiValue(o); - if (unwrapped instanceof List) { - unwrappedList.addAll((List) unwrapped); + if (list.isEmpty()) { + return null; + } else { + if (isPrimitive(list) == false) { + if (list.size() == 1 || multiValueHandling == MultiValueHandling.EXTRACT_ONE) { + return unwrapFieldsMultiValue(list.get(0)); } else { - unwrappedList.add(unwrapped); + throw new QlIllegalArgumentException("Arrays (returned by [{}]) are not supported", fieldName); } } - return unwrappedList; } } @@ -227,132 +178,13 @@ protected Object unwrapMultiValue(Object values) { return unwrapped; } - // The Jackson json parser can generate for numerics - Integers, Longs, BigIntegers (if Long is not enough) - // and BigDecimal (if Double is not enough) - if (values instanceof Number || values instanceof String || values instanceof Boolean) { - return unwrapNumberOrStringOrBoolean(values); - } - - throw new QlIllegalArgumentException("Type {} (returned by [{}]) is not supported", values.getClass().getSimpleName(), fieldName); - } - - private Object unwrapNumberOrStringOrBoolean(Object value) { - if (dataType == null) { - return value; - } - if (dataType.isNumeric() && isFromDocValuesOnly(dataType) == false) { - if (dataType == DataTypes.DOUBLE || dataType == DataTypes.FLOAT || dataType == DataTypes.HALF_FLOAT) { - Number result = null; - try { - result = numberType(dataType).parse(value, true); - } catch(IllegalArgumentException iae) { - return null; - } - // docvalue_fields is always returning a Double value even if the underlying floating point data type is not Double - // even if we don't extract from docvalue_fields anymore, the behavior should be consistent - return result.doubleValue(); - } else { - Number result = null; - try { - result = numberType(dataType).parse(value, true); - } catch(IllegalArgumentException iae) { - return null; - } - return result; - } - } else if (DataTypes.isString(dataType) || dataType == DataTypes.IP) { - return value.toString(); - } else { - return value; - } - } - - protected boolean isFromDocValuesOnly(DataType dataType) { - return dataType == KEYWORD // because of ignore_above. - || dataType == DATETIME - || dataType == DATETIME_NANOS - || dataType == SCALED_FLOAT; // because of scaling_factor - } - - private static NumberType numberType(DataType dataType) { - return NumberType.valueOf(dataType.esType().toUpperCase(Locale.ROOT)); + return values; } protected abstract Object unwrapCustomValue(Object values); protected abstract boolean isPrimitive(List list); - @SuppressWarnings({ "unchecked", "rawtypes" }) - public Object extractFromSource(Map map) { - List values = new ArrayList<>(); - - // Used to avoid recursive method calls - // Holds the sub-maps in the document hierarchy that are pending to be inspected along with the current index of the `path`. - Deque>> queue = new ArrayDeque<>(); - queue.add(new Tuple<>(-1, map)); - - while (queue.isEmpty() == false) { - Tuple> tuple = queue.removeFirst(); - int idx = tuple.v1(); - Map subMap = tuple.v2(); - - // Find all possible entries by examining all combinations under the current level ("idx") of the "path" - // e.g.: If the path == "a.b.c.d" and the idx == 0, we need to check the current subMap against the keys: - // "b", "b.c" and "b.c.d" - StringJoiner sj = new StringJoiner("."); - for (int i = idx + 1; i < path.length; i++) { - sj.add(path[i]); - String currentPath = sj.toString(); - // need to differentiate between Map#get() returning null b/c there's no mapping VS the mapping's actual value - if (subMap.containsKey(currentPath)) { - Object node = subMap.get(currentPath); - - if (node instanceof List) { // {"a": {"b": [...]}} - List listOfValues = (List) node; - // if the path is not yet exhausted (ex. at "a.b.c" level for a "a.b.c.d" path), queue whatever's in the list and - // inspect it with next outer loop iteration. - if (i < path.length - 1) { - final int level = i; - listOfValues.forEach(o -> { - if (o instanceof Map) { - queue.add(new Tuple<>(level, (Map) o)); - } else { - // another list or an "end"/concrete value: smth is wrong, either with the mapping or with the map/doc - throw new QlIllegalArgumentException("Cannot extract field [{}] value [{}] from source", fieldName, - node); - } - }); - } - // if the path is exhausted, just add the list to the output list and let extractMultiValue & co deal with it - else { - values.add(node); - } - } else if (node instanceof Map) { - if (i < path.length - 1) { - // Add the sub-map to the queue along with the current path index - queue.add(new Tuple<>(i, (Map) node)); - } else { - // We exhausted the path and got a map - // If it is an object - it will be handled in the value extractor - values.add(node); - } - } else { - if (i < path.length - 1) { - if (node != null) { - // If we reach a concrete value without exhausting the full path, something is wrong with the mapping - // e.g.: map is {"a" : { "b" : "value }} and we are looking for a path: "a.b.c.d" - throw new QlIllegalArgumentException("Cannot extract value [{}] from source", fieldName); - } - } else { - values.add(node); - } - } - } - } - } - return extractMultiValue(values); - } - @Override public String hitName() { return hitName; @@ -362,10 +194,6 @@ public String fieldName() { return fieldName; } - public String fullFieldName() { - return fullFieldName; - } - public ZoneId zoneId() { return zoneId; } @@ -374,10 +202,6 @@ public DataType dataType() { return dataType; } - public boolean useDocValues() { - return useDocValue; - } - public MultiValueHandling multiValueExtraction() { return multiValueHandling; } @@ -395,12 +219,11 @@ public boolean equals(Object obj) { AbstractFieldHitExtractor other = (AbstractFieldHitExtractor) obj; return fieldName.equals(other.fieldName) && hitName.equals(other.hitName) - && useDocValue == other.useDocValue && multiValueHandling == other.multiValueHandling; } @Override public int hashCode() { - return Objects.hash(fieldName, useDocValue, hitName, multiValueHandling); + return Objects.hash(fieldName, hitName, multiValueHandling); } } diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/plugin/TransportActionUtils.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/plugin/TransportActionUtils.java new file mode 100644 index 0000000000000..f839ba54c55ef --- /dev/null +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/plugin/TransportActionUtils.java @@ -0,0 +1,71 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.ql.plugin; + +import org.apache.logging.log4j.Logger; +import org.elasticsearch.action.search.SearchPhaseExecutionException; +import org.elasticsearch.action.search.VersionMismatchException; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.xpack.ql.util.Holder; + +import java.util.function.Consumer; + +public final class TransportActionUtils { + + /** + * Execute a *QL request and re-try it in case the first request failed with a {@code VersionMismatchException} + * + * @param clusterService The cluster service instance + * @param onFailure On-failure handler in case the request doesn't fail with a {@code VersionMismatchException} + * @param queryRunner *QL query execution code, typically a Plan Executor running the query + * @param retryRequest Re-trial logic + * @param log Log4j logger + */ + public static void executeRequestWithRetryAttempt(ClusterService clusterService, Consumer onFailure, + Consumer> queryRunner, Consumer retryRequest, Logger log) { + + Holder retrySecondTime = new Holder(false); + queryRunner.accept(e -> { + // the search request likely ran on nodes with different versions of ES + // we will retry on a node with an older version that should generate a backwards compatible _search request + if (e instanceof SearchPhaseExecutionException + && ((SearchPhaseExecutionException) e).getCause() instanceof VersionMismatchException) { + if (log.isDebugEnabled()) { + log.debug("Caught exception type [{}] with cause [{}].", e.getClass().getName(), e.getCause()); + } + DiscoveryNode localNode = clusterService.state().nodes().getLocalNode(); + DiscoveryNode candidateNode = null; + for (DiscoveryNode node : clusterService.state().nodes()) { + // find the first node that's older than the current node + if (node != localNode && node.getVersion().before(localNode.getVersion())) { + candidateNode = node; + break; + } + } + if (candidateNode != null) { + if (log.isDebugEnabled()) { + log.debug("Candidate node to resend the request to: address [{}], id [{}], name [{}], version [{}]", + candidateNode.getAddress(), candidateNode.getId(), candidateNode.getName(), candidateNode.getVersion()); + } + // re-send the request to the older node + retryRequest.accept(candidateNode); + } else { + retrySecondTime.set(true); + } + } else { + onFailure.accept(e); + } + }); + if (retrySecondTime.get()) { + if (log.isDebugEnabled()) { + log.debug("No candidate node found, likely all were upgraded in the meantime. Re-trying the original request."); + } + queryRunner.accept(onFailure); + } + } +} diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/querydsl/query/NestedQuery.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/querydsl/query/NestedQuery.java index 634f009087284..2b17a30a90efe 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/querydsl/query/NestedQuery.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/querydsl/query/NestedQuery.java @@ -16,7 +16,6 @@ import org.elasticsearch.xpack.ql.tree.Source; import java.util.AbstractMap; -import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -116,26 +115,16 @@ public QueryBuilder asBuilder() { ihb.setSize(MAX_INNER_HITS); ihb.setName(path + "_" + COUNTER++); - boolean noSourceNeeded = true; - List sourceFields = new ArrayList<>(); - for (Map.Entry> entry : fields.entrySet()) { if (entry.getValue().getKey()) { - ihb.addDocValueField(entry.getKey(), entry.getValue().getValue()); + ihb.addFetchField(entry.getKey(), entry.getValue().getValue()); } else { - sourceFields.add(entry.getKey()); - noSourceNeeded = false; + ihb.addFetchField(entry.getKey()); } } - - if (noSourceNeeded) { - ihb.setFetchSourceContext(FetchSourceContext.DO_NOT_FETCH_SOURCE); - ihb.setStoredFieldNames(NO_STORED_FIELD); - } - else { - ihb.setFetchSourceContext(new FetchSourceContext(true, sourceFields.toArray(new String[sourceFields.size()]), null)); - } + ihb.setFetchSourceContext(FetchSourceContext.DO_NOT_FETCH_SOURCE); + ihb.setStoredFieldNames(NO_STORED_FIELD); query.innerHit(ihb); } diff --git a/x-pack/plugin/ql/test/src/main/java/org/elasticsearch/xpack/ql/TestNode.java b/x-pack/plugin/ql/test/src/main/java/org/elasticsearch/xpack/ql/TestNode.java new file mode 100644 index 0000000000000..746ff91dc902e --- /dev/null +++ b/x-pack/plugin/ql/test/src/main/java/org/elasticsearch/xpack/ql/TestNode.java @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.ql; + +import org.apache.http.HttpHost; +import org.elasticsearch.Version; + +public final class TestNode { + + private final String id; + private final Version version; + private final HttpHost publishAddress; + + public TestNode(String id, Version version, HttpHost publishAddress) { + this.id = id; + this.version = version; + this.publishAddress = publishAddress; + } + + public String getId() { + return id; + } + + public HttpHost getPublishAddress() { + return publishAddress; + } + + public Version getVersion() { + return version; + } + + @Override + public String toString() { + return "Node{" + "id='" + id + '\'' + ", version=" + version + '}'; + } +} diff --git a/x-pack/plugin/ql/test/src/main/java/org/elasticsearch/xpack/ql/TestNodes.java b/x-pack/plugin/ql/test/src/main/java/org/elasticsearch/xpack/ql/TestNodes.java new file mode 100644 index 0000000000000..9681e66183530 --- /dev/null +++ b/x-pack/plugin/ql/test/src/main/java/org/elasticsearch/xpack/ql/TestNodes.java @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.ql; + +import org.elasticsearch.Version; + +import java.util.HashMap; +import java.util.List; +import java.util.stream.Collectors; + +public final class TestNodes extends HashMap { + + public void add(TestNode node) { + put(node.getId(), node); + } + + public List getNewNodes() { + Version bwcVersion = getBWCVersion(); + return values().stream().filter(n -> n.getVersion().after(bwcVersion)).collect(Collectors.toList()); + } + + public List getBWCNodes() { + Version bwcVersion = getBWCVersion(); + return values().stream().filter(n -> n.getVersion().equals(bwcVersion)).collect(Collectors.toList()); + } + + public Version getBWCVersion() { + if (isEmpty()) { + throw new IllegalStateException("no nodes available"); + } + return Version.fromId(values().stream().map(node -> node.getVersion().id).min(Integer::compareTo).get()); + } + + @Override + public String toString() { + return "Nodes{" + values().stream().map(TestNode::toString).collect(Collectors.joining("\n")) + '}'; + } +} diff --git a/x-pack/plugin/ql/test/src/main/java/org/elasticsearch/xpack/ql/TestUtils.java b/x-pack/plugin/ql/test/src/main/java/org/elasticsearch/xpack/ql/TestUtils.java index 84eb576d2dd31..bc933c8b1fe9d 100644 --- a/x-pack/plugin/ql/test/src/main/java/org/elasticsearch/xpack/ql/TestUtils.java +++ b/x-pack/plugin/ql/test/src/main/java/org/elasticsearch/xpack/ql/TestUtils.java @@ -7,6 +7,8 @@ package org.elasticsearch.xpack.ql; +import org.apache.http.HttpHost; +import org.elasticsearch.Version; import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; import org.elasticsearch.client.RestClient; @@ -16,6 +18,7 @@ import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.elasticsearch.test.rest.yaml.ObjectPath; import org.elasticsearch.xpack.ql.expression.Expression; import org.elasticsearch.xpack.ql.expression.FieldAttribute; import org.elasticsearch.xpack.ql.expression.Literal; @@ -265,4 +268,36 @@ public static Tuple pathAndName(String string) { } return new Tuple<>(folder, file); } + + public static TestNodes buildNodeAndVersions(RestClient client) throws IOException { + Response response = client.performRequest(new Request("GET", "_nodes")); + ObjectPath objectPath = ObjectPath.createFromResponse(response); + Map nodesAsMap = objectPath.evaluate("nodes"); + TestNodes nodes = new TestNodes(); + for (String id : nodesAsMap.keySet()) { + nodes.add( + new TestNode( + id, + Version.fromString(objectPath.evaluate("nodes." + id + ".version")), + HttpHost.create(objectPath.evaluate("nodes." + id + ".http.publish_address")) + ) + ); + } + return nodes; + } + + public static String readResource(InputStream input) throws IOException { + StringBuilder builder = new StringBuilder(); + try (BufferedReader reader = new BufferedReader(new InputStreamReader(input, StandardCharsets.UTF_8))) { + String line = reader.readLine(); + while (line != null) { + if (line.trim().startsWith("//") == false) { + builder.append(line); + builder.append('\n'); + } + line = reader.readLine(); + } + return builder.toString(); + } + } } diff --git a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/BaseSearchableSnapshotsIntegTestCase.java b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/BaseSearchableSnapshotsIntegTestCase.java index 62efbc0ac72f7..8bab23714d365 100644 --- a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/BaseSearchableSnapshotsIntegTestCase.java +++ b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/BaseSearchableSnapshotsIntegTestCase.java @@ -23,6 +23,7 @@ import org.elasticsearch.index.IndexSettings; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.snapshots.AbstractSnapshotIntegTestCase; +import org.elasticsearch.snapshots.SnapshotsService; import org.elasticsearch.xpack.core.searchablesnapshots.MountSearchableSnapshotAction; import org.elasticsearch.xpack.core.searchablesnapshots.MountSearchableSnapshotRequest; import org.elasticsearch.xpack.searchablesnapshots.cache.CacheService; @@ -72,7 +73,7 @@ protected Settings nodeSettings(int nodeOrdinal) { ); } builder.put( - FrozenCacheService.SNAPSHOT_CACHE_SIZE_SETTING.getKey(), + SnapshotsService.SNAPSHOT_CACHE_SIZE_SETTING.getKey(), rarely() ? randomBoolean() ? new ByteSizeValue(randomIntBetween(0, 10), ByteSizeUnit.KB) @@ -80,14 +81,14 @@ protected Settings nodeSettings(int nodeOrdinal) { : new ByteSizeValue(randomIntBetween(1, 10), ByteSizeUnit.MB) ); builder.put( - FrozenCacheService.SNAPSHOT_CACHE_REGION_SIZE_SETTING.getKey(), + SnapshotsService.SNAPSHOT_CACHE_REGION_SIZE_SETTING.getKey(), rarely() ? new ByteSizeValue(randomIntBetween(4, 1024), ByteSizeUnit.KB) : new ByteSizeValue(randomIntBetween(1, 10), ByteSizeUnit.MB) ); if (randomBoolean()) { builder.put( - FrozenCacheService.FROZEN_CACHE_RANGE_SIZE_SETTING.getKey(), + SnapshotsService.SHARED_CACHE_RANGE_SIZE_SETTING.getKey(), rarely() ? new ByteSizeValue(randomIntBetween(4, 1024), ByteSizeUnit.KB) : new ByteSizeValue(randomIntBetween(1, 10), ByteSizeUnit.MB) diff --git a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsIntegTests.java b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsIntegTests.java index e81ad34dd6e5b..9af516afdaf6f 100644 --- a/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsIntegTests.java +++ b/x-pack/plugin/searchable-snapshots/src/internalClusterTest/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshotsIntegTests.java @@ -9,6 +9,7 @@ import com.carrotsearch.hppc.cursors.ObjectCursor; import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.search.TotalHits; +import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse; import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotIndexShardStatus; @@ -96,7 +97,9 @@ import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshots.getDataTiersPreference; import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshotsConstants.SNAPSHOT_DIRECTORY_FACTORY_KEY; import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshotsConstants.SNAPSHOT_RECOVERY_STATE_FACTORY_KEY; +import static org.hamcrest.Matchers.allOf; import static org.hamcrest.Matchers.anyOf; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.greaterThanOrEqualTo; @@ -1288,6 +1291,34 @@ public void testSnapshotOfSearchableSnapshotIncludesNoDataButCanBeRestored() thr logger.info("--> finished restoring snapshot-2"); assertTotalHits(restoredIndexName, originalAllHits, originalBarHits); + + final IllegalArgumentException remountException = expectThrows(IllegalArgumentException.class, () -> { + try { + mountSnapshot( + restoreRepositoryName, + snapshotTwo.getName(), + restoredIndexName, + randomAlphaOfLength(10).toLowerCase(Locale.ROOT), + Settings.EMPTY + ); + } catch (Exception e) { + final Throwable cause = ExceptionsHelper.unwrap(e, IllegalArgumentException.class); + throw cause == null ? e : cause; + } + }); + assertThat( + remountException.getMessage(), + allOf( + containsString("is a snapshot of a searchable snapshot index backed by index"), + containsString(repositoryName), + containsString(snapshotOne.getName()), + containsString(indexName), + containsString(restoreRepositoryName), + containsString(snapshotTwo.getName()), + containsString(restoredIndexName), + containsString("cannot be mounted; did you mean to restore it instead?") + ) + ); } private void assertTotalHits(String indexName, TotalHits originalAllHits, TotalHits originalBarHits) throws Exception { diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/index/store/SearchableSnapshotDirectory.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/index/store/SearchableSnapshotDirectory.java index 8f3ab7aa7cf4f..8a3e52f7b21df 100644 --- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/index/store/SearchableSnapshotDirectory.java +++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/index/store/SearchableSnapshotDirectory.java @@ -495,7 +495,7 @@ private void prewarmCache(ActionListener listener) { final int numberOfParts = file.numberOfParts(); final StepListener> fileCompletionListener = new StepListener<>(); fileCompletionListener.whenComplete(voids -> input.close(), e -> IOUtils.closeWhileHandlingException(input)); - fileCompletionListener.whenComplete(voids -> completionListener.onResponse(null), completionListener::onFailure); + fileCompletionListener.addListener(completionListener.map(voids -> null)); final GroupedActionListener partsListener = new GroupedActionListener<>(fileCompletionListener, numberOfParts); diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshots.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshots.java index d3ad5807c7b23..1b76d6b3312b8 100644 --- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshots.java +++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshots.java @@ -57,6 +57,7 @@ import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestHandler; import org.elasticsearch.script.ScriptService; +import org.elasticsearch.snapshots.SnapshotsService; import org.elasticsearch.snapshots.SourceOnlySnapshotRepository; import org.elasticsearch.threadpool.ExecutorBuilder; import org.elasticsearch.threadpool.ScalingExecutorBuilder; @@ -250,9 +251,9 @@ public List> getSettings() { CacheService.SNAPSHOT_CACHE_MAX_FILES_TO_SYNC_AT_ONCE_SETTING, CacheService.SNAPSHOT_CACHE_SYNC_SHUTDOWN_TIMEOUT, SearchableSnapshotEnableAllocationDecider.SEARCHABLE_SNAPSHOTS_ALLOCATE_ON_ROLLING_RESTART, - FrozenCacheService.SNAPSHOT_CACHE_SIZE_SETTING, - FrozenCacheService.SNAPSHOT_CACHE_REGION_SIZE_SETTING, - FrozenCacheService.FROZEN_CACHE_RANGE_SIZE_SETTING, + SnapshotsService.SNAPSHOT_CACHE_SIZE_SETTING, + SnapshotsService.SNAPSHOT_CACHE_REGION_SIZE_SETTING, + SnapshotsService.SHARED_CACHE_RANGE_SIZE_SETTING, FrozenCacheService.FROZEN_CACHE_RECOVERY_RANGE_SIZE_SETTING, FrozenCacheService.SNAPSHOT_CACHE_MAX_FREQ_SETTING, FrozenCacheService.SNAPSHOT_CACHE_DECAY_INTERVAL_SETTING, diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/TransportMountSearchableSnapshotAction.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/TransportMountSearchableSnapshotAction.java index 3fa05b4b89c07..7f31b53466125 100644 --- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/TransportMountSearchableSnapshotAction.java +++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/TransportMountSearchableSnapshotAction.java @@ -43,6 +43,7 @@ import org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshotsConstants; import java.util.Arrays; +import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.Optional; @@ -50,6 +51,7 @@ import static org.elasticsearch.index.IndexModule.INDEX_RECOVERY_TYPE_SETTING; import static org.elasticsearch.index.IndexModule.INDEX_STORE_TYPE_SETTING; import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshots.getDataTiersPreference; +import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshotsConstants.isSearchableSnapshotStore; /** * Action that mounts a snapshot as a searchable snapshot, by converting the mount request into a restore request with specific settings @@ -179,6 +181,25 @@ protected void masterOperation( final String[] ignoreIndexSettings = Arrays.copyOf(request.ignoreIndexSettings(), request.ignoreIndexSettings().length + 1); ignoreIndexSettings[ignoreIndexSettings.length - 1] = IndexMetadata.SETTING_DATA_PATH; + final IndexMetadata indexMetadata = repository.getSnapshotIndexMetaData(repoData, snapshotId, indexId); + if (isSearchableSnapshotStore(indexMetadata.getSettings())) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "index [%s] in snapshot [%s/%s:%s] is a snapshot of a searchable snapshot index " + + "backed by index [%s] in snapshot [%s/%s:%s] and cannot be mounted; did you mean to restore it instead?", + indexName, + repoName, + repository.getMetadata().uuid(), + snapName, + SearchableSnapshots.SNAPSHOT_INDEX_NAME_SETTING.get(indexMetadata.getSettings()), + SearchableSnapshots.SNAPSHOT_REPOSITORY_NAME_SETTING.get(indexMetadata.getSettings()), + SearchableSnapshots.SNAPSHOT_REPOSITORY_UUID_SETTING.get(indexMetadata.getSettings()), + SearchableSnapshots.SNAPSHOT_SNAPSHOT_NAME_SETTING.get(indexMetadata.getSettings()) + ) + ); + } + client.admin() .cluster() .restoreSnapshot( diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/FrozenCacheService.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/FrozenCacheService.java index f9c498545a5e9..94d192cb11973 100644 --- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/FrozenCacheService.java +++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/FrozenCacheService.java @@ -46,35 +46,19 @@ import java.util.function.LongSupplier; import java.util.function.Predicate; +import static org.elasticsearch.snapshots.SnapshotsService.SHARED_CACHE_RANGE_SIZE_SETTING; +import static org.elasticsearch.snapshots.SnapshotsService.SHARED_CACHE_SETTINGS_PREFIX; +import static org.elasticsearch.snapshots.SnapshotsService.SNAPSHOT_CACHE_REGION_SIZE_SETTING; +import static org.elasticsearch.snapshots.SnapshotsService.SNAPSHOT_CACHE_SIZE_SETTING; import static org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshotsUtils.toIntBytes; public class FrozenCacheService implements Releasable { - private static final String SETTINGS_PREFIX = "xpack.searchable.snapshot.shared_cache."; - - public static final Setting SNAPSHOT_CACHE_SIZE_SETTING = Setting.byteSizeSetting( - SETTINGS_PREFIX + "size", - ByteSizeValue.ZERO, - Setting.Property.NodeScope - ); - public static final ByteSizeValue MIN_SNAPSHOT_CACHE_RANGE_SIZE = new ByteSizeValue(4, ByteSizeUnit.KB); public static final ByteSizeValue MAX_SNAPSHOT_CACHE_RANGE_SIZE = new ByteSizeValue(Integer.MAX_VALUE, ByteSizeUnit.BYTES); - public static final Setting FROZEN_CACHE_RANGE_SIZE_SETTING = Setting.byteSizeSetting( - SETTINGS_PREFIX + "range_size", - ByteSizeValue.ofMb(16), // default - Setting.Property.NodeScope - ); - - public static final Setting SNAPSHOT_CACHE_REGION_SIZE_SETTING = Setting.byteSizeSetting( - SETTINGS_PREFIX + "region_size", - FROZEN_CACHE_RANGE_SIZE_SETTING, - Setting.Property.NodeScope - ); - public static final Setting FROZEN_CACHE_RECOVERY_RANGE_SIZE_SETTING = Setting.byteSizeSetting( - SETTINGS_PREFIX + "recovery_range_size", + SHARED_CACHE_SETTINGS_PREFIX + "recovery_range_size", new ByteSizeValue(128, ByteSizeUnit.KB), // default MIN_SNAPSHOT_CACHE_RANGE_SIZE, // min MAX_SNAPSHOT_CACHE_RANGE_SIZE, // max @@ -83,7 +67,7 @@ public class FrozenCacheService implements Releasable { public static final TimeValue MIN_SNAPSHOT_CACHE_DECAY_INTERVAL = TimeValue.timeValueSeconds(1L); public static final Setting SNAPSHOT_CACHE_DECAY_INTERVAL_SETTING = Setting.timeSetting( - SETTINGS_PREFIX + "decay.interval", + SHARED_CACHE_SETTINGS_PREFIX + "decay.interval", TimeValue.timeValueSeconds(60L), // default MIN_SNAPSHOT_CACHE_DECAY_INTERVAL, // min Setting.Property.NodeScope, @@ -91,14 +75,14 @@ public class FrozenCacheService implements Releasable { ); public static final Setting SNAPSHOT_CACHE_MAX_FREQ_SETTING = Setting.intSetting( - SETTINGS_PREFIX + "max_freq", + SHARED_CACHE_SETTINGS_PREFIX + "max_freq", 100, // default 1, // min Setting.Property.NodeScope ); public static final Setting SNAPSHOT_CACHE_MIN_TIME_DELTA_SETTING = Setting.timeSetting( - SETTINGS_PREFIX + "min_time_delta", + SHARED_CACHE_SETTINGS_PREFIX + "min_time_delta", TimeValue.timeValueSeconds(60L), // default TimeValue.timeValueSeconds(0L), // min Setting.Property.NodeScope @@ -157,7 +141,7 @@ public FrozenCacheService(Environment environment, ThreadPool threadPool) { } decayTask = new CacheDecayTask(threadPool, SNAPSHOT_CACHE_DECAY_INTERVAL_SETTING.get(settings)); decayTask.rescheduleIfNecessary(); - this.rangeSize = FROZEN_CACHE_RANGE_SIZE_SETTING.get(settings); + this.rangeSize = SHARED_CACHE_RANGE_SIZE_SETTING.get(settings); this.recoveryRangeSize = FROZEN_CACHE_RECOVERY_RANGE_SIZE_SETTING.get(settings); } diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/SharedBytes.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/SharedBytes.java index 82bcb1082c3a4..4dfd9ba195b85 100644 --- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/SharedBytes.java +++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/SharedBytes.java @@ -15,6 +15,8 @@ import org.elasticsearch.common.util.concurrent.ConcurrentCollections; import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.env.Environment; +import org.elasticsearch.snapshots.SnapshotUtils; +import org.elasticsearch.snapshots.SnapshotsService; import java.io.IOException; import java.nio.ByteBuffer; @@ -33,8 +35,6 @@ public class SharedBytes extends AbstractRefCounted { StandardOpenOption.WRITE, StandardOpenOption.CREATE }; - private static final String CACHE_FILE_NAME = "snap_cache"; - final int numRegions; final long regionSize; @@ -51,29 +51,21 @@ public class SharedBytes extends AbstractRefCounted { final long fileSize = numRegions * regionSize; Path cacheFile = null; if (fileSize > 0) { - for (Path path : environment.dataFiles()) { - // TODO: be resilient to this check failing and try next path? - long usableSpace = getUsableSpace(path); - Path p = path.resolve(CACHE_FILE_NAME); - if (Files.exists(p)) { - usableSpace += Files.size(p); - } - // TODO: leave some margin for error here - if (usableSpace > fileSize) { - cacheFile = p; - break; - } - } + cacheFile = SnapshotUtils.findCacheSnapshotCacheFilePath(environment, fileSize); if (cacheFile == null) { throw new IOException("Could not find a directory with adequate free space for cache file"); } // TODO: maybe make this faster by allocating a larger direct buffer if this is too slow for very large files // We fill either the full file or the bytes between its current size and the desired size once with zeros to fully allocate // the file up front - logger.info("creating shared snapshot cache file [size={}, path={}]", fileSize, cacheFile); final ByteBuffer fillBytes = ByteBuffer.allocate(Channels.WRITE_CHUNK_SIZE); this.fileChannel = FileChannel.open(cacheFile, OPEN_OPTIONS); long written = fileChannel.size(); + if (fileSize < written) { + logger.info("creating shared snapshot cache file [size={}, path={}]", fileSize, cacheFile); + } else if (fileSize == written) { + logger.debug("reusing existing shared snapshot cache file [size={}, path={}]", fileSize, cacheFile); + } fileChannel.position(written); while (written < fileSize) { final int toWrite = Math.toIntExact(Math.min(fileSize - written, Channels.WRITE_CHUNK_SIZE)); @@ -87,23 +79,12 @@ public class SharedBytes extends AbstractRefCounted { } else { this.fileChannel = null; for (Path path : environment.dataFiles()) { - Files.deleteIfExists(path.resolve(CACHE_FILE_NAME)); + Files.deleteIfExists(path.resolve(SnapshotsService.CACHE_FILE_NAME)); } } this.path = cacheFile; } - // TODO: dry up against MLs usage of the same method - private static long getUsableSpace(Path path) throws IOException { - long freeSpaceInBytes = Environment.getFileStore(path).getUsableSpace(); - - /* See: https://bugs.openjdk.java.net/browse/JDK-8162520 */ - if (freeSpaceInBytes < 0) { - freeSpaceInBytes = Long.MAX_VALUE; - } - return freeSpaceInBytes; - } - @Override protected void closeInternal() { try { diff --git a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/index/store/cache/FrozenIndexInputTests.java b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/index/store/cache/FrozenIndexInputTests.java index 6e55d51e920ae..d943a83c8664f 100644 --- a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/index/store/cache/FrozenIndexInputTests.java +++ b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/index/store/cache/FrozenIndexInputTests.java @@ -23,6 +23,7 @@ import org.elasticsearch.index.store.StoreFileMetadata; import org.elasticsearch.repositories.IndexId; import org.elasticsearch.snapshots.SnapshotId; +import org.elasticsearch.snapshots.SnapshotsService; import org.elasticsearch.xpack.searchablesnapshots.AbstractSearchableSnapshotsTestCase; import org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshots; import org.elasticsearch.xpack.searchablesnapshots.cache.CacheService; @@ -55,7 +56,7 @@ public void testRandomReads() throws IOException { final ByteSizeValue rangeSize; if (rarely()) { - rangeSize = FrozenCacheService.FROZEN_CACHE_RANGE_SIZE_SETTING.get(Settings.EMPTY); + rangeSize = SnapshotsService.SHARED_CACHE_RANGE_SIZE_SETTING.get(Settings.EMPTY); } else if (randomBoolean()) { rangeSize = new ByteSizeValue( randomLongBetween(CacheService.MIN_SNAPSHOT_CACHE_RANGE_SIZE.getBytes(), ByteSizeValue.ofKb(8L).getBytes()) @@ -68,7 +69,7 @@ public void testRandomReads() throws IOException { final ByteSizeValue regionSize; if (rarely()) { - regionSize = FrozenCacheService.SNAPSHOT_CACHE_REGION_SIZE_SETTING.get(Settings.EMPTY); + regionSize = SnapshotsService.SNAPSHOT_CACHE_REGION_SIZE_SETTING.get(Settings.EMPTY); } else if (randomBoolean()) { regionSize = new ByteSizeValue(randomLongBetween(ByteSizeValue.ofKb(1L).getBytes(), ByteSizeValue.ofKb(8L).getBytes())); } else { @@ -83,9 +84,9 @@ public void testRandomReads() throws IOException { } final Settings settings = Settings.builder() - .put(FrozenCacheService.SNAPSHOT_CACHE_REGION_SIZE_SETTING.getKey(), regionSize) - .put(FrozenCacheService.FROZEN_CACHE_RANGE_SIZE_SETTING.getKey(), rangeSize) - .put(FrozenCacheService.SNAPSHOT_CACHE_SIZE_SETTING.getKey(), cacheSize) + .put(SnapshotsService.SNAPSHOT_CACHE_REGION_SIZE_SETTING.getKey(), regionSize) + .put(SnapshotsService.SHARED_CACHE_RANGE_SIZE_SETTING.getKey(), rangeSize) + .put(SnapshotsService.SNAPSHOT_CACHE_SIZE_SETTING.getKey(), cacheSize) .put("path.home", createTempDir()) .build(); final Environment environment = TestEnvironment.newEnvironment(settings); diff --git a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/index/store/cache/SparseFileTrackerTests.java b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/index/store/cache/SparseFileTrackerTests.java index 54b43a9ad86a1..97d541e342f77 100644 --- a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/index/store/cache/SparseFileTrackerTests.java +++ b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/index/store/cache/SparseFileTrackerTests.java @@ -91,17 +91,6 @@ public void testInvalidRange() { containsString("unable to listen to range") ); assertThat(invoked.get(), is(false)); - } else { - e = expectThrows( - IllegalArgumentException.class, - () -> sparseFileTracker.waitForRange(ByteRange.of(start, end), ByteRange.of(start - 1L, end), listener) - ); - assertThat( - "listener range start must not be smaller than zero", - e.getMessage(), - containsString("invalid range to listen to") - ); - assertThat(invoked.get(), is(false)); } if (end < length) { diff --git a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/AbstractSearchableSnapshotsTestCase.java b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/AbstractSearchableSnapshotsTestCase.java index 1b8c5d5c5dfd5..6044db3f31dab 100644 --- a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/AbstractSearchableSnapshotsTestCase.java +++ b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/AbstractSearchableSnapshotsTestCase.java @@ -33,6 +33,7 @@ import org.elasticsearch.repositories.IndexId; import org.elasticsearch.snapshots.Snapshot; import org.elasticsearch.snapshots.SnapshotId; +import org.elasticsearch.snapshots.SnapshotsService; import org.elasticsearch.test.ClusterServiceUtils; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.TestThreadPool; @@ -141,13 +142,13 @@ protected FrozenCacheService defaultFrozenCacheService() { protected FrozenCacheService randomFrozenCacheService() { final Settings.Builder cacheSettings = Settings.builder(); if (randomBoolean()) { - cacheSettings.put(FrozenCacheService.SNAPSHOT_CACHE_SIZE_SETTING.getKey(), randomFrozenCacheSize()); + cacheSettings.put(SnapshotsService.SNAPSHOT_CACHE_SIZE_SETTING.getKey(), randomFrozenCacheSize()); } if (randomBoolean()) { - cacheSettings.put(FrozenCacheService.SNAPSHOT_CACHE_REGION_SIZE_SETTING.getKey(), randomFrozenCacheSize()); + cacheSettings.put(SnapshotsService.SNAPSHOT_CACHE_REGION_SIZE_SETTING.getKey(), randomFrozenCacheSize()); } if (randomBoolean()) { - cacheSettings.put(FrozenCacheService.FROZEN_CACHE_RANGE_SIZE_SETTING.getKey(), randomCacheRangeSize()); + cacheSettings.put(SnapshotsService.SHARED_CACHE_RANGE_SIZE_SETTING.getKey(), randomCacheRangeSize()); } if (randomBoolean()) { cacheSettings.put(FrozenCacheService.FROZEN_CACHE_RECOVERY_RANGE_SIZE_SETTING.getKey(), randomCacheRangeSize()); @@ -174,8 +175,8 @@ protected FrozenCacheService createFrozenCacheService(final ByteSizeValue cacheS return new FrozenCacheService( newEnvironment( Settings.builder() - .put(FrozenCacheService.SNAPSHOT_CACHE_SIZE_SETTING.getKey(), cacheSize) - .put(FrozenCacheService.FROZEN_CACHE_RANGE_SIZE_SETTING.getKey(), cacheRangeSize) + .put(SnapshotsService.SNAPSHOT_CACHE_SIZE_SETTING.getKey(), cacheSize) + .put(SnapshotsService.SHARED_CACHE_RANGE_SIZE_SETTING.getKey(), cacheRangeSize) .build() ), threadPool diff --git a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/cache/FrozenCacheServiceTests.java b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/cache/FrozenCacheServiceTests.java index 769b924a24238..bda1258f3bee2 100644 --- a/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/cache/FrozenCacheServiceTests.java +++ b/x-pack/plugin/searchable-snapshots/src/test/java/org/elasticsearch/xpack/searchablesnapshots/cache/FrozenCacheServiceTests.java @@ -13,6 +13,7 @@ import org.elasticsearch.env.TestEnvironment; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.store.cache.CacheKey; +import org.elasticsearch.snapshots.SnapshotsService; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.searchablesnapshots.cache.FrozenCacheService.CacheFileRegion; @@ -28,8 +29,8 @@ public class FrozenCacheServiceTests extends ESTestCase { public void testBasicEviction() throws IOException { Settings settings = Settings.builder() .put(NODE_NAME_SETTING.getKey(), "node") - .put(FrozenCacheService.SNAPSHOT_CACHE_SIZE_SETTING.getKey(), "500b") - .put(FrozenCacheService.SNAPSHOT_CACHE_REGION_SIZE_SETTING.getKey(), "100b") + .put(SnapshotsService.SNAPSHOT_CACHE_SIZE_SETTING.getKey(), "500b") + .put(SnapshotsService.SNAPSHOT_CACHE_REGION_SIZE_SETTING.getKey(), "100b") .put("path.home", createTempDir()) .build(); final DeterministicTaskQueue taskQueue = new DeterministicTaskQueue(settings, random()); @@ -74,8 +75,8 @@ public void testBasicEviction() throws IOException { public void testAutoEviction() throws IOException { Settings settings = Settings.builder() .put(NODE_NAME_SETTING.getKey(), "node") - .put(FrozenCacheService.SNAPSHOT_CACHE_SIZE_SETTING.getKey(), "200b") - .put(FrozenCacheService.SNAPSHOT_CACHE_REGION_SIZE_SETTING.getKey(), "100b") + .put(SnapshotsService.SNAPSHOT_CACHE_SIZE_SETTING.getKey(), "200b") + .put(SnapshotsService.SNAPSHOT_CACHE_REGION_SIZE_SETTING.getKey(), "100b") .put("path.home", createTempDir()) .build(); final DeterministicTaskQueue taskQueue = new DeterministicTaskQueue(settings, random()); @@ -111,8 +112,8 @@ public void testAutoEviction() throws IOException { public void testForceEviction() throws IOException { Settings settings = Settings.builder() .put(NODE_NAME_SETTING.getKey(), "node") - .put(FrozenCacheService.SNAPSHOT_CACHE_SIZE_SETTING.getKey(), "500b") - .put(FrozenCacheService.SNAPSHOT_CACHE_REGION_SIZE_SETTING.getKey(), "100b") + .put(SnapshotsService.SNAPSHOT_CACHE_SIZE_SETTING.getKey(), "500b") + .put(SnapshotsService.SNAPSHOT_CACHE_REGION_SIZE_SETTING.getKey(), "100b") .put("path.home", createTempDir()) .build(); final DeterministicTaskQueue taskQueue = new DeterministicTaskQueue(settings, random()); @@ -140,8 +141,8 @@ public void testForceEviction() throws IOException { public void testDecay() throws IOException { Settings settings = Settings.builder() .put(NODE_NAME_SETTING.getKey(), "node") - .put(FrozenCacheService.SNAPSHOT_CACHE_SIZE_SETTING.getKey(), "500b") - .put(FrozenCacheService.SNAPSHOT_CACHE_REGION_SIZE_SETTING.getKey(), "100b") + .put(SnapshotsService.SNAPSHOT_CACHE_SIZE_SETTING.getKey(), "500b") + .put(SnapshotsService.SNAPSHOT_CACHE_REGION_SIZE_SETTING.getKey(), "100b") .put("path.home", createTempDir()) .build(); final DeterministicTaskQueue taskQueue = new DeterministicTaskQueue(settings, random()); diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java index 166dd60909d6e..86e907187514a 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java @@ -380,7 +380,7 @@ private void runRequestInterceptors(RequestInfo requestInfo, AuthorizationInfo a prevListener = current; } - prevListener.whenComplete(v -> listener.onResponse(null), listener::onFailure); + prevListener.addListener(listener); first.intercept(requestInfo, authorizationEngine, authorizationInfo, firstStepListener); } } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/SecurityRestFilter.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/SecurityRestFilter.java index 32b0003dff6cc..2f567c5e53c4c 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/SecurityRestFilter.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/SecurityRestFilter.java @@ -11,9 +11,9 @@ import org.apache.logging.log4j.message.ParameterizedMessage; import org.apache.logging.log4j.util.Supplier; import org.elasticsearch.ExceptionsHelper; -import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.compatibility.RestApiCompatibleVersion; import org.elasticsearch.common.util.Maps; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.common.xcontent.MediaType; @@ -25,15 +25,13 @@ import org.elasticsearch.rest.RestHandler; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.RestRequest.Method; -import org.elasticsearch.rest.RestStatus; import org.elasticsearch.rest.RestRequestFilter; - +import org.elasticsearch.rest.RestStatus; import org.elasticsearch.xpack.security.authc.AuthenticationService; import org.elasticsearch.xpack.security.authc.support.SecondaryAuthenticator; import org.elasticsearch.xpack.security.transport.SSLEngineUtils; import java.io.IOException; - import java.util.List; import java.util.Map; @@ -163,7 +161,7 @@ public MediaTypeRegistry validAcceptMediaTypes() { } @Override - public Version compatibleWithVersion() { + public RestApiCompatibleVersion compatibleWithVersion() { return restHandler.compatibleWithVersion(); } } diff --git a/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ResultSetTestCase.java b/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ResultSetTestCase.java index cc693caa91dd9..3567611f18f00 100644 --- a/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ResultSetTestCase.java +++ b/x-pack/plugin/sql/qa/jdbc/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ResultSetTestCase.java @@ -135,7 +135,7 @@ public void testMultiValueFieldWithMultiValueLeniencyDisabled() throws IOExcepti SQLException.class, () -> doWithQuery(() -> esWithLeniency(false), "SELECT int, keyword FROM test", results -> {}) ); - assertTrue(expected.getMessage().contains("Cannot return multiple values for field [int]; use ARRAY(int) instead")); + assertTrue(expected.getMessage().contains("Arrays (returned by [int]) are not supported")); // default has multi value disabled expectThrows(SQLException.class, () -> doWithQuery(this::esJdbc, "SELECT int, keyword FROM test", results -> {})); @@ -171,8 +171,7 @@ public void testMultiValueFields_InsideObjects_WithMultiValueLeniencyDisabled() results -> {} ) ); - assertTrue(expected.getMessage().contains("Cannot return multiple values for field [object.intsubfield]; " + - "use ARRAY(object.intsubfield) instead")); + assertTrue(expected.getMessage().contains("Arrays (returned by [object.intsubfield]) are not supported")); // default has multi value disabled expectThrows( diff --git a/x-pack/plugin/sql/qa/mixed-node/build.gradle b/x-pack/plugin/sql/qa/mixed-node/build.gradle new file mode 100644 index 0000000000000..8a39ecc795848 --- /dev/null +++ b/x-pack/plugin/sql/qa/mixed-node/build.gradle @@ -0,0 +1,66 @@ +apply plugin: 'elasticsearch.testclusters' +apply plugin: 'elasticsearch.standalone-rest-test' +apply from : "$rootDir/gradle/bwc-test.gradle" +apply plugin: 'elasticsearch.rest-test' + +import org.elasticsearch.gradle.Version +import org.elasticsearch.gradle.VersionProperties +import org.elasticsearch.gradle.info.BuildParams +import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask + +dependencies { + testImplementation project(':x-pack:qa') + testImplementation(project(xpackModule('ql:test'))) + testImplementation project(path: xpackModule('sql'), configuration: 'default') +} + +tasks.named("integTest").configure{ enabled = false} + +// A bug (https://github.com/elastic/elasticsearch/issues/68439) limits us to perform tests with versions from 7.10.3 onwards +for (Version bwcVersion : BuildParams.bwcVersions.wireCompatible.findAll { it.onOrAfter('7.10.0') }) { + if (bwcVersion == VersionProperties.getElasticsearchVersion()) { + // Not really a mixed cluster + continue; + } + + String baseName = "v${bwcVersion}" + + testClusters { + "${baseName}" { + versions = [bwcVersion.toString(), project.version] + numberOfNodes = 3 + testDistribution = 'DEFAULT' + setting 'xpack.security.enabled', 'false' + setting 'xpack.watcher.enabled', 'false' + setting 'xpack.ml.enabled', 'false' + setting 'xpack.license.self_generated.type', 'trial' + // for debugging purposes + // setting 'logger.org.elasticsearch.xpack.sql.plugin.TransportSqlQueryAction', 'TRACE' + } + } + + tasks.register("${baseName}#mixedClusterTest", StandaloneRestIntegTestTask) { + useCluster testClusters."${baseName}" + mustRunAfter("precommit") + doFirst { + // Getting the endpoints causes a wait for the cluster + println "Endpoints are: ${-> testClusters."${baseName}".allHttpSocketURI.join(",")}" + println "Upgrading one node to create a mixed cluster" + testClusters."${baseName}".nextNodeToNextVersion() + + println "Upgrade complete, endpoints are: ${-> testClusters."${baseName}".allHttpSocketURI.join(",")}" + nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}") + nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}") + } + onlyIf { project.bwc_tests_enabled } + } + + tasks.register(bwcTaskName(bwcVersion)) { + dependsOn "${baseName}#mixedClusterTest" + } + + // run these bwc tests as part of the "check" task + tasks.named("check").configure { + dependsOn "${baseName}#mixedClusterTest" + } +} diff --git a/x-pack/plugin/sql/qa/mixed-node/src/test/java/org/elasticsearch/xpack/sql/qa/mixed_node/SqlSearchIT.java b/x-pack/plugin/sql/qa/mixed-node/src/test/java/org/elasticsearch/xpack/sql/qa/mixed_node/SqlSearchIT.java new file mode 100644 index 0000000000000..267ae786c3667 --- /dev/null +++ b/x-pack/plugin/sql/qa/mixed-node/src/test/java/org/elasticsearch/xpack/sql/qa/mixed_node/SqlSearchIT.java @@ -0,0 +1,272 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.sql.qa.mixed_node; + +import org.apache.http.HttpHost; +import org.elasticsearch.Version; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.elasticsearch.test.NotEqualMessageBuilder; +import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.xpack.ql.TestNode; +import org.elasticsearch.xpack.ql.TestNodes; +import org.elasticsearch.xpack.sql.type.SqlDataTypes; +import org.junit.After; +import org.junit.Before; + +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.function.BiConsumer; +import java.util.function.Consumer; +import java.util.stream.Collectors; + +import static java.util.Collections.unmodifiableMap; +import static org.elasticsearch.xpack.ql.TestUtils.buildNodeAndVersions; +import static org.elasticsearch.xpack.ql.TestUtils.readResource; +import static org.elasticsearch.xpack.ql.execution.search.QlSourceBuilder.SWITCH_TO_FIELDS_API_VERSION; + +public class SqlSearchIT extends ESRestTestCase { + + /* + * The version where we made a significant change to how we query ES and how we interpret the results we get from ES, is 7.12 + * (the switch from extracting from _source and docvalues to using the "fields" API). The behavior of the tests is slightly + * changed on some versions and it all depends on when this above mentioned change was made. + */ + private static final Version FIELDS_API_QL_INTRODUCTION = Version.V_7_12_0; + private static final String index = "test_sql_mixed_versions"; + private static int numShards; + private static int numReplicas = 1; + private static int numDocs; + private static TestNodes nodes; + private static List newNodes; + private static List bwcNodes; + private static Version bwcVersion; + private static Version newVersion; + private static boolean isBwcNodeBeforeFieldsApiInQL; + private static boolean isBwcNodeBeforeFieldsApiInES; + + @Before + public void createIndex() throws IOException { + nodes = buildNodeAndVersions(client()); + numShards = nodes.size(); + numDocs = randomIntBetween(numShards, 15); + newNodes = new ArrayList<>(nodes.getNewNodes()); + bwcNodes = new ArrayList<>(nodes.getBWCNodes()); + bwcVersion = nodes.getBWCNodes().get(0).getVersion(); + newVersion = nodes.getNewNodes().get(0).getVersion(); + isBwcNodeBeforeFieldsApiInQL = bwcVersion.before(FIELDS_API_QL_INTRODUCTION); + isBwcNodeBeforeFieldsApiInES = bwcVersion.before(SWITCH_TO_FIELDS_API_VERSION); + + String mappings = readResource(SqlSearchIT.class.getResourceAsStream("/all_field_types.json")); + createIndex( + index, + Settings.builder() + .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), numShards) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, numReplicas) + .build(), + mappings + ); + } + + @After + public void cleanUpIndex() throws IOException { + if (indexExists(index)) { + deleteIndex(index); + } + } + + public void testAllTypesWithRequestToOldNodes() throws Exception { + Map expectedResponse = prepareTestData( + columns -> { + columns.add(columnInfo("geo_point_field", "geo_point")); + columns.add(columnInfo("float_field", "float")); + columns.add(columnInfo("half_float_field", "half_float")); + }, + (builder, fieldValues) -> { + Float randomFloat = randomFloat(); + // before "fields" API being added to QL, numbers were re-parsed from _source with a similar approach to + // indexing docvalues and for floating point numbers this may be different from the actual value passed in the _source + // floats were indexed as Doubles and the values returned had a greater precision and more decimals + builder.append(","); + if (isBwcNodeBeforeFieldsApiInQL) { + builder.append("\"geo_point_field\":{\"lat\":\"37.386483\", \"lon\":\"-122.083843\"},"); + fieldValues.put("geo_point_field", "POINT (-122.08384302444756 37.38648299127817)"); + builder.append("\"float_field\":" + randomFloat + ","); + fieldValues.put("float_field", Double.valueOf(randomFloat)); + builder.append("\"half_float_field\":123.456"); + fieldValues.put("half_float_field", 123.45600128173828d); + } else { + builder.append("\"geo_point_field\":{\"lat\":\"37.386483\", \"lon\":\"-122.083843\"},"); + fieldValues.put("geo_point_field", "POINT (-122.083843 37.386483)"); + builder.append("\"float_field\":" + randomFloat + ","); + fieldValues.put("float_field", Double.valueOf(Float.valueOf(randomFloat).toString())); + builder.append("\"half_float_field\":" + fieldValues.computeIfAbsent("half_float_field", v -> 123.456)); + } + } + ); + assertAllTypesWithNodes(expectedResponse, bwcNodes); + } + + public void testAllTypesWithRequestToUpgradedNodes() throws Exception { + Map expectedResponse = prepareTestData( + columns -> { + columns.add(columnInfo("geo_point_field", "geo_point")); + columns.add(columnInfo("float_field", "float")); + columns.add(columnInfo("half_float_field", "half_float")); + }, + (builder, fieldValues) -> { + Float randomFloat = randomFloat(); + builder.append(","); + if (isBwcNodeBeforeFieldsApiInQL && isBwcNodeBeforeFieldsApiInES) { + builder.append("\"geo_point_field\":{\"lat\":\"37.386483\", \"lon\":\"-122.083843\"},"); + fieldValues.put("geo_point_field", "POINT (-122.08384302444756 37.38648299127817)"); + builder.append("\"float_field\":" + randomFloat + ","); + fieldValues.put("float_field", Double.valueOf(randomFloat)); + builder.append("\"half_float_field\":123.456"); + fieldValues.put("half_float_field", 123.45600128173828d); + } else { + builder.append("\"geo_point_field\":{\"lat\":\"37.386483\", \"lon\":\"-122.083843\"},"); + fieldValues.put("geo_point_field", "POINT (-122.083843 37.386483)"); + builder.append("\"float_field\":" + randomFloat + ","); + fieldValues.put("float_field", Double.valueOf(Float.valueOf(randomFloat).toString())); + builder.append("\"half_float_field\":" + fieldValues.computeIfAbsent("half_float_field", v -> 123.456)); + } + } + ); + assertAllTypesWithNodes(expectedResponse, newNodes); + } + + @SuppressWarnings("unchecked") + private Map prepareTestData(Consumer>> additionalColumns, + BiConsumer> additionalValues) throws IOException { + Map expectedResponse = new HashMap<>(); + List> columns = new ArrayList<>(); + columns.add(columnInfo("interval_year", "interval_year")); + columns.add(columnInfo("interval_minute", "interval_minute")); + columns.add(columnInfo("long_field", "long")); + columns.add(columnInfo("integer_field", "integer")); + columns.add(columnInfo("short_field", "short")); + columns.add(columnInfo("byte_field", "byte")); + columns.add(columnInfo("double_field", "double")); + columns.add(columnInfo("scaled_float_field", "scaled_float")); + columns.add(columnInfo("boolean_field", "boolean")); + columns.add(columnInfo("ip_field", "ip")); + columns.add(columnInfo("text_field", "text")); + columns.add(columnInfo("keyword_field", "keyword")); + columns.add(columnInfo("constant_keyword_field", "keyword")); + columns.add(columnInfo("wildcard_field", "keyword")); + columns.add(columnInfo("geo_point_no_dv_field", "geo_point")); + columns.add(columnInfo("geo_shape_field", "geo_shape")); + columns.add(columnInfo("shape_field", "shape")); + + expectedResponse.put("columns", columns); + additionalColumns.accept(columns); + List> rows = new ArrayList<>(numDocs); + expectedResponse.put("rows", rows); + + Map fieldValues; + String constantKeywordValue = randomAlphaOfLength(5); + for (int i = 0; i < numDocs; i++) { + fieldValues = new LinkedHashMap<>(); + fieldValues.put("interval_year", "P150Y"); + fieldValues.put("interval_minute", "PT2H43M"); + + StringBuilder builder = new StringBuilder(); + builder.append("{"); + builder.append("\"id\":" + i + ","); + builder.append("\"long_field\":" + fieldValues.computeIfAbsent("long_field", v -> randomLong()) + ","); + builder.append("\"integer_field\":" + fieldValues.computeIfAbsent("integer_field", v -> randomInt()) + ","); + builder.append("\"short_field\":" + fieldValues.computeIfAbsent("short_field", v -> Integer.valueOf(randomShort())) + ","); + builder.append("\"byte_field\":" + fieldValues.computeIfAbsent("byte_field", v -> Integer.valueOf(randomByte())) + ","); + builder.append("\"double_field\":" + fieldValues.computeIfAbsent("double_field", v -> randomDouble()) + ","); + builder.append("\"scaled_float_field\":" + fieldValues.computeIfAbsent("scaled_float_field", v -> 123.5d) + ","); + builder.append("\"boolean_field\":" + fieldValues.computeIfAbsent("boolean_field", v -> randomBoolean()) + ","); + builder.append("\"ip_field\":\"" + fieldValues.computeIfAbsent("ip_field", v -> "123.123.123.123") + "\","); + builder.append("\"text_field\": \"" + fieldValues.computeIfAbsent("text_field", v -> randomAlphaOfLength(5)) + "\","); + builder.append("\"keyword_field\": \"" + fieldValues.computeIfAbsent("keyword_field", v -> randomAlphaOfLength(5)) + "\","); + builder.append("\"constant_keyword_field\": \"" + fieldValues.computeIfAbsent("constant_keyword_field", + v -> constantKeywordValue) + "\","); + builder.append("\"wildcard_field\": \"" + fieldValues.computeIfAbsent("wildcard_field", v -> randomAlphaOfLength(5)) + "\","); + builder.append("\"geo_point_no_dv_field\":{\"lat\":\"40.123456\", \"lon\":\"100.234567\"},"); + fieldValues.put("geo_point_no_dv_field", "POINT (100.234567 40.123456)"); + builder.append("\"geo_shape_field\":\"POINT (-122.083843 37.386483 30)\","); + fieldValues.put("geo_shape_field", "POINT (-122.083843 37.386483 30.0)"); + builder.append("\"shape_field\":\"POINT (-122.083843 37.386483 30)\""); + fieldValues.put("shape_field", "POINT (-122.083843 37.386483 30.0)"); + additionalValues.accept(builder, fieldValues); + builder.append("}"); + + Request request = new Request("PUT", index + "/_doc/" + i); + request.setJsonEntity(builder.toString()); + assertOK(client().performRequest(request)); + + List row = new ArrayList<>(fieldValues.values()); + rows.add(row); + } + return expectedResponse; + } + + private Map columnInfo(String name, String type) { + Map column = new HashMap<>(); + column.put("name", name); + column.put("type", type); + column.put("display_size", SqlDataTypes.displaySize(SqlDataTypes.fromTypeName(type))); + return unmodifiableMap(column); + } + + private void assertAllTypesWithNodes(Map expectedResponse, List nodesList) throws Exception { + try ( + RestClient client = buildClient(restClientSettings(), + nodesList.stream().map(TestNode::getPublishAddress).toArray(HttpHost[]::new)) + ) { + Request request = new Request("POST", "_sql"); + String version = ",\"version\":\"" + newVersion.toString() + "\""; + String binaryFormat = ",\"binary_format\":\"false\""; + + @SuppressWarnings("unchecked") + List> columns = (List>) expectedResponse.get("columns"); + String intervalYearMonth = "INTERVAL '150' YEAR AS interval_year, "; + String intervalDayTime = "INTERVAL '163' MINUTE AS interval_minute, "; + + // get all fields names from the expected response built earlier, skipping the intervals as they execute locally + // and not taken from the index itself + String fieldsList = columns.stream().map(m -> (String) m.get("name")).filter(str -> str.startsWith("interval") == false) + .collect(Collectors.toList()).stream().collect(Collectors.joining(", ")); + String query = "SELECT " + intervalYearMonth + intervalDayTime + fieldsList + " FROM " + index + " ORDER BY id"; + request.setJsonEntity( + "{\"mode\":\"jdbc\"" + version + binaryFormat + ",\"query\":\"" + query + "\"}" + ); + assertBusy(() -> { assertResponse(expectedResponse, runSql(client, request)); }); + } + } + + private void assertResponse(Map expected, Map actual) { + if (false == expected.equals(actual)) { + NotEqualMessageBuilder message = new NotEqualMessageBuilder(); + message.compareMaps(actual, expected); + fail("Response does not match:\n" + message.toString()); + } + } + + private Map runSql(RestClient client, Request request) throws IOException { + Response response = client.performRequest(request); + try (InputStream content = response.getEntity().getContent()) { + return XContentHelper.convertToMap(JsonXContent.jsonXContent, content, false); + } + } +} diff --git a/x-pack/plugin/sql/qa/mixed-node/src/test/resources/all_field_types.json b/x-pack/plugin/sql/qa/mixed-node/src/test/resources/all_field_types.json new file mode 100644 index 0000000000000..491be6f0f96b6 --- /dev/null +++ b/x-pack/plugin/sql/qa/mixed-node/src/test/resources/all_field_types.json @@ -0,0 +1,59 @@ +"properties": { + "long_field": { + "type": "long" + }, + "integer_field": { + "type": "integer" + }, + "short_field": { + "type": "short" + }, + "byte_field": { + "type": "byte" + }, + "double_field": { + "type": "double" + }, + "float_field": { + "type": "float" + }, + "half_float_field": { + "type": "half_float" + }, + "scaled_float_field": { + "type": "scaled_float", + "scaling_factor": 10 + }, + "boolean_field": { + "type": "boolean" + }, + "ip_field": { + "type": "ip" + }, + "text_field": { + "type": "text" + }, + "keyword_field": { + "type": "keyword" + }, + "constant_keyword_field": { + "type": "constant_keyword" + }, + // added in 7.9.0 + "wildcard_field": { + "type": "wildcard" + }, + "geo_point_field": { + "type": "geo_point" + }, + "geo_point_no_dv_field": { + "type": "geo_point", + "doc_values": "false" + }, + "geo_shape_field": { + "type": "geo_shape" + }, + "shape_field": { + "type": "shape" + } +} \ No newline at end of file diff --git a/x-pack/plugin/sql/qa/server/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/CliExplainIT.java b/x-pack/plugin/sql/qa/server/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/CliExplainIT.java index 60245d1144dd5..82b49bd9f4250 100644 --- a/x-pack/plugin/sql/qa/server/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/CliExplainIT.java +++ b/x-pack/plugin/sql/qa/server/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/CliExplainIT.java @@ -40,12 +40,12 @@ public void testExplainBasic() throws IOException { assertThat(command("EXPLAIN (PLAN EXECUTABLE) SELECT * FROM test"), containsString("plan")); assertThat(readLine(), startsWith("----------")); assertThat(readLine(), startsWith("EsQueryExec[test,{")); - assertThat(readLine(), startsWith(" \"_source\" : {")); - assertThat(readLine(), startsWith(" \"includes\" : [")); - assertThat(readLine(), startsWith(" \"test_field\"")); - assertThat(readLine(), startsWith(" ],")); - assertThat(readLine(), startsWith(" \"excludes\" : [ ]")); - assertThat(readLine(), startsWith(" },")); + assertThat(readLine(), startsWith(" \"_source\" : false,")); + assertThat(readLine(), startsWith(" \"fields\" : [")); + assertThat(readLine(), startsWith(" {")); + assertThat(readLine(), startsWith(" \"field\" : \"test_field\"")); + assertThat(readLine(), startsWith(" }")); + assertThat(readLine(), startsWith(" ],")); assertThat(readLine(), startsWith(" \"sort\" : [")); assertThat(readLine(), startsWith(" {")); assertThat(readLine(), startsWith(" \"_doc\" :")); @@ -97,13 +97,15 @@ public void testExplainWithWhere() throws IOException { assertThat(readLine(), startsWith(" }")); assertThat(readLine(), startsWith(" }")); assertThat(readLine(), startsWith(" },")); - assertThat(readLine(), startsWith(" \"_source\" : {")); - assertThat(readLine(), startsWith(" \"includes\" : [")); - assertThat(readLine(), startsWith(" \"i\"")); - assertThat(readLine(), startsWith(" \"test_field\"")); - assertThat(readLine(), startsWith(" ],")); - assertThat(readLine(), startsWith(" \"excludes\" : [ ]")); - assertThat(readLine(), startsWith(" },")); + assertThat(readLine(), startsWith(" \"_source\" : false,")); + assertThat(readLine(), startsWith(" \"fields\" : [")); + assertThat(readLine(), startsWith(" {")); + assertThat(readLine(), startsWith(" \"field\" : \"i\"")); + assertThat(readLine(), startsWith(" },")); + assertThat(readLine(), startsWith(" {")); + assertThat(readLine(), startsWith(" \"field\" : \"test_field\"")); + assertThat(readLine(), startsWith(" }")); + assertThat(readLine(), startsWith(" ],")); assertThat(readLine(), startsWith(" \"sort\" : [")); assertThat(readLine(), startsWith(" {")); assertThat(readLine(), startsWith(" \"_doc\" :")); @@ -143,7 +145,6 @@ public void testExplainWithCount() throws IOException { assertThat(readLine(), startsWith("EsQueryExec[test,{")); assertThat(readLine(), startsWith(" \"size\" : 0,")); assertThat(readLine(), startsWith(" \"_source\" : false,")); - assertThat(readLine(), startsWith(" \"stored_fields\" : \"_none_\",")); assertThat(readLine(), startsWith(" \"sort\" : [")); assertThat(readLine(), startsWith(" {")); assertThat(readLine(), startsWith(" \"_doc\" :")); diff --git a/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/FieldExtractorTestCase.java b/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/FieldExtractorTestCase.java index cd12021aa3593..2a0391290bc1d 100644 --- a/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/FieldExtractorTestCase.java +++ b/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/FieldExtractorTestCase.java @@ -49,7 +49,7 @@ public void testTextField() throws IOException { String query = "SELECT text_field FROM test"; String text = randomAlphaOfLength(20); boolean explicitSourceSetting = randomBoolean(); // default (no _source setting) or explicit setting - boolean enableSource = randomBoolean(); // enable _source at index level + boolean enableSource = randomBoolean(); // enable _source at index level Map indexProps = new HashMap<>(1); indexProps.put("_source", enableSource); @@ -74,10 +74,10 @@ public void testTextField() throws IOException { * } */ public void testKeywordField() throws IOException { + String query = "SELECT keyword_field FROM test"; String keyword = randomAlphaOfLength(20); - // _source for `keyword` fields doesn't matter, as they should be taken from docvalue_fields boolean explicitSourceSetting = randomBoolean(); // default (no _source setting) or explicit setting - boolean enableSource = randomBoolean(); // enable _source at index level + boolean enableSource = randomBoolean(); // enable _source at index level boolean ignoreAbove = randomBoolean(); Map indexProps = new HashMap<>(1); @@ -94,10 +94,14 @@ public void testKeywordField() throws IOException { createIndexWithFieldTypeAndProperties("keyword", fieldProps, explicitSourceSetting ? indexProps : null); index("{\"keyword_field\":\"" + keyword + "\"}"); - Map expected = new HashMap<>(); - expected.put("columns", Arrays.asList(columnInfo("plain", "keyword_field", "keyword", JDBCType.VARCHAR, Integer.MAX_VALUE))); - expected.put("rows", singletonList(singletonList(ignoreAbove ? null : keyword))); - assertResponse(expected, runSql("SELECT keyword_field FROM test")); + if (explicitSourceSetting == false || enableSource) { + Map expected = new HashMap<>(); + expected.put("columns", Arrays.asList(columnInfo("plain", "keyword_field", "keyword", JDBCType.VARCHAR, Integer.MAX_VALUE))); + expected.put("rows", singletonList(singletonList(ignoreAbove ? null : keyword))); + assertResponse(expected, runSql(query)); + } else { + expectSourceDisabledError(query); + } } /* @@ -107,10 +111,10 @@ public void testKeywordField() throws IOException { * } */ public void testConstantKeywordField() throws IOException { + String query = "SELECT constant_keyword_field FROM test"; String value = randomAlphaOfLength(20); - // _source for `constant_keyword` fields doesn't matter, as they should be taken from docvalue_fields boolean explicitSourceSetting = randomBoolean(); // default (no _source setting) or explicit setting - boolean enableSource = randomBoolean(); // enable _source at index level + boolean enableSource = randomBoolean(); // enable _source at index level Map indexProps = new HashMap<>(1); indexProps.put("_source", enableSource); @@ -126,13 +130,17 @@ public void testConstantKeywordField() throws IOException { createIndexWithFieldTypeAndProperties("constant_keyword", fieldProps, explicitSourceSetting ? indexProps : null); index("{\"constant_keyword_field\":\"" + value + "\"}"); - Map expected = new HashMap<>(); - expected.put( - "columns", - Arrays.asList(columnInfo("plain", "constant_keyword_field", "keyword", JDBCType.VARCHAR, Integer.MAX_VALUE)) - ); - expected.put("rows", singletonList(singletonList(value))); - assertResponse(expected, runSql("SELECT constant_keyword_field FROM test")); + if (explicitSourceSetting == false || enableSource) { + Map expected = new HashMap<>(); + expected.put( + "columns", + Arrays.asList(columnInfo("plain", "constant_keyword_field", "keyword", JDBCType.VARCHAR, Integer.MAX_VALUE)) + ); + expected.put("rows", singletonList(singletonList(value))); + assertResponse(expected, runSql(query)); + } else { + expectSourceDisabledError(query); + } } /* @@ -142,10 +150,10 @@ public void testConstantKeywordField() throws IOException { * } */ public void testWildcardField() throws IOException { + String query = "SELECT wildcard_field FROM test"; String wildcard = randomAlphaOfLength(20); - // _source for `wildcard` fields doesn't matter, as they should be taken from docvalue_fields boolean explicitSourceSetting = randomBoolean(); // default (no _source setting) or explicit setting - boolean enableSource = randomBoolean(); // enable _source at index level + boolean enableSource = randomBoolean(); // enable _source at index level boolean ignoreAbove = randomBoolean(); Map indexProps = new HashMap<>(1); @@ -162,10 +170,14 @@ public void testWildcardField() throws IOException { createIndexWithFieldTypeAndProperties("wildcard", fieldProps, explicitSourceSetting ? indexProps : null); index("{\"wildcard_field\":\"" + wildcard + "\"}"); - Map expected = new HashMap<>(); - expected.put("columns", Arrays.asList(columnInfo("plain", "wildcard_field", "keyword", JDBCType.VARCHAR, Integer.MAX_VALUE))); - expected.put("rows", singletonList(singletonList(ignoreAbove ? null : wildcard))); - assertResponse(expected, runSql("SELECT wildcard_field FROM test")); + if (explicitSourceSetting == false || enableSource) { + Map expected = new HashMap<>(); + expected.put("columns", Arrays.asList(columnInfo("plain", "wildcard_field", "keyword", JDBCType.VARCHAR, Integer.MAX_VALUE))); + expected.put("rows", singletonList(singletonList(ignoreAbove ? null : wildcard))); + assertResponse(expected, runSql(query)); + } else { + expectSourceDisabledError(query); + } } /* @@ -222,14 +234,7 @@ public void testCoerceForFloatingPointTypes() throws IOException { // because "coerce" is true, a "123.456" floating point number STRING should be converted to 123.456 as number // and converted to 123.5 for "scaled_float" type - expected.put( - "rows", - singletonList( - singletonList( - isScaledFloat ? 123.5 : (fieldType != "double" ? Double.valueOf(123.456f) : Double.valueOf(floatingPointNumber)) - ) - ) - ); + expected.put("rows", singletonList(singletonList(isScaledFloat ? 123.5 : 123.456d))); assertResponse(expected, runSql("SELECT " + fieldType + "_field FROM test")); } @@ -282,7 +287,7 @@ private void testField(String fieldType, Object value) throws IOException { String query = "SELECT " + fieldName + " FROM test"; Object actualValue = value; boolean explicitSourceSetting = randomBoolean(); // default (no _source setting) or explicit setting - boolean enableSource = randomBoolean(); // enable _source at index level + boolean enableSource = randomBoolean(); // enable _source at index level boolean ignoreMalformed = randomBoolean(); // ignore_malformed is true, thus test a non-number value Map indexProps = new HashMap<>(1); @@ -320,7 +325,7 @@ public void testBooleanField() throws IOException { String query = "SELECT boolean_field FROM test"; boolean booleanField = randomBoolean(); boolean explicitSourceSetting = randomBoolean(); // default (no _source setting) or explicit setting - boolean enableSource = randomBoolean(); // enable _source at index level + boolean enableSource = randomBoolean(); // enable _source at index level boolean asString = randomBoolean(); // pass true or false as string "true" or "false Map indexProps = new HashMap<>(1); @@ -337,7 +342,7 @@ public void testBooleanField() throws IOException { Map expected = new HashMap<>(); expected.put("columns", Arrays.asList(columnInfo("plain", "boolean_field", "boolean", JDBCType.BOOLEAN, Integer.MAX_VALUE))); // adding the boolean as a String here because parsing the response will yield a "true"/"false" String - expected.put("rows", singletonList(singletonList(asString ? String.valueOf(booleanField) : booleanField))); + expected.put("rows", singletonList(singletonList(booleanField))); assertResponse(expected, runSql(query)); } else { expectSourceDisabledError(query); @@ -355,7 +360,7 @@ public void testIpField() throws IOException { String ipField = "192.168.1.1"; String actualValue = ipField; boolean explicitSourceSetting = randomBoolean(); // default (no _source setting) or explicit setting - boolean enableSource = randomBoolean(); // enable _source at index level + boolean enableSource = randomBoolean(); // enable _source at index level boolean ignoreMalformed = randomBoolean(); Map indexProps = new HashMap<>(1); @@ -392,7 +397,6 @@ public void testIpField() throws IOException { public void testGeoPointField() throws IOException { String query = "SELECT geo_point_field FROM test"; String geoPointField = "41.12,-71.34"; - String geoPointFromDocValues = "POINT (-71.34000004269183 41.1199999647215)"; String actualValue = geoPointField; boolean explicitSourceSetting = randomBoolean(); // default (no _source setting) or explicit setting boolean enableSource = randomBoolean(); // enable _source at index level @@ -413,11 +417,17 @@ public void testGeoPointField() throws IOException { createIndexWithFieldTypeAndProperties("geo_point", fieldProps, explicitSourceSetting ? indexProps : null); index("{\"geo_point_field\":\"" + actualValue + "\"}"); - // the values come from docvalues (vs from _source) so disabling the source doesn't have any impact on the values returned - Map expected = new HashMap<>(); - expected.put("columns", Arrays.asList(columnInfo("plain", "geo_point_field", "geo_point", JDBCType.VARCHAR, Integer.MAX_VALUE))); - expected.put("rows", singletonList(singletonList(ignoreMalformed ? null : geoPointFromDocValues))); - assertResponse(expected, runSql(query)); + if (explicitSourceSetting == false || enableSource) { + Map expected = new HashMap<>(); + expected.put( + "columns", + Arrays.asList(columnInfo("plain", "geo_point_field", "geo_point", JDBCType.VARCHAR, Integer.MAX_VALUE)) + ); + expected.put("rows", singletonList(singletonList(ignoreMalformed ? null : "POINT (-71.34 41.12)"))); + assertResponse(expected, runSql(query)); + } else { + expectSourceDisabledError(query); + } } /* @@ -467,7 +477,6 @@ public void testGeoShapeField() throws IOException { * "ignore_malformed": true/false * } */ - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/66678") public void testShapeField() throws IOException { String query = "SELECT shape_field FROM test"; String shapeField = "POINT (-377.03653 389.897676)"; @@ -494,7 +503,7 @@ public void testShapeField() throws IOException { if (explicitSourceSetting == false || enableSource) { Map expected = new HashMap<>(); expected.put("columns", Arrays.asList(columnInfo("plain", "shape_field", "shape", JDBCType.VARCHAR, Integer.MAX_VALUE))); - expected.put("rows", singletonList(singletonList(ignoreMalformed ? null : "POINT (-377.03653 389.897676)"))); + expected.put("rows", singletonList(singletonList(ignoreMalformed ? null : shapeField))); assertResponse(expected, runSql(query)); } else { expectSourceDisabledError(query); @@ -561,7 +570,7 @@ public void testAliasFromSourceField() throws IOException { columnInfo("plain", "a.b.c.text_field_alias", "text", JDBCType.VARCHAR, Integer.MAX_VALUE) ) ); - expected.put("rows", singletonList(Arrays.asList(text, null, null))); + expected.put("rows", singletonList(Arrays.asList(text, text, text))); assertResponse(expected, runSql("SELECT text_field, text_field_alias, a.b.c.text_field_alias FROM test")); } @@ -593,7 +602,7 @@ public void testAliasAggregatableFromSourceField() throws IOException { columnInfo("plain", "a.b.c.integer_field_alias", "integer", JDBCType.INTEGER, Integer.MAX_VALUE) ) ); - expected.put("rows", singletonList(Arrays.asList(number, null, number))); + expected.put("rows", singletonList(Arrays.asList(number, number, number))); assertResponse(expected, runSql("SELECT integer_field, integer_field_alias, a.b.c.integer_field_alias FROM test")); } @@ -610,9 +619,8 @@ public void testAliasAggregatableFromSourceField() throws IOException { */ public void testTextFieldWithKeywordSubfield() throws IOException { String text = randomAlphaOfLength(10) + " " + randomAlphaOfLength(10); - // _source for `keyword` fields doesn't matter, as they should be taken from docvalue_fields boolean explicitSourceSetting = randomBoolean(); // default (no _source setting) or explicit setting - boolean enableSource = randomBoolean(); // enable _source at index level + boolean enableSource = randomBoolean(); // enable _source at index level boolean ignoreAbove = randomBoolean(); String fieldName = "text_field"; String subFieldName = "text_field.keyword_subfield"; @@ -646,13 +654,7 @@ public void testTextFieldWithKeywordSubfield() throws IOException { assertResponse(expected, runSql(query)); } else { expectSourceDisabledError(query); - - // even if the _source is disabled, selecting only the keyword sub-field should work as expected - Map expected = new HashMap<>(); - expected.put("columns", Arrays.asList(columnInfo("plain", subFieldName, "keyword", JDBCType.VARCHAR, Integer.MAX_VALUE))); - - expected.put("rows", singletonList(singletonList(ignoreAbove ? null : text))); - assertResponse(expected, runSql("SELECT text_field.keyword_subfield FROM test")); + expectSourceDisabledError("SELECT " + subFieldName + " FROM test"); } } @@ -670,7 +672,7 @@ public void testTextFieldWithKeywordSubfield() throws IOException { public void testTextFieldWithIntegerNumberSubfield() throws IOException { Integer number = randomInt(); boolean explicitSourceSetting = randomBoolean(); // default (no _source setting) or explicit setting - boolean enableSource = randomBoolean(); // enable _source at index level + boolean enableSource = randomBoolean(); // enable _source at index level boolean ignoreMalformed = randomBoolean(); // ignore_malformed is true, thus test a non-number value Object actualValue = number; String fieldName = "text_field"; @@ -788,7 +790,7 @@ public void testTextFieldWithIpSubfield() throws IOException { public void testNumberFieldWithTextOrKeywordSubfield() throws IOException { Integer number = randomInt(); boolean explicitSourceSetting = randomBoolean(); // default (no _source setting) or explicit setting - boolean enableSource = randomBoolean(); // enable _source at index level + boolean enableSource = randomBoolean(); // enable _source at index level boolean ignoreMalformed = randomBoolean(); // ignore_malformed is true, thus test a non-number value boolean isKeyword = randomBoolean(); // text or keyword subfield Object actualValue = number; @@ -834,21 +836,12 @@ public void testNumberFieldWithTextOrKeywordSubfield() throws IOException { } assertResponse(expected, runSql(query)); } else { + // disabling the _source means that nothing should be retrieved by the "fields" API if (isKeyword) { - // selecting only the keyword subfield when the _source is disabled should work - Map expected = new HashMap<>(); - expected.put("columns", singletonList(columnInfo("plain", subFieldName, "keyword", JDBCType.VARCHAR, Integer.MAX_VALUE))); - if (ignoreMalformed) { - expected.put("rows", singletonList(singletonList("foo"))); - } else { - expected.put("rows", singletonList(singletonList(String.valueOf(number)))); - } - assertResponse(expected, runSql("SELECT integer_field.keyword_subfield FROM test")); + expectSourceDisabledError("SELECT integer_field.keyword_subfield FROM test"); } else { expectSourceDisabledError(query); } - - // if the _source is disabled, selecting only the integer field shouldn't work expectSourceDisabledError("SELECT " + fieldName + " FROM test"); } } @@ -913,22 +906,9 @@ public void testIpFieldWithTextOrKeywordSubfield() throws IOException { } assertResponse(expected, runSql(query)); } else { - if (isKeyword) { - // selecting only the keyword subfield when the _source is disabled should work - Map expected = new HashMap<>(); - expected.put("columns", singletonList(columnInfo("plain", subFieldName, "keyword", JDBCType.VARCHAR, Integer.MAX_VALUE))); - if (ignoreMalformed) { - expected.put("rows", singletonList(singletonList("foo"))); - } else { - expected.put("rows", singletonList(singletonList(ip))); - } - assertResponse(expected, runSql("SELECT ip_field.keyword_subfield FROM test")); - } else { - expectSourceDisabledError(query); - } - - // if the _source is disabled, selecting only the ip field shouldn't work + expectSourceDisabledError(query); expectSourceDisabledError("SELECT " + fieldName + " FROM test"); + expectSourceDisabledError("SELECT " + subFieldName + " FROM test"); } } @@ -948,7 +928,7 @@ public void testIntegerFieldWithByteSubfield() throws IOException { boolean isByte = randomBoolean(); Integer number = isByte ? randomByte() : randomIntBetween(Byte.MAX_VALUE + 1, Integer.MAX_VALUE); boolean explicitSourceSetting = randomBoolean(); // default (no _source setting) or explicit setting - boolean enableSource = randomBoolean(); // enable _source at index level + boolean enableSource = randomBoolean(); // enable _source at index level boolean rootIgnoreMalformed = randomBoolean(); // root field ignore_malformed boolean subFieldIgnoreMalformed = randomBoolean(); // sub-field ignore_malformed String fieldName = "integer_field"; @@ -1017,7 +997,7 @@ public void testByteFieldWithIntegerSubfield() throws IOException { boolean isByte = randomBoolean(); Integer number = isByte ? randomByte() : randomIntBetween(Byte.MAX_VALUE + 1, Integer.MAX_VALUE); boolean explicitSourceSetting = randomBoolean(); // default (no _source setting) or explicit setting - boolean enableSource = randomBoolean(); // enable _source at index level + boolean enableSource = randomBoolean(); // enable _source at index level boolean rootIgnoreMalformed = randomBoolean(); // root field ignore_malformed boolean subFieldIgnoreMalformed = randomBoolean(); // sub-field ignore_malformed String fieldName = "byte_field"; @@ -1074,7 +1054,7 @@ private void expectSourceDisabledError(String query) { expectBadRequest(() -> { client().performRequest(buildRequest(query)); return Collections.emptyMap(); - }, containsString("unable to fetch fields from _source field: _source is disabled in the mappings for index [test]")); + }, containsString("Unable to retrieve the requested [fields] since _source is disabled in the mappings for index [test]")); } private void createIndexWithFieldTypeAndAlias(String type, Map> fieldProps, Map indexProps) diff --git a/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/rest/RestSqlTestCase.java b/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/rest/RestSqlTestCase.java index b4b6201d70bfe..5d9341e799377 100644 --- a/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/rest/RestSqlTestCase.java +++ b/x-pack/plugin/sql/qa/server/src/main/java/org/elasticsearch/xpack/sql/qa/rest/RestSqlTestCase.java @@ -669,11 +669,10 @@ public void testBasicTranslateQuery() throws IOException { Map response = runTranslateSql(query("SELECT * FROM test").toString()); assertEquals(1000, response.get("size")); + assertFalse((Boolean) response.get("_source")); @SuppressWarnings("unchecked") - Map source = (Map) response.get("_source"); - assertNotNull(source); - assertEquals(emptyList(), source.get("excludes")); - assertEquals(singletonList("test"), source.get("includes")); + List> source = (List>) response.get("fields"); + assertEquals(singletonList(singletonMap("field", "test")), source); } public void testBasicQueryWithFilter() throws IOException { @@ -734,6 +733,7 @@ public void testBasicQueryWithParameters() throws IOException { ); } + @AwaitsFix(bugUrl = "Test disabled while merging fields API in") public void testBasicQueryWithMultiValues() throws IOException { List values = randomList(1, 5, ESTestCase::randomLong); String field = randomAlphaOfLength(5); @@ -754,6 +754,7 @@ public void testBasicQueryWithMultiValues() throws IOException { ); } + @AwaitsFix(bugUrl = "Test disabled while merging fields API in") public void testBasicQueryWithMultiValuesAndMultiPathAndMultiDoc() throws IOException { // formatter will leave first argument as is, but fold the following on a line index( @@ -812,6 +813,7 @@ public void testBasicQueryWithMultiValuesAndMultiPathAndMultiDoc() throws IOExce ); } + @AwaitsFix(bugUrl = "Test disabled while merging fields API in") public void testFilteringQueryWithMultiValuesAndWithout() throws IOException { index("{\"a\": [2, 3, 4, 5]}", "{\"a\": 6}", "{\"a\": [7, 8]}"); String mode = randomMode(); @@ -841,11 +843,10 @@ public void testBasicTranslateQueryWithFilter() throws IOException { Map response = runTranslateSql(query("SELECT * FROM test").filter("{\"match\": {\"test\": \"foo\"}}").toString()); assertEquals(response.get("size"), 1000); + assertFalse((Boolean) response.get("_source")); @SuppressWarnings("unchecked") - Map source = (Map) response.get("_source"); - assertNotNull(source); - assertEquals(emptyList(), source.get("excludes")); - assertEquals(singletonList("test"), source.get("includes")); + List> source = (List>) response.get("fields"); + assertEquals(singletonList(singletonMap("field", "test")), source); @SuppressWarnings("unchecked") Map query = (Map) response.get("query"); @@ -882,7 +883,7 @@ public void testTranslateQueryWithGroupByAndHaving() throws IOException { assertEquals(response.get("size"), 0); assertEquals(false, response.get("_source")); - assertEquals("_none_", response.get("stored_fields")); + assertNull(response.get("stored_fields")); @SuppressWarnings("unchecked") Map aggregations = (Map) response.get("aggregations"); @@ -1099,6 +1100,7 @@ private void executeQueryWithNextPage(String format, String expectedHeader, Stri assertEquals(0, getNumberOfSearchContexts(client(), "test")); } + @AwaitsFix(bugUrl = "Test disabled while merging fields API in") public void testMultiValueQueryText() throws IOException { index( "{" diff --git a/x-pack/plugin/sql/qa/server/src/main/resources/datetime.csv-spec b/x-pack/plugin/sql/qa/server/src/main/resources/datetime.csv-spec index ec80d4669c0d4..3263dc8f7f747 100644 --- a/x-pack/plugin/sql/qa/server/src/main/resources/datetime.csv-spec +++ b/x-pack/plugin/sql/qa/server/src/main/resources/datetime.csv-spec @@ -153,6 +153,16 @@ SELECT WEEK(birth_date) week, birth_date FROM test_emp WHERE WEEK(birth_date) > 2 |1953-01-07T00:00:00.000Z ; +minuteOfDayFilterEquality +SELECT MINUTE_OF_DAY(CONCAT(CONCAT('2021-01-22T14:26:06.', (salary % 2)::text), 'Z')::datetime) AS min_of_day +FROM test_emp WHERE min_of_day = 866 LIMIT 2; + + min_of_day:i +--------------- +866 +866 +; + selectAddWithDateTime schema::dt_year:s|dt_quarter:s|dt_month:s|dt_week:s|dt_day:s|dt_hours:s|dt_min:s|dt_sec:s|dt_millis:s|dt_mcsec:s|dt_nsec:s SELECT DATE_ADD('year', 10, '2019-09-04T11:22:33.123Z'::datetime)::string as dt_year, DATE_ADD('quarter', -10, '2019-09-04T11:22:33.123Z'::datetime)::string as dt_quarter, DATE_ADD('month', 20, '2019-09-04T11:22:33.123Z'::datetime)::string as dt_month, diff --git a/x-pack/plugin/sql/qa/server/src/main/resources/docs/geo.csv-spec b/x-pack/plugin/sql/qa/server/src/main/resources/docs/geo.csv-spec index 899147fd3e6d9..d10890e227e44 100644 --- a/x-pack/plugin/sql/qa/server/src/main/resources/docs/geo.csv-spec +++ b/x-pack/plugin/sql/qa/server/src/main/resources/docs/geo.csv-spec @@ -15,7 +15,7 @@ selectAsWKT SELECT city, ST_AsWKT(location) location FROM "geo" WHERE city = 'Amsterdam'; city:s | location:s -Amsterdam |POINT (4.850311987102032 52.347556999884546) +Amsterdam |POINT (4.850312 52.347557) // end::aswkt ; diff --git a/x-pack/plugin/sql/qa/server/src/main/resources/geo/geosql.csv-spec b/x-pack/plugin/sql/qa/server/src/main/resources/geo/geosql.csv-spec index c9d7bc85448de..391f0effc6154 100644 --- a/x-pack/plugin/sql/qa/server/src/main/resources/geo/geosql.csv-spec +++ b/x-pack/plugin/sql/qa/server/src/main/resources/geo/geosql.csv-spec @@ -32,72 +32,72 @@ shape | GEOMETRY | shape selectAllPointsAsStrings SELECT city, CAST(location AS STRING) location, CAST(location_no_dv AS STRING) location_no_dv, CAST(geoshape AS STRING) geoshape, CAST(shape AS STRING) shape, region FROM "geo" ORDER BY "city"; - city:s | location:s | location_no_dv:s | geoshape:s | shape:s | region:s -Amsterdam |POINT (4.850311987102032 52.347556999884546) |POINT (4.850312 52.347557) |POINT (4.850312 52.347557 2.0) |POINT (4.850312 52.347557 2.0) |Europe -Berlin |POINT (13.390888944268227 52.48670099303126) |POINT (13.390889 52.486701) |POINT (13.390889 52.486701 34.0) |POINT (13.390889 52.486701 34.0) |Europe -Chicago |POINT (-87.63787407428026 41.888782968744636) |POINT (-87.637874 41.888783) |POINT (-87.637874 41.888783 181.0) |POINT (-87.637874 41.888783 181.0) |Americas -Hong Kong |POINT (114.18392493389547 22.28139698971063) |POINT (114.183925 22.281397) |POINT (114.183925 22.281397 552.0) |POINT (114.183925 22.281397 552.0) |Asia -London |POINT (-0.12167204171419144 51.51087098289281)|POINT (-0.121672 51.510871) |POINT (-0.121672 51.510871 11.0) |POINT (-0.121672 51.510871 11.0) |Europe -Mountain View |POINT (-122.08384302444756 37.38648299127817) |POINT (-122.083843 37.386483) |POINT (-122.083843 37.386483 30.0) |POINT (-122.083843 37.386483 30.0) |Americas -Munich |POINT (11.537504978477955 48.14632098656148) |POINT (11.537505 48.146321) |POINT (11.537505 48.146321 519.0) |POINT (11.537505 48.146321 519.0) |Europe -New York |POINT (-73.9900270756334 40.74517097789794) |POINT (-73.990027 40.745171) |POINT (-73.990027 40.745171 10.0) |POINT (-73.990027 40.745171 10.0) |Americas -Paris |POINT (2.3517729341983795 48.84553796611726) |POINT (2.351773 48.845538) |POINT (2.351773 48.845538 35.0) |POINT (2.351773 48.845538 35.0) |Europe -Phoenix |POINT (-111.97350500151515 33.37624196894467) |POINT (-111.973505 33.376242) |POINT (-111.973505 33.376242 331.0)|POINT (-111.973505 33.376242 331.0)|Americas -San Francisco |POINT (-122.39422800019383 37.789540970698) |POINT (-122.394228 37.789541) |POINT (-122.394228 37.789541 16.0) |POINT (-122.394228 37.789541 16.0) |Americas -Seoul |POINT (127.06085099838674 37.50913198571652) |POINT (127.060851 37.509132) |POINT (127.060851 37.509132 38.0) |POINT (127.060851 37.509132 38.0) |Asia -Singapore |POINT (103.8555349688977 1.2958679627627134) |POINT (103.855535 1.295868) |POINT (103.855535 1.295868 15.0) |POINT (103.855535 1.295868 15.0) |Asia -Sydney |POINT (151.20862897485495 -33.863385021686554)|POINT (151.208629 -33.863385) |POINT (151.208629 -33.863385 100.0)|POINT (151.208629 -33.863385 100.0)|Asia -Tokyo |POINT (139.76402222178876 35.66961596254259) |POINT (139.76402225 35.669616)|POINT (139.76402225 35.669616 40.0)|POINT (139.76402225 35.669616 40.0)|Asia + city:s | location:s | location_no_dv:s | geoshape:s | shape:s | region:s +Amsterdam |POINT (4.850312 52.347557) |POINT (4.850312 52.347557) |POINT (4.850312 52.347557 2.0) |POINT (4.850312 52.347557 2.0) |Europe +Berlin |POINT (13.390889 52.486701) |POINT (13.390889 52.486701) |POINT (13.390889 52.486701 34.0) |POINT (13.390889 52.486701 34.0) |Europe +Chicago |POINT (-87.637874 41.888783) |POINT (-87.637874 41.888783) |POINT (-87.637874 41.888783 181.0) |POINT (-87.637874 41.888783 181.0) |Americas +Hong Kong |POINT (114.183925 22.281397) |POINT (114.183925 22.281397) |POINT (114.183925 22.281397 552.0) |POINT (114.183925 22.281397 552.0) |Asia +London |POINT (-0.121672 51.510871) |POINT (-0.121672 51.510871) |POINT (-0.121672 51.510871 11.0) |POINT (-0.121672 51.510871 11.0) |Europe +Mountain View |POINT (-122.083843 37.386483) |POINT (-122.083843 37.386483) |POINT (-122.083843 37.386483 30.0) |POINT (-122.083843 37.386483 30.0) |Americas +Munich |POINT (11.537505 48.146321) |POINT (11.537505 48.146321) |POINT (11.537505 48.146321 519.0) |POINT (11.537505 48.146321 519.0) |Europe +New York |POINT (-73.990027 40.745171) |POINT (-73.990027 40.745171) |POINT (-73.990027 40.745171 10.0) |POINT (-73.990027 40.745171 10.0) |Americas +Paris |POINT (2.351773 48.845538) |POINT (2.351773 48.845538) |POINT (2.351773 48.845538 35.0) |POINT (2.351773 48.845538 35.0) |Europe +Phoenix |POINT (-111.973505 33.376242) |POINT (-111.973505 33.376242) |POINT (-111.973505 33.376242 331.0)|POINT (-111.973505 33.376242 331.0)|Americas +San Francisco |POINT (-122.394228 37.789541) |POINT (-122.394228 37.789541) |POINT (-122.394228 37.789541 16.0) |POINT (-122.394228 37.789541 16.0) |Americas +Seoul |POINT (127.060851 37.509132) |POINT (127.060851 37.509132) |POINT (127.060851 37.509132 38.0) |POINT (127.060851 37.509132 38.0) |Asia +Singapore |POINT (103.855535 1.295868) |POINT (103.855535 1.295868) |POINT (103.855535 1.295868 15.0) |POINT (103.855535 1.295868 15.0) |Asia +Sydney |POINT (151.208629 -33.863385) |POINT (151.208629 -33.863385) |POINT (151.208629 -33.863385 100.0)|POINT (151.208629 -33.863385 100.0)|Asia +Tokyo |POINT (139.76402225 35.669616)|POINT (139.76402225 35.669616)|POINT (139.76402225 35.669616 40.0)|POINT (139.76402225 35.669616 40.0)|Asia ; // TODO: Both shape and location contain the same data for now, we should change it later to make things more interesting selectAllPointsAsWKT SELECT city, ST_ASWKT(location) location_wkt, ST_ASWKT(geoshape) geoshape_wkt, region FROM "geo" ORDER BY "city"; - city:s | location_wkt:s | geoshape_wkt:s | region:s -Amsterdam |POINT (4.850311987102032 52.347556999884546) |POINT (4.850312 52.347557 2.0) |Europe -Berlin |POINT (13.390888944268227 52.48670099303126) |POINT (13.390889 52.486701 34.0) |Europe -Chicago |POINT (-87.63787407428026 41.888782968744636) |POINT (-87.637874 41.888783 181.0) |Americas -Hong Kong |POINT (114.18392493389547 22.28139698971063) |POINT (114.183925 22.281397 552.0) |Asia -London |POINT (-0.12167204171419144 51.51087098289281)|POINT (-0.121672 51.510871 11.0) |Europe -Mountain View |POINT (-122.08384302444756 37.38648299127817) |POINT (-122.083843 37.386483 30.0) |Americas -Munich |POINT (11.537504978477955 48.14632098656148) |POINT (11.537505 48.146321 519.0) |Europe -New York |POINT (-73.9900270756334 40.74517097789794) |POINT (-73.990027 40.745171 10.0) |Americas -Paris |POINT (2.3517729341983795 48.84553796611726) |POINT (2.351773 48.845538 35.0) |Europe -Phoenix |POINT (-111.97350500151515 33.37624196894467) |POINT (-111.973505 33.376242 331.0) |Americas -San Francisco |POINT (-122.39422800019383 37.789540970698) |POINT (-122.394228 37.789541 16.0) |Americas -Seoul |POINT (127.06085099838674 37.50913198571652) |POINT (127.060851 37.509132 38.0) |Asia -Singapore |POINT (103.8555349688977 1.2958679627627134) |POINT (103.855535 1.295868 15.0) |Asia -Sydney |POINT (151.20862897485495 -33.863385021686554)|POINT (151.208629 -33.863385 100.0) |Asia -Tokyo |POINT (139.76402222178876 35.66961596254259) |POINT (139.76402225 35.669616 40.0) |Asia + city:s | location_wkt:s | geoshape_wkt:s | region:s +Amsterdam |POINT (4.850312 52.347557) |POINT (4.850312 52.347557 2.0) |Europe +Berlin |POINT (13.390889 52.486701) |POINT (13.390889 52.486701 34.0) |Europe +Chicago |POINT (-87.637874 41.888783) |POINT (-87.637874 41.888783 181.0) |Americas +Hong Kong |POINT (114.183925 22.281397) |POINT (114.183925 22.281397 552.0) |Asia +London |POINT (-0.121672 51.510871) |POINT (-0.121672 51.510871 11.0) |Europe +Mountain View |POINT (-122.083843 37.386483) |POINT (-122.083843 37.386483 30.0) |Americas +Munich |POINT (11.537505 48.146321) |POINT (11.537505 48.146321 519.0) |Europe +New York |POINT (-73.990027 40.745171) |POINT (-73.990027 40.745171 10.0) |Americas +Paris |POINT (2.351773 48.845538) |POINT (2.351773 48.845538 35.0) |Europe +Phoenix |POINT (-111.973505 33.376242) |POINT (-111.973505 33.376242 331.0)|Americas +San Francisco |POINT (-122.394228 37.789541) |POINT (-122.394228 37.789541 16.0) |Americas +Seoul |POINT (127.060851 37.509132) |POINT (127.060851 37.509132 38.0) |Asia +Singapore |POINT (103.855535 1.295868) |POINT (103.855535 1.295868 15.0) |Asia +Sydney |POINT (151.208629 -33.863385) |POINT (151.208629 -33.863385 100.0)|Asia +Tokyo |POINT (139.76402225 35.669616)|POINT (139.76402225 35.669616 40.0)|Asia ; selectWithAsWKTInWhere SELECT city, ST_ASWKT(location) location_wkt, region FROM "geo" WHERE LOCATE('114', ST_ASWKT(location)) > 0 ORDER BY "city"; - city:s | location_wkt:s | region:s -Hong Kong |POINT (114.18392493389547 22.28139698971063)|Asia + city:s | location_wkt:s | region:s +Hong Kong |POINT (114.183925 22.281397)|Asia ; selectAllPointsOrderByLonFromAsWKT SELECT city, SUBSTRING(ST_ASWKT(location), 8, LOCATE(' ', ST_ASWKT(location), 8) - 8) lon FROM "geo" ORDER BY lon; city:s | lon:s -London |-0.12167204171419144 -Phoenix |-111.97350500151515 -Mountain View |-122.08384302444756 -San Francisco |-122.39422800019383 -New York |-73.9900270756334 -Chicago |-87.63787407428026 -Singapore |103.8555349688977 -Munich |11.537504978477955 -Hong Kong |114.18392493389547 -Seoul |127.06085099838674 -Berlin |13.390888944268227 -Tokyo |139.76402222178876 -Sydney |151.20862897485495 -Paris |2.3517729341983795 -Amsterdam |4.850311987102032 +London |-0.121672 +Phoenix |-111.973505 +Mountain View |-122.083843 +San Francisco |-122.394228 +New York |-73.990027 +Chicago |-87.637874 +Singapore |103.855535 +Munich |11.537505 +Hong Kong |114.183925 +Seoul |127.060851 +Berlin |13.390889 +Tokyo |139.76402225 +Sydney |151.208629 +Paris |2.351773 +Amsterdam |4.850312 ; selectAllPointsGroupByHemisphereFromAsWKT @@ -157,11 +157,11 @@ selectCitiesByDistance SELECT region, city, ST_Distance(location, ST_WktToSQL('POINT (-71 42)')) distance FROM geo WHERE distance < 5000000 ORDER BY region, city; region:s | city:s | distance:d -Americas |Chicago |1373941.5140200066 -Americas |Mountain View |4335936.909375596 -Americas |New York |285839.6579622518 -Americas |Phoenix |3692895.0346903414 -Americas |San Francisco |4343565.010996301 +Americas |Chicago |1373941.5075370357 +Americas |Mountain View |4335936.907008218 +Americas |New York |285839.6512191343 +Americas |Phoenix |3692895.0329883597 +Americas |San Francisco |4343565.009715615 ; selectCitiesByDistanceFloored @@ -267,27 +267,27 @@ SELECT COUNT(*) cnt, FLOOR(ST_Y(location)/45) north, FLOOR(ST_X(location)/90) ea selectFilterByXOfLocation SELECT city, ST_X(geoshape) x, ST_Y(geoshape) y, ST_Z(geoshape) z, ST_X(location) lx, ST_Y(location) ly FROM geo WHERE lx > 0 ORDER BY ly; - city:s | x:d | y:d | z:d | lx:d | ly:d -Sydney |151.208629 |-33.863385 |100.0 |151.20862897485495|-33.863385021686554 -Singapore |103.855535 |1.295868 |15.0 |103.8555349688977 |1.2958679627627134 -Hong Kong |114.183925 |22.281397 |552.0 |114.18392493389547|22.28139698971063 -Tokyo |139.76402225 |35.669616 |40.0 |139.76402222178876|35.66961596254259 -Seoul |127.060851 |37.509132 |38.0 |127.06085099838674|37.50913198571652 -Munich |11.537505 |48.146321 |519.0 |11.537504978477955|48.14632098656148 -Paris |2.351773 |48.845538 |35.0 |2.3517729341983795|48.84553796611726 -Amsterdam |4.850312 |52.347557 |2.0 |4.850311987102032 |52.347556999884546 -Berlin |13.390889 |52.486701 |34.0 |13.390888944268227|52.48670099303126 + city:s | x:d | y:d | z:d | lx:d | ly:d +Sydney |151.208629 |-33.863385 |100.0 |151.208629 |-33.863385 +Singapore |103.855535 |1.295868 |15.0 |103.855535 |1.295868 +Hong Kong |114.183925 |22.281397 |552.0 |114.183925 |22.281397 +Tokyo |139.76402225 |35.669616 |40.0 |139.76402225 |35.669616 +Seoul |127.060851 |37.509132 |38.0 |127.060851 |37.509132 +Munich |11.537505 |48.146321 |519.0 |11.537505 |48.146321 +Paris |2.351773 |48.845538 |35.0 |2.351773 |48.845538 +Amsterdam |4.850312 |52.347557 |2.0 |4.850312 |52.347557 +Berlin |13.390889 |52.486701 |34.0 |13.390889 |52.486701 ; selectFilterByRegionPoint SELECT city, region, ST_X(location) x FROM geo WHERE ST_X(ST_WKTTOSQL(region_point)) < 0 ORDER BY x; city:s | region:s | x:d -San Francisco |Americas |-122.39422800019383 -Mountain View |Americas |-122.08384302444756 -Phoenix |Americas |-111.97350500151515 -Chicago |Americas |-87.63787407428026 -New York |Americas |-73.9900270756334 +San Francisco |Americas |-122.394228 +Mountain View |Americas |-122.083843 +Phoenix |Americas |-111.973505 +Chicago |Americas |-87.637874 +New York |Americas |-73.990027 ; selectLargeLat diff --git a/x-pack/plugin/sql/sql-action/src/test/java/org/elasticsearch/xpack/sql/action/SqlQueryResponseTests.java b/x-pack/plugin/sql/sql-action/src/test/java/org/elasticsearch/xpack/sql/action/SqlQueryResponseTests.java index de29df32f1a29..d3e14f5a00a52 100644 --- a/x-pack/plugin/sql/sql-action/src/test/java/org/elasticsearch/xpack/sql/action/SqlQueryResponseTests.java +++ b/x-pack/plugin/sql/sql-action/src/test/java/org/elasticsearch/xpack/sql/action/SqlQueryResponseTests.java @@ -54,7 +54,8 @@ public static SqlQueryResponse createRandomInstance(String cursor, Mode mode, bo if (randomBoolean()) { columns = new ArrayList<>(columnCount); for (int i = 0; i < columnCount; i++) { - columns.add(new ColumnInfo(randomAlphaOfLength(10), randomAlphaOfLength(10), randomAlphaOfLength(10), randomInt(25))); + columns.add(new ColumnInfo(randomAlphaOfLength(10), randomAlphaOfLength(10), randomAlphaOfLength(10), + randomBoolean() ? null : randomInt(25))); } } diff --git a/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/ColumnInfo.java b/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/ColumnInfo.java index 1efc1820a4170..ecb22a4f0274b 100644 --- a/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/ColumnInfo.java +++ b/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/proto/ColumnInfo.java @@ -107,7 +107,7 @@ public String esType() { /** * Used by JDBC */ - public int displaySize() { + public Integer displaySize() { return displaySize; } diff --git a/x-pack/plugin/sql/src/internalClusterTest/java/org/elasticsearch/xpack/sql/action/SqlTranslateActionIT.java b/x-pack/plugin/sql/src/internalClusterTest/java/org/elasticsearch/xpack/sql/action/SqlTranslateActionIT.java index a32d4deb55e88..a4c30bb35aebe 100644 --- a/x-pack/plugin/sql/src/internalClusterTest/java/org/elasticsearch/xpack/sql/action/SqlTranslateActionIT.java +++ b/x-pack/plugin/sql/src/internalClusterTest/java/org/elasticsearch/xpack/sql/action/SqlTranslateActionIT.java @@ -9,10 +9,12 @@ import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.elasticsearch.search.fetch.subphase.FetchSourceContext; import org.elasticsearch.search.fetch.subphase.FieldAndFormat; import org.elasticsearch.search.sort.SortBuilders; +import java.util.ArrayList; +import java.util.List; + import static java.util.Collections.singletonList; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; @@ -32,12 +34,19 @@ public void testSqlTranslateAction() { SqlTranslateResponse response = new SqlTranslateRequestBuilder(client(), SqlTranslateAction.INSTANCE) .query("SELECT " + columns + " FROM test ORDER BY count").get(); SearchSourceBuilder source = response.source(); - FetchSourceContext fetch = source.fetchSource(); - assertTrue(fetch.fetchSource()); - assertArrayEquals(new String[] { "data", "count" }, fetch.includes()); - assertEquals( - singletonList(new FieldAndFormat("date", "epoch_millis")), - source.docValueFields()); + List actualFields = source.fetchFields(); + List expectedFields = new ArrayList<>(3); + if (columnOrder) { + expectedFields.add(new FieldAndFormat("data", null)); + expectedFields.add(new FieldAndFormat("count", null)); + expectedFields.add(new FieldAndFormat("date", "epoch_millis")); + } else { + expectedFields.add(new FieldAndFormat("date", "epoch_millis")); + expectedFields.add(new FieldAndFormat("data", null)); + expectedFields.add(new FieldAndFormat("count", null)); + } + + assertEquals(expectedFields, actualFields); assertEquals(singletonList(SortBuilders.fieldSort("count").missing("_last").unmappedType("long")), source.sorts()); } } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Verifier.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Verifier.java index 066c8c10a0313..41a2be89b034f 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Verifier.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Verifier.java @@ -397,9 +397,9 @@ private static boolean checkGroupByHaving(LogicalPlan p, Set localFailu Expressions.names(unsupported))); groupingFailures.add(a); return false; + } } } - } return true; } @@ -668,11 +668,20 @@ private static void checkGroupingFunctionTarget(GroupingFunction f, Set private static void checkFilterOnAggs(LogicalPlan p, Set localFailures, AttributeMap attributeRefs) { if (p instanceof Filter) { Filter filter = (Filter) p; - if ((filter.child() instanceof Aggregate) == false) { + LogicalPlan filterChild = filter.child(); + if (filterChild instanceof Aggregate == false) { filter.condition().forEachDown(Expression.class, e -> { if (Functions.isAggregate(attributeRefs.getOrDefault(e, e))) { - localFailures.add( - fail(e, "Cannot use WHERE filtering on aggregate function [{}], use HAVING instead", Expressions.name(e))); + if (filterChild instanceof Project) { + filter.condition().forEachDown(FieldAttribute.class, + f -> localFailures.add(fail(e, "[{}] field must appear in the GROUP BY clause or in an aggregate function", + Expressions.name(f))) + ); + } else { + localFailures.add(fail(e, "Cannot use WHERE filtering on aggregate function [{}], use HAVING instead", + Expressions.name(e))); + + } } }); } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/PlanExecutor.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/PlanExecutor.java index 4b07b007ce68a..b5a7ccb1194f4 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/PlanExecutor.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/PlanExecutor.java @@ -23,9 +23,9 @@ import org.elasticsearch.xpack.sql.planner.Planner; import org.elasticsearch.xpack.sql.planner.PlanningException; import org.elasticsearch.xpack.sql.proto.SqlTypedParamValue; -import org.elasticsearch.xpack.sql.session.SqlConfiguration; import org.elasticsearch.xpack.sql.session.Cursor; import org.elasticsearch.xpack.sql.session.Cursor.Page; +import org.elasticsearch.xpack.sql.session.SqlConfiguration; import org.elasticsearch.xpack.sql.session.SqlSession; import org.elasticsearch.xpack.sql.stats.Metrics; import org.elasticsearch.xpack.sql.stats.QueryMetric; diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/Querier.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/Querier.java index d2484af60d113..7c5e3b7fef4f0 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/Querier.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/Querier.java @@ -84,13 +84,13 @@ import static java.util.Collections.singletonList; import static org.elasticsearch.action.ActionListener.wrap; +import static org.elasticsearch.xpack.ql.execution.search.QlSourceBuilder.SWITCH_TO_FIELDS_API_VERSION; import static org.elasticsearch.xpack.ql.execution.search.extractor.AbstractFieldHitExtractor.MultiValueHandling.EXTRACT_ARRAY; import static org.elasticsearch.xpack.ql.execution.search.extractor.AbstractFieldHitExtractor.MultiValueHandling.EXTRACT_ONE; import static org.elasticsearch.xpack.ql.execution.search.extractor.AbstractFieldHitExtractor.MultiValueHandling.FAIL_IF_MULTIVALUE; // TODO: add retry/back-off public class Querier { - private static final Logger log = LogManager.getLogger(Querier.class); private final PlanExecutor planExecutor; @@ -147,12 +147,17 @@ public void query(List output, QueryContainer query, String index, Ac public static SearchRequest prepareRequest(Client client, SearchSourceBuilder source, TimeValue timeout, boolean includeFrozen, String... indices) { - return client.prepareSearch(indices) - // always track total hits accurately - .setTrackTotalHits(true).setAllowPartialSearchResults(false).setSource(source).setTimeout(timeout) - .setIndicesOptions( - includeFrozen ? IndexResolver.FIELD_CAPS_FROZEN_INDICES_OPTIONS : IndexResolver.FIELD_CAPS_INDICES_OPTIONS) - .request(); + source.trackTotalHits(true); + source.timeout(timeout); + + SearchRequest searchRequest = new SearchRequest(SWITCH_TO_FIELDS_API_VERSION); + searchRequest.indices(indices); + searchRequest.source(source); + searchRequest.allowPartialSearchResults(false); + searchRequest.indicesOptions( + includeFrozen ? IndexResolver.FIELD_CAPS_FROZEN_INDICES_OPTIONS : IndexResolver.FIELD_CAPS_INDICES_OPTIONS); + + return searchRequest; } protected static void logSearchResponse(SearchResponse response, Logger logger) { @@ -491,17 +496,14 @@ protected void handleResponse(SearchResponse response, ActionListener list private HitExtractor createExtractor(FieldExtraction ref) { if (ref instanceof SearchHitFieldRef) { SearchHitFieldRef f = (SearchHitFieldRef) ref; - return new FieldHitExtractor(f.name(), f.fullFieldName(), f.getDataType(), cfg.zoneId(), f.useDocValue(), f.hitName(), - f.asArray() - ? EXTRACT_ARRAY - : (multiValueFieldLeniency ? EXTRACT_ONE : FAIL_IF_MULTIVALUE)); + return new FieldHitExtractor(f.name(), f.getDataType(), cfg.zoneId(), f.hitName(), + f.asArray() ? EXTRACT_ARRAY : (multiValueFieldLeniency ? EXTRACT_ONE : FAIL_IF_MULTIVALUE)); } // TODO: this is dead code; remove? (and possibly simplify FieldReference out?) if (ref instanceof ScriptFieldRef) { ScriptFieldRef f = (ScriptFieldRef) ref; - return new FieldHitExtractor(f.name(), null, cfg.zoneId(), true, - multiValueFieldLeniency ? EXTRACT_ONE : FAIL_IF_MULTIVALUE); + return new FieldHitExtractor(f.name(), null, cfg.zoneId(), multiValueFieldLeniency ? EXTRACT_ONE : FAIL_IF_MULTIVALUE); } if (ref instanceof ComputedRef) { diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/SourceGenerator.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/SourceGenerator.java index 93dfba550ea52..1e9a72c67fa70 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/SourceGenerator.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/SourceGenerator.java @@ -10,7 +10,6 @@ import org.elasticsearch.search.aggregations.AggregationBuilder; import org.elasticsearch.search.aggregations.bucket.composite.CompositeAggregationBuilder; import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.elasticsearch.search.fetch.StoredFieldsContext; import org.elasticsearch.search.fetch.subphase.FetchSourceContext; import org.elasticsearch.search.sort.FieldSortBuilder; import org.elasticsearch.search.sort.NestedSortBuilder; @@ -25,9 +24,6 @@ import org.elasticsearch.xpack.sql.querydsl.container.QueryContainer; import org.elasticsearch.xpack.sql.querydsl.container.ScoreSort; -import java.util.List; - -import static java.util.Collections.singletonList; import static org.elasticsearch.index.query.QueryBuilders.boolQuery; import static org.elasticsearch.search.sort.SortBuilders.fieldSort; import static org.elasticsearch.search.sort.SortBuilders.scoreSort; @@ -37,8 +33,6 @@ public abstract class SourceGenerator { private SourceGenerator() {} - private static final List NO_STORED_FIELD = singletonList(StoredFieldsContext._NONE_); - public static SearchSourceBuilder sourceBuilder(QueryContainer container, QueryBuilder filter, Integer size) { QueryBuilder finalQuery = null; // add the source @@ -64,7 +58,6 @@ public static SearchSourceBuilder sourceBuilder(QueryContainer container, QueryB // NB: the sortBuilder takes care of eliminating duplicates container.fields().forEach(f -> f.v1().collectFields(sortBuilder)); sortBuilder.build(source); - optimize(sortBuilder, source); // add the aggs (if present) AggregationBuilder aggBuilder = container.aggs().asAggBuilder(); @@ -166,29 +159,15 @@ private static void sorting(QueryContainer container, SearchSourceBuilder source } } - private static void optimize(QlSourceBuilder sqlSource, SearchSourceBuilder builder) { - if (sqlSource.noSource()) { - disableSource(builder); - } - } - private static void optimize(QueryContainer query, SearchSourceBuilder builder) { // if only aggs are needed, don't retrieve any docs and remove scoring if (query.isAggsOnly()) { builder.size(0); builder.trackScores(false); - // disable source fetching (only doc values are used) - disableSource(builder); } if (query.shouldTrackHits()) { builder.trackTotalHits(true); } - } - - private static void disableSource(SearchSourceBuilder builder) { builder.fetchSource(FetchSourceContext.DO_NOT_FETCH_SOURCE); - if (builder.storedFields() == null) { - builder.storedFields(NO_STORED_FIELD); - } } } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/FieldHitExtractor.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/FieldHitExtractor.java index e0d67aaa947ca..0cddd336ec5c5 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/FieldHitExtractor.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/FieldHitExtractor.java @@ -42,18 +42,16 @@ public class FieldHitExtractor extends AbstractFieldHitExtractor { */ static final String NAME = "f"; - public FieldHitExtractor(String name, DataType dataType, ZoneId zoneId, boolean useDocValue, - MultiValueHandling multiValueHandling) { - super(name, dataType, zoneId, useDocValue, multiValueHandling); + public FieldHitExtractor(String name, DataType dataType, ZoneId zoneId, MultiValueHandling multiValueHandling) { + super(name, dataType, zoneId, multiValueHandling); } - public FieldHitExtractor(String name, DataType dataType, ZoneId zoneId, boolean useDocValue) { - super(name, dataType, zoneId, useDocValue); + public FieldHitExtractor(String name, DataType dataType, ZoneId zoneId) { + super(name, dataType, zoneId); } - public FieldHitExtractor(String name, String fullFieldName, DataType dataType, ZoneId zoneId, boolean useDocValue, String hitName, - MultiValueHandling multiValueHandling) { - super(name, fullFieldName, dataType, zoneId, useDocValue, hitName, multiValueHandling); + public FieldHitExtractor(String name, DataType dataType, ZoneId zoneId, String hitName, MultiValueHandling multiValueHandling) { + super(name, dataType, zoneId, hitName, multiValueHandling); } public FieldHitExtractor(StreamInput in) throws IOException { @@ -92,19 +90,15 @@ private boolean isGeoPointArray(List list) { return list.get(0) instanceof Number; } - - @Override - protected boolean isFromDocValuesOnly(DataType dataType) { - return SqlDataTypes.isFromDocValuesOnly(dataType); - } - @Override protected Object unwrapCustomValue(Object values) { DataType dataType = dataType(); if (dataType == GEO_POINT) { try { - GeoPoint geoPoint = GeoUtils.parseGeoPoint(values, true); + @SuppressWarnings("unchecked") + Map map = (Map) values; + GeoPoint geoPoint = GeoUtils.parseGeoPoint(map.get("coordinates"), true); return new GeoShape(geoPoint.lon(), geoPoint.lat()); } catch (ElasticsearchParseException ex) { throw new SqlIllegalArgumentException("Cannot parse geo_point value [{}] (returned by [{}])", values, fieldName()); diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/MinuteOfDay.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/MinuteOfDay.java index a9fb385547bb0..ef83c5c0a2471 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/MinuteOfDay.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/MinuteOfDay.java @@ -34,6 +34,6 @@ protected MinuteOfDay replaceChild(Expression newChild) { @Override public String dateTimeFormat() { - throw new UnsupportedOperationException("is there a format for it?"); + return null; } } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/TransportSqlQueryAction.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/TransportSqlQueryAction.java index 4ff9ef54d8452..e2f067ed95f3f 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/TransportSqlQueryAction.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/TransportSqlQueryAction.java @@ -6,7 +6,10 @@ */ package org.elasticsearch.xpack.sql.plugin; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionListenerResponseHandler; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.HandledTransportAction; import org.elasticsearch.cluster.service.ClusterService; @@ -43,15 +46,18 @@ import static java.util.Collections.unmodifiableList; import static org.elasticsearch.action.ActionListener.wrap; +import static org.elasticsearch.xpack.ql.plugin.TransportActionUtils.executeRequestWithRetryAttempt; import static org.elasticsearch.xpack.sql.plugin.Transports.clusterName; import static org.elasticsearch.xpack.sql.plugin.Transports.username; import static org.elasticsearch.xpack.sql.proto.Mode.CLI; public class TransportSqlQueryAction extends HandledTransportAction { + private static final Logger log = LogManager.getLogger(TransportSqlQueryAction.class); private final SecurityContext securityContext; private final ClusterService clusterService; private final PlanExecutor planExecutor; private final SqlLicenseChecker sqlLicenseChecker; + private final TransportService transportService; @Inject public TransportSqlQueryAction(Settings settings, ClusterService clusterService, TransportService transportService, @@ -64,19 +70,21 @@ public TransportSqlQueryAction(Settings settings, ClusterService clusterService, this.clusterService = clusterService; this.planExecutor = planExecutor; this.sqlLicenseChecker = sqlLicenseChecker; + this.transportService = transportService; } @Override protected void doExecute(Task task, SqlQueryRequest request, ActionListener listener) { sqlLicenseChecker.checkIfSqlAllowed(request.mode()); - operation(planExecutor, request, listener, username(securityContext), clusterName(clusterService)); + operation(planExecutor, request, listener, username(securityContext), clusterName(clusterService), transportService, + clusterService); } /** * Actual implementation of the action. Statically available to support embedded mode. */ static void operation(PlanExecutor planExecutor, SqlQueryRequest request, ActionListener listener, - String username, String clusterName) { + String username, String clusterName, TransportService transportService, ClusterService clusterService) { // The configuration is always created however when dealing with the next page, only the timeouts are relevant // the rest having default values (since the query is already created) SqlConfiguration cfg = new SqlConfiguration(request.zoneId(), request.fetchSize(), request.requestTimeout(), request.pageTimeout(), @@ -84,8 +92,12 @@ static void operation(PlanExecutor planExecutor, SqlQueryRequest request, Action request.fieldMultiValueLeniency(), request.indexIncludeFrozen()); if (Strings.hasText(request.cursor()) == false) { - planExecutor.sql(cfg, request.query(), request.params(), - wrap(p -> listener.onResponse(createResponseWithSchema(request, p)), listener::onFailure)); + executeRequestWithRetryAttempt(clusterService, listener::onFailure, + onFailure -> planExecutor.sql(cfg, request.query(), request.params(), + wrap(p -> listener.onResponse(createResponseWithSchema(request, p)), onFailure)), + node -> transportService.sendRequest(node, SqlQueryAction.NAME, request, + new ActionListenerResponseHandler<>(listener, SqlQueryResponse::new, ThreadPool.Names.SAME)), + log); } else { Tuple decoded = Cursors.decodeFromStringWithZone(request.cursor()); planExecutor.nextPage(cfg, decoded.v1(), diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/container/QueryContainer.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/container/QueryContainer.java index e13d7fd4788b7..340b044591014 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/container/QueryContainer.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/container/QueryContainer.java @@ -27,7 +27,6 @@ import org.elasticsearch.xpack.ql.querydsl.query.NestedQuery; import org.elasticsearch.xpack.ql.querydsl.query.Query; import org.elasticsearch.xpack.ql.tree.Source; -import org.elasticsearch.xpack.ql.type.DataTypes; import org.elasticsearch.xpack.sql.SqlIllegalArgumentException; import org.elasticsearch.xpack.sql.execution.search.SourceGenerator; import org.elasticsearch.xpack.sql.expression.function.Array; @@ -334,29 +333,6 @@ private FieldExtraction topHitFieldRef(FieldAttribute fieldAttr, boolean asMulti FieldAttribute rootField = fieldAttr; StringBuilder fullFieldName = new StringBuilder(fieldAttr.field().getName()); - // Only if the field is not an alias (in which case it will be taken out from docvalue_fields if it's isAggregatable()), - // go up the tree of parents until a non-object (and non-nested) type of field is found and use that specific parent - // as the field to extract data from, from _source. We do it like this because sub-fields are not in the _source, only - // the root field to which those sub-fields belong to, are. Instead of "text_field.keyword_subfield" for _source extraction, - // we use "text_field", because there is no source for "keyword_subfield". - /* - * "text_field": { - * "type": "text", - * "fields": { - * "keyword_subfield": { - * "type": "keyword" - * } - * } - * } - */ - if (fieldAttr.field().isAlias() == false) { - while (actualField.parent() != null - && actualField.parent().field().getDataType() != DataTypes.OBJECT - && actualField.parent().field().getDataType() != DataTypes.NESTED - && SqlDataTypes.isFromDocValuesOnly(actualField.field().getDataType()) == false) { - actualField = actualField.parent(); - } - } while (rootField.parent() != null) { fullFieldName.insert(0, ".").insert(0, rootField.parent().field().getName()); rootField = rootField.parent(); diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/container/SearchHitFieldRef.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/container/SearchHitFieldRef.java index 0180b4c01f64e..55c5be4614e8f 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/container/SearchHitFieldRef.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/container/SearchHitFieldRef.java @@ -66,11 +66,7 @@ public void collectFields(QlSourceBuilder sourceBuilder) { if (hitName != null) { return; } - if (docValue) { - sourceBuilder.addDocField(name, SqlDataTypes.format(dataType)); - } else { - sourceBuilder.addSourceField(name); - } + sourceBuilder.addFetchField(name, SqlDataTypes.format(dataType)); } @Override diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/VerifierErrorMessagesTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/VerifierErrorMessagesTests.java index 05b47e87837ef..ce380848aebff 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/VerifierErrorMessagesTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/VerifierErrorMessagesTests.java @@ -912,7 +912,24 @@ public void testIifWithDifferentResultAndDefaultValueDataTypes() { public void testAggsInWhere() { assertEquals("1:33: Cannot use WHERE filtering on aggregate function [MAX(int)], use HAVING instead", - error("SELECT MAX(int) FROM test WHERE MAX(int) > 10 GROUP BY bool")); + error("SELECT MAX(int) FROM test WHERE MAX(int) > 10 GROUP BY bool")); + } + + public void testHavingInAggs() { + assertEquals("1:29: [int] field must appear in the GROUP BY clause or in an aggregate function", + error("SELECT int FROM test HAVING MAX(int) = 0")); + + assertEquals("1:35: [int] field must appear in the GROUP BY clause or in an aggregate function", + error("SELECT int FROM test HAVING int = count(1)")); + } + + public void testHavingAsWhere() { + // TODO: this query works, though it normally shouldn't; a check about it could only be enforced if the Filter would be qualified + // (WHERE vs HAVING). Otoh, this "extra flexibility" shouldn't be harmful atp. + accept("SELECT int FROM test HAVING int = 1"); + accept("SELECT int FROM test HAVING SIN(int) + 5 > 5.5"); + // HAVING's expression being AND'ed to WHERE's + accept("SELECT int FROM test WHERE int > 3 HAVING POWER(int, 2) < 100"); } public void testHistogramInFilter() { diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/SqlSourceBuilderTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/SqlSourceBuilderTests.java index ca9de884103ee..1f5922c4d1f35 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/SqlSourceBuilderTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/SqlSourceBuilderTests.java @@ -8,26 +8,26 @@ import org.elasticsearch.script.Script; import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.elasticsearch.search.fetch.subphase.FetchSourceContext; +import org.elasticsearch.search.fetch.subphase.FieldAndFormat; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.ql.execution.search.QlSourceBuilder; -import java.util.Arrays; +import java.util.List; import java.util.Map; import java.util.stream.Collectors; -import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.nullValue; public class SqlSourceBuilderTests extends ESTestCase { + public void testSqlSourceBuilder() { final QlSourceBuilder ssb = new QlSourceBuilder(); final SearchSourceBuilder source = new SearchSourceBuilder(); ssb.trackScores(); - ssb.addSourceField("foo"); - ssb.addSourceField("foo2"); - ssb.addDocField("bar", null); - ssb.addDocField("bar2", null); + ssb.addFetchField("foo", null); + ssb.addFetchField("foo2", "test"); final Script s = new Script("eggplant"); ssb.addScriptField("baz", s); final Script s2 = new Script("potato"); @@ -35,9 +35,16 @@ public void testSqlSourceBuilder() { ssb.build(source); assertTrue(source.trackScores()); - FetchSourceContext fsc = source.fetchSource(); - assertThat(Arrays.asList(fsc.includes()), contains("foo", "foo2")); - assertThat(source.docValueFields().stream().map(ff -> ff.field).collect(Collectors.toList()), contains("bar", "bar2")); + assertNull(source.fetchSource()); + assertNull(source.docValueFields()); + + List fetchFields = source.fetchFields(); + assertThat(fetchFields.size(), equalTo(2)); + assertThat(fetchFields.get(0).field, equalTo("foo")); + assertThat(fetchFields.get(0).format, is(nullValue())); + assertThat(fetchFields.get(1).field, equalTo("foo2")); + assertThat(fetchFields.get(1).format, equalTo("test")); + Map scriptFields = source.scriptFields() .stream() .collect(Collectors.toMap(SearchSourceBuilder.ScriptField::fieldName, SearchSourceBuilder.ScriptField::script)); diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/ComputingExtractorTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/ComputingExtractorTests.java index 1797447ac40b4..561c0551f8b24 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/ComputingExtractorTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/ComputingExtractorTests.java @@ -75,7 +75,7 @@ protected ComputingExtractor mutateInstance(ComputingExtractor instance) throws public void testGet() { String fieldName = randomAlphaOfLength(5); ChainingProcessor extractor = new ChainingProcessor( - new HitExtractorProcessor(new FieldHitExtractor(fieldName, DOUBLE, UTC, true, FAIL_IF_MULTIVALUE)), + new HitExtractorProcessor(new FieldHitExtractor(fieldName, DOUBLE, UTC, FAIL_IF_MULTIVALUE)), new MathProcessor(MathOperation.LOG)); int times = between(1, 1000); diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/FieldHitExtractorTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/FieldHitExtractorTests.java index bc58ecf2b15c5..e2218bc9c1bf4 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/FieldHitExtractorTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/FieldHitExtractorTests.java @@ -6,11 +6,8 @@ */ package org.elasticsearch.xpack.sql.execution.search.extractor; -import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.document.DocumentField; import org.elasticsearch.common.io.stream.Writeable.Reader; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.search.SearchHit; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.ql.QlIllegalArgumentException; @@ -29,28 +26,20 @@ import java.time.ZonedDateTime; import java.util.ArrayList; import java.util.Collections; -import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.StringJoiner; import java.util.function.Supplier; -import static java.lang.String.join; import static java.util.Arrays.asList; -import static java.util.Collections.emptyList; import static java.util.Collections.singletonList; import static java.util.Collections.singletonMap; import static org.elasticsearch.common.time.DateUtils.toMilliSeconds; import static org.elasticsearch.xpack.ql.execution.search.extractor.AbstractFieldHitExtractor.MultiValueHandling.EXTRACT_ARRAY; -import static org.elasticsearch.xpack.ql.execution.search.extractor.AbstractFieldHitExtractor.MultiValueHandling.FAIL_IF_MULTIVALUE; import static org.elasticsearch.xpack.ql.execution.search.extractor.AbstractFieldHitExtractor.MultiValueHandling.EXTRACT_ONE; +import static org.elasticsearch.xpack.ql.execution.search.extractor.AbstractFieldHitExtractor.MultiValueHandling.FAIL_IF_MULTIVALUE; import static org.elasticsearch.xpack.ql.type.DataTypes.DATETIME; import static org.elasticsearch.xpack.ql.type.DataTypes.DATETIME_NANOS; -import static org.elasticsearch.xpack.ql.type.DataTypes.DOUBLE; -import static org.elasticsearch.xpack.ql.type.DataTypes.INTEGER; -import static org.elasticsearch.xpack.ql.type.DataTypes.LONG; -import static org.elasticsearch.xpack.sql.type.SqlDataTypes.GEO_POINT; import static org.elasticsearch.xpack.sql.type.SqlDataTypes.GEO_SHAPE; import static org.elasticsearch.xpack.sql.type.SqlDataTypes.SHAPE; import static org.elasticsearch.xpack.sql.util.DateUtils.UTC; @@ -61,7 +50,7 @@ public class FieldHitExtractorTests extends AbstractSqlWireSerializingTestCase randomFrom(SqlDataTypes.types())), randomValueOtherThan(instance.zoneId(), ESTestCase::randomZone), - randomBoolean(), instance.hitName() + "mutated", - randomFrom(MultiValueHandling.values())); + randomFrom(MultiValueHandling.values()) + ); } public void testGetDottedValueWithDocValues() { @@ -97,7 +85,7 @@ public void testGetDottedValueWithDocValues() { String child = randomAlphaOfLength(5); String fieldName = grandparent + "." + parent + "." + child; - FieldHitExtractor extractor = getFieldHitExtractor(fieldName, true); + FieldHitExtractor extractor = getFieldHitExtractor(fieldName); int times = between(1, 1000); for (int i = 0; i < times; i++) { @@ -114,52 +102,9 @@ public void testGetDottedValueWithDocValues() { } } - public void testGetDottedValueWithSource() throws Exception { - String grandparent = randomAlphaOfLength(5); - String parent = randomAlphaOfLength(5); - String child = randomAlphaOfLength(5); - String fieldName = grandparent + "." + parent + "." + child; - - FieldHitExtractor extractor = getFieldHitExtractor(fieldName, false); - - int times = between(1, 1000); - for (int i = 0; i < times; i++) { - /* We use values that are parsed from json as "equal" to make the - * test simpler. */ - Object value = randomValue(); - SearchHit hit = new SearchHit(1); - XContentBuilder source = JsonXContent.contentBuilder(); - boolean hasGrandparent = randomBoolean(); - boolean hasParent = randomBoolean(); - boolean hasChild = randomBoolean(); - boolean hasSource = hasGrandparent && hasParent && hasChild; - - source.startObject(); - if (hasGrandparent) { - source.startObject(grandparent); - if (hasParent) { - source.startObject(parent); - if (hasChild) { - source.field(child, value); - if (randomBoolean()) { - source.field(fieldName + randomAlphaOfLength(3), value + randomAlphaOfLength(3)); - } - } - source.endObject(); - } - source.endObject(); - } - source.endObject(); - BytesReference sourceRef = BytesReference.bytes(source); - hit.sourceRef(sourceRef); - Object extract = extractor.extract(hit); - assertFieldHitEquals(hasSource ? value : null, extract); - } - } - public void testGetDocValue() { String fieldName = randomAlphaOfLength(5); - FieldHitExtractor extractor = getFieldHitExtractor(fieldName, true); + FieldHitExtractor extractor = getFieldHitExtractor(fieldName); int times = between(1, 1000); for (int i = 0; i < times; i++) { @@ -180,7 +125,7 @@ public void testGetDate() { List documentFieldValues = Collections.singletonList(Long.toString(millis)); DocumentField field = new DocumentField("my_date_field", documentFieldValues); SearchHit hit = new SearchHit(1, null, singletonMap("my_date_field", field), null); - FieldHitExtractor extractor = new FieldHitExtractor("my_date_field", DATETIME, zoneId, true); + FieldHitExtractor extractor = new FieldHitExtractor("my_date_field", DATETIME, zoneId, EXTRACT_ONE); assertEquals(DateUtils.asDateTimeWithMillis(millis, zoneId), extractor.extract(hit)); } @@ -193,683 +138,92 @@ public void testGetDateNanos() { List documentFieldValues = Collections.singletonList(StringUtils.toString(zdt)); DocumentField field = new DocumentField("my_date_nanos_field", documentFieldValues); SearchHit hit = new SearchHit(1, null, singletonMap("my_date_nanos_field", field), null); - FieldHitExtractor extractor = new FieldHitExtractor("my_date_nanos_field", DATETIME_NANOS, zoneId, true); + FieldHitExtractor extractor = new FieldHitExtractor("my_date_nanos_field", DATETIME_NANOS, zoneId, EXTRACT_ONE); assertEquals(zdt, extractor.extract(hit)); } - public void testGetSource() throws IOException { - String fieldName = randomAlphaOfLength(5); - FieldHitExtractor extractor = getFieldHitExtractor(fieldName, false); - - int times = between(1, 1000); - for (int i = 0; i < times; i++) { - /* We use values that are parsed from json as "equal" to make the - * test simpler. */ - Object value = randomValue(); - SearchHit hit = new SearchHit(1); - XContentBuilder source = JsonXContent.contentBuilder(); - source.startObject(); { - source.field(fieldName, value); - if (randomBoolean()) { - source.field(fieldName + "_random_junk", value + "_random_junk"); - } - } - source.endObject(); - BytesReference sourceRef = BytesReference.bytes(source); - hit.sourceRef(sourceRef); - assertFieldHitEquals(value, extractor.extract(hit)); - } - } - public void testToString() { - assertEquals("hit.field@hit@Europe/Berlin", - new FieldHitExtractor("hit.field", null, null, ZoneId.of("Europe/Berlin"), true, "hit", FAIL_IF_MULTIVALUE) - .toString()); + assertEquals( + "hit.field@hit@Europe/Berlin", + new FieldHitExtractor("hit.field", null, ZoneId.of("Europe/Berlin"), "hit", FAIL_IF_MULTIVALUE).toString() + ); } public void testMultiValuedDocValue() { String fieldName = randomAlphaOfLength(5); - FieldHitExtractor fe = getFieldHitExtractor(fieldName, true); + FieldHitExtractor fe = getFieldHitExtractor(fieldName); DocumentField field = new DocumentField(fieldName, asList("a", "b")); SearchHit hit = new SearchHit(1, null, singletonMap(fieldName, field), null); QlIllegalArgumentException ex = expectThrows(QlIllegalArgumentException.class, () -> fe.extract(hit)); - assertThat(ex.getMessage(), is("Cannot return multiple values for field [" + fieldName + "]; " + - "use ARRAY(" + fieldName + ") instead")); + assertThat(ex.getMessage(), is("Arrays (returned by [" + fieldName + "]) are not supported")); } - public void testMultiValuedSourceValue() throws IOException { - String fieldName = randomAlphaOfLength(5); - FieldHitExtractor fe = getFieldHitExtractor(fieldName, false); - SearchHit hit = new SearchHit(1); - XContentBuilder source = JsonXContent.contentBuilder(); - source.startObject(); { - source.field(fieldName, asList("a", "b")); - } - source.endObject(); - BytesReference sourceRef = BytesReference.bytes(source); - hit.sourceRef(sourceRef); - QlIllegalArgumentException ex = expectThrows(QlIllegalArgumentException.class, () -> fe.extract(hit)); - assertThat(ex.getMessage(), is("Cannot return multiple values for field [" + fieldName + "]; " + - "use ARRAY(" + fieldName + ") instead")); - } - - public void testSingleValueArrayInSource() throws IOException { - String fieldName = randomAlphaOfLength(5); - FieldHitExtractor fe = getFieldHitExtractor(fieldName, false); - SearchHit hit = new SearchHit(1); - XContentBuilder source = JsonXContent.contentBuilder(); + public void testExtractSourcePath() throws IOException { + FieldHitExtractor fe = getFieldHitExtractor("a.b.c"); Object value = randomValue(); - source.startObject(); { - source.field(fieldName, Collections.singletonList(value)); - } - source.endObject(); - BytesReference sourceRef = BytesReference.bytes(source); - hit.sourceRef(sourceRef); - assertFieldHitEquals(value, fe.extract(hit)); - } - - public void testExtractSourcePath() { - FieldHitExtractor fe = getFieldHitExtractor("a.b.c", false); - Object value = randomValue(); - Map map = singletonMap("a", singletonMap("b", singletonMap("c", value))); - assertThat(fe.extractFromSource(map), is(value)); - } - - public void testExtractSourceIncorrectPath() { - FieldHitExtractor fe = getFieldHitExtractor("a.b.c.d", false); - Object value = randomNonNullValue(); - Map map = singletonMap("a", singletonMap("b", singletonMap("c", value))); - QlIllegalArgumentException ex = expectThrows(QlIllegalArgumentException.class, () -> fe.extractFromSource(map)); - assertThat(ex.getMessage(), is("Cannot extract value [a.b.c.d] from source")); + DocumentField field = new DocumentField("a.b.c", singletonList(value)); + SearchHit hit = new SearchHit(1, null, null, singletonMap("a.b.c", field), null); + assertThat(fe.extract(hit), is(value)); } public void testMultiValuedSource() { - FieldHitExtractor fe = getFieldHitExtractor("a", false); + FieldHitExtractor fe = getFieldHitExtractor("a"); Object value = randomValue(); - Map map = singletonMap("a", asList(value, value)); - QlIllegalArgumentException ex = expectThrows(QlIllegalArgumentException.class, () -> fe.extractFromSource(map)); - assertThat(ex.getMessage(), is("Cannot return multiple values for field [a]; use ARRAY(a) instead")); + DocumentField field = new DocumentField("a", asList(value, value)); + SearchHit hit = new SearchHit(1, null, null, singletonMap("a", field), null); + QlIllegalArgumentException ex = expectThrows(QlIllegalArgumentException.class, () -> fe.extract(hit)); + assertThat(ex.getMessage(), is("Arrays (returned by [a]) are not supported")); } public void testMultiValuedSourceAllowed() { - FieldHitExtractor fe = new FieldHitExtractor("a", null, UTC, false, EXTRACT_ONE); + FieldHitExtractor fe = new FieldHitExtractor("a", null, UTC, EXTRACT_ONE); Object valueA = randomValue(); Object valueB = randomValue(); - Map map = singletonMap("a", asList(valueA, valueB)); - assertEquals(valueA, fe.extractFromSource(map)); - } - - public void testFieldWithDots() { - FieldHitExtractor fe = getFieldHitExtractor("a.b", false); - Object value = randomValue(); - Map map = singletonMap("a.b", value); - assertEquals(value, fe.extractFromSource(map)); - } - - public void testNestedFieldWithDots() { - FieldHitExtractor fe = getFieldHitExtractor("a.b.c", false); - Object value = randomValue(); - Map map = singletonMap("a", singletonMap("b.c", value)); - assertEquals(value, fe.extractFromSource(map)); - } - - public void testNestedFieldWithDotsWithNestedField() { - FieldHitExtractor fe = getFieldHitExtractor("a.b.c.d", false); - Object value = randomValue(); - Map map = singletonMap("a", singletonMap("b.c", singletonMap("d", value))); - assertEquals(value, fe.extractFromSource(map)); - } - - public void testNestedFieldWithDotsWithNestedFieldWithDots() { - FieldHitExtractor fe = getFieldHitExtractor("a.b.c.d.e", false); - Object value = randomValue(); - Map map = singletonMap("a", singletonMap("b.c", singletonMap("d.e", value))); - assertEquals(value, fe.extractFromSource(map)); - } - - @SuppressWarnings({ "rawtypes", "unchecked" }) - public void testNestedFieldsWithDotsAndRandomHierarchy() { - String[] path = new String[100]; - StringJoiner sj = new StringJoiner("."); - for (int i = 0; i < 100; i++) { - path[i] = randomAlphaOfLength(randomIntBetween(1, 10)); - sj.add(path[i]); - } - MultiValueHandling extractionMode = randomFrom(FAIL_IF_MULTIVALUE, EXTRACT_ONE, EXTRACT_ARRAY); - FieldHitExtractor fe = new FieldHitExtractor(sj.toString(), null, UTC, false, extractionMode); - - List paths = new ArrayList<>(path.length); - int start = 0; - while (start < path.length) { - int end = randomIntBetween(start + 1, path.length); - sj = new StringJoiner("."); - for (int j = start; j < end; j++) { - sj.add(path[j]); - } - paths.add(sj.toString()); - start = end; - } - - /* - * Randomize how many values the field to look for will have (1 - 3). It's not really relevant how many values there are in the list - * but that the list has one element or more than one. - * If it has one value, then randomize the way it's indexed: as a single-value array or not e.g.: "a":"value" or "a":["value"]. - * If it has more than one value, it will always be an array e.g.: "a":["v1","v2","v3"]. - */ - int valuesCount = randomIntBetween(1, 3); - Object value = randomValue(); - if (valuesCount == 1) { - value = randomBoolean() ? singletonList(value) : value; - } else { - value = new ArrayList(valuesCount); - for(int i = 0; i < valuesCount; i++) { - ((List) value).add(randomValue()); - } - } - - // the path to the randomly generated fields path - StringBuilder expected = new StringBuilder(paths.get(paths.size() - 1)); - // the actual value we will be looking for in the test at the end - Map map = singletonMap(paths.get(paths.size() - 1), value); - // build the rest of the path and the expected path to check against in the error message - for (int i = paths.size() - 2; i >= 0; i--) { - map = singletonMap(paths.get(i), randomBoolean() ? singletonList(map) : map); - expected.insert(0, paths.get(i) + "."); - } - - List expectList = value instanceof List ? (List) value : (value == null ? emptyList() : singletonList(value)); - Object expectSingleton = value instanceof List ? ((List) value).get(0) : value; - if (valuesCount == 1) { - // if the number of generated values is 1, just check we return the correct value - assertEquals(extractionMode == EXTRACT_ARRAY ? expectList : expectSingleton, fe.extractFromSource(map)); - } else { - switch (extractionMode) { - case FAIL_IF_MULTIVALUE: - // if we have an array with more than one value in it, check that we throw the correct exception and exception message - final Map map2 = map; - QlIllegalArgumentException ex = expectThrows(QlIllegalArgumentException.class, () -> fe.extractFromSource(map2)); - assertThat(ex.getMessage(), is("Cannot return multiple values for field [" + expected+ "]; " + - "use ARRAY(" + expected + ") instead")); - break; - case EXTRACT_ONE: - assertEquals(expectSingleton, fe.extractFromSource(map)); - break; - case EXTRACT_ARRAY: - assertEquals(expectList, fe.extractFromSource(map)); - break; - default: - fail(); - } - } - } - - public void testExtractSourceIncorrectPathWithFieldWithDots() { - FieldHitExtractor fe = getFieldHitExtractor("a.b.c.d.e", false); - Object value = randomNonNullValue(); - Map map = singletonMap("a", singletonMap("b.c", singletonMap("d", value))); - QlIllegalArgumentException ex = expectThrows(QlIllegalArgumentException.class, () -> fe.extractFromSource(map)); - assertThat(ex.getMessage(), is("Cannot extract value [a.b.c.d.e] from source")); - } - - public void testFieldWithDotsAndCommonPrefix() { - FieldHitExtractor fe1 = getFieldHitExtractor("a.d", false); - FieldHitExtractor fe2 = getFieldHitExtractor("a.b.c", false); - Object value = randomNonNullValue(); - Map map = new HashMap<>(); - map.put("a", singletonMap("d", value)); - map.put("a.b", singletonMap("c", value)); - assertEquals(value, fe1.extractFromSource(map)); - assertEquals(value, fe2.extractFromSource(map)); - } - - public void testFieldWithDotsAndCommonPrefixes() { - FieldHitExtractor fe1 = getFieldHitExtractor("a1.b.c.d1.e.f.g1", false); - FieldHitExtractor fe2 = getFieldHitExtractor("a2.b.c.d2.e.f.g2", false); - Object value = randomNonNullValue(); - Map map = new HashMap<>(); - map.put("a1", singletonMap("b.c", singletonMap("d1", singletonMap("e.f", singletonMap("g1", value))))); - map.put("a2", singletonMap("b.c", singletonMap("d2", singletonMap("e.f", singletonMap("g2", value))))); - assertEquals(value, fe1.extractFromSource(map)); - assertEquals(value, fe2.extractFromSource(map)); - } - - public void testFieldWithDotsAndSamePathButDifferentHierarchy() { - FieldHitExtractor fe = getFieldHitExtractor("a.b.c.d.e.f.g", false); - Object value = randomNonNullValue(); - Map map = new HashMap<>(); - map.put("a.b", singletonMap("c", singletonMap("d.e", singletonMap("f.g", value)))); - map.put("a", singletonMap("b.c", singletonMap("d.e", singletonMap("f", singletonMap("g", value))))); - QlIllegalArgumentException ex = expectThrows(QlIllegalArgumentException.class, () -> fe.extractFromSource(map)); - assertThat(ex.getMessage(), is("Cannot return multiple values for field [a.b.c.d.e.f.g]; use ARRAY(a.b.c.d.e.f.g) instead")); - } - - public void testFieldsWithSingleValueArrayAsSubfield() { - FieldHitExtractor fe = getFieldHitExtractor("a.b", false); - Object value = randomNonNullValue(); - Map map = new HashMap<>(); - // "a" : [{"b" : "value"}] - map.put("a", singletonList(singletonMap("b", value))); - assertEquals(value, fe.extractFromSource(map)); - } - - public void testFieldsWithMultiValueArrayAsSubfield() { - FieldHitExtractor fe = getFieldHitExtractor("a.b", false); - Map map = new HashMap<>(); - // "a" : [{"b" : "value1"}, {"b" : "value2"}] - map.put("a", asList(singletonMap("b", randomNonNullValue()), singletonMap("b", randomNonNullValue()))); - QlIllegalArgumentException ex = expectThrows(QlIllegalArgumentException.class, () -> fe.extractFromSource(map)); - assertThat(ex.getMessage(), is("Cannot return multiple values for field [a.b]; use ARRAY(a.b) instead")); - } - - public void testFieldsWithSingleValueArrayAsSubfield_TwoNestedLists() { - FieldHitExtractor fe = getFieldHitExtractor("a.b.c", false); - Object value = randomNonNullValue(); - Map map = new HashMap<>(); - // "a" : [{"b" : [{"c" : "value"}]}] - map.put("a", singletonList(singletonMap("b", singletonList(singletonMap("c", value))))); - assertEquals(value, fe.extractFromSource(map)); - } - - public void testFieldsWithMultiValueArrayAsSubfield_ThreeNestedLists() { - FieldHitExtractor fe = getFieldHitExtractor("a.b.c", false); - Map map = new HashMap<>(); - // "a" : [{"b" : [{"c" : ["value1", "value2"]}]}] - map.put("a", singletonList(singletonMap("b", singletonList(singletonMap("c", asList("value1", "value2")))))); - QlIllegalArgumentException ex = expectThrows(QlIllegalArgumentException.class, () -> fe.extractFromSource(map)); - assertThat(ex.getMessage(), is("Cannot return multiple values for field [a.b.c]; use ARRAY(a.b.c) instead")); - } - - public void testFieldsWithSingleValueArrayAsSubfield_TwoNestedLists2() { - FieldHitExtractor fe = getFieldHitExtractor("a.b.c", false); - Object value = randomNonNullValue(); - Map map = new HashMap<>(); - // "a" : [{"b" : {"c" : ["value"]}]}] - map.put("a", singletonList(singletonMap("b", singletonMap("c", singletonList(value))))); - assertEquals(value, fe.extractFromSource(map)); - } - - // "a": null - public void testMultiValueNull() { - Map map = new HashMap<>(); - map.put("a", null); - FieldHitExtractor fea = getArrayFieldHitExtractor("a", INTEGER); - assertEquals(singletonList(null), fea.extractFromSource(map)); - } - - // "a": [] / missing - public void testMultiValueEmpty() { - Map map = new HashMap<>(); - map.put("a", emptyList()); - FieldHitExtractor fea = getArrayFieldHitExtractor("a", INTEGER); - assertEquals(emptyList(), fea.extractFromSource(map)); - - FieldHitExtractor feb = getArrayFieldHitExtractor("b", INTEGER); - assertEquals(emptyList(), feb.extractFromSource(map)); - } - - // "a": [i1, i2, ..] as Map - public void testMultiValueImmediateFromMap() { - String fieldName = randomAlphaOfLength(5); - Map map = new HashMap<>(); - List list = randomList(2, 10, ESTestCase::randomInt); - map.put(fieldName, list); - FieldHitExtractor fe = getArrayFieldHitExtractor(fieldName, INTEGER); - assertEquals(list, fe.extractFromSource(map)); - } - - // "a": [i1, i2, ..] XContentBuilder-serialized - public void testMultiValueImmediateFromHit() throws IOException { - String fieldName = randomAlphaOfLength(5); - List list = randomList(2, 10, ESTestCase::randomLong); - - SearchHit hit = new SearchHit(1); - XContentBuilder source = JsonXContent.contentBuilder(); - source.startObject(); { - source.startArray(fieldName); { - list.forEach(x -> { - try { - source.value(x); - } catch (IOException ignored) { - fail(); - } - }); - } - source.endArray(); - } - source.endObject(); - BytesReference sourceRef = BytesReference.bytes(source); - hit.sourceRef(sourceRef); - - FieldHitExtractor fe = getArrayFieldHitExtractor(fieldName, LONG); - assertEquals(list, fe.extract(hit)); - } - - // {"a": {"b": {"c": 1}}, "a": {"b.c": [2, 3]}, "a.b": [{"c": 4}, {"c": 5}], ...} - @SuppressWarnings("unchecked") - public void testMultiValueDifferentPathsWithArraysAndNulls() { - int depth = randomIntBetween(3, 10); // depth excluding the leaf - List nodes = randomList(2, depth, () -> randomAlphaOfLength(randomIntBetween(1, 5))); - List> paths = generatePaths(nodes); - - List valuesList = new ArrayList<>(paths.size()); - Map map = new HashMap<>(); - for (List path : paths) { - Object value; - Supplier supplier = () -> randomNonNegativeByte() < 30 ? null : randomDouble(); - if (randomBoolean()) { - value = supplier.get(); - valuesList.add((Double) value); - } else { - value = randomList(1, 5, supplier); - valuesList.addAll((List) value); - } - if (path.size() == 1) { // "a.b.c": 3 - map.put(path.get(0), value); - } else { - Map crrMap = new HashMap<>(); - crrMap.put(path.get(path.size() - 1), value); - for (int j = path.size() - 2; j > 0; j--) { - Map newMap = new HashMap<>(); - newMap.put(path.get(j), crrMap); - crrMap = newMap; - } - mergeMaps(map, path.get(0), crrMap); - } - } - randomlyMultiplicateSubmaps(map, valuesList); - - String fieldName = join(".", paths.get(0)); - FieldHitExtractor fe = getArrayFieldHitExtractor(fieldName, DOUBLE); - - Object result = fe.extractFromSource(map); - assertTrue(result instanceof List); - List resultList = (List) result; - - // order of array retrieval is stable, not trivial to predict and ultimately irrelevant / not guaranteed - valuesList.sort(Comparator.nullsLast(Double::compare)); - resultList.sort(Comparator.nullsLast(Double::compare)); - assertEquals(valuesList, resultList); - } - - // "a": {"b": 2} => "a": [{"b": 2}, {"b": 2}] - private static void randomlyMultiplicateSubmaps(Map map, List valuesList) { - map.keySet().forEach(key -> { - Object val = map.get(key); - if (val instanceof Map) { - @SuppressWarnings("unchecked") - Map innerMap = (Map) val; - if (randomNonNegativeByte() < 60) { - List> replacementList = new ArrayList<>(); - int multiplicate = randomIntBetween(1, 5); - for (int i = 0; i < multiplicate; i++) { - Map copy = new HashMap<>(innerMap); - replacementList.add(copy); - if (i > 0) { // the initial copy is already part of valuesList - collectLeaves(copy, valuesList); - } - } - map.put(key, replacementList); - } else { - randomlyMultiplicateSubmaps(innerMap, valuesList); - } - } - }); - } - - @SuppressWarnings("unchecked") - private static void collectLeaves(Map map, List valuesList) { - for (Object val : map.values()) { - if (val instanceof Map) { - collectLeaves((Map) val, valuesList); - } else if (val instanceof List) { - for (Object o : (List) val) { - if (o instanceof Map) { - collectLeaves((Map) o, valuesList); - } else { - valuesList.add((Double) o); - } - } - } else { - valuesList.add((Double) val); - } - } - } - - // {"a" : {"b": {"c": 3}}} + "a", {"b.c": 4} => {"a": {"b": {"c": 3}, "b.c": 4}} - @SuppressWarnings("unchecked") - private static void mergeMaps(Map destination, String key, Map singleKeys) { - Object o = singleKeys; - while (destination.containsKey(key)) { - destination = (Map) destination.get(key); - key = singleKeys.keySet().toArray(new String[0])[0]; - o = singleKeys.get(key); - if (o instanceof Map == false) { - break; - } - singleKeys = (Map) o; - } - destination.put(key, o); - } - - // generate all possible path combinations with given node names: a, b, c => (a, b, c), (a, b.c), (a.b, c), (a.b.c) - private static List> generatePaths(List nodes) { - if (nodes.size() == 0) { - return emptyList(); - } - List> paths = new ArrayList<>(singletonList(singletonList(nodes.get(0)))); - for (int i = 1; i < nodes.size(); i++) { - List> newPaths = new ArrayList<>(); - for (List crrPath : paths) { - newPaths.addAll(extendPaths(crrPath, nodes.get(i))); - } - paths = newPaths; - } - return paths; - } - - // (a, b) + c => (a, b, c), (a, bc) - private static List> extendPaths(List paths, String node) { - List> extendedPaths = new ArrayList<>(paths.size() * 2); - List listA = new ArrayList<>(paths); - listA.add(node); - extendedPaths.add(listA); - if (paths.isEmpty() == false) { - List listB; - if (paths.size() > 1) { - listB = paths.subList(0, paths.size() - 1); - listB.add(paths.get(paths.size() - 1) + "." + node); - } else { - listB = singletonList(paths.get(0) + "." + node); - } - extendedPaths.add(listB); - } - return extendedPaths; - } - - public void testObjectsForSourceValue() throws IOException { - String fieldName = randomAlphaOfLength(5); - FieldHitExtractor fe = getFieldHitExtractor(fieldName, false); - SearchHit hit = new SearchHit(1); - XContentBuilder source = JsonXContent.contentBuilder(); - source.startObject(); { - source.startObject(fieldName); { - source.field("b", "c"); - } - source.endObject(); - } - source.endObject(); - BytesReference sourceRef = BytesReference.bytes(source); - hit.sourceRef(sourceRef); - QlIllegalArgumentException ex = expectThrows(QlIllegalArgumentException.class, () -> fe.extract(hit)); - assertThat(ex.getMessage(), is("Objects (returned by [" + fieldName + "]) are not supported")); - } - - public void testMultipleObjectsForSourceValue() throws IOException { - String fieldName = randomAlphaOfLength(5); - FieldHitExtractor fe = randomBoolean() - ? getFieldHitExtractor(fieldName, false) - : getArrayFieldHitExtractor(fieldName, randomFrom(SqlDataTypes.types())); - SearchHit hit = new SearchHit(1); - int arraySize = randomIntBetween(1, 4); - XContentBuilder source = JsonXContent.contentBuilder(); - source.startObject(); { - source.startArray(fieldName); { - for (int i = 0; i < arraySize; i++) { - source.startObject(); { - source.field("b" + i, "c"); - } - source.endObject(); - } - } - source.endArray(); - } - source.endObject(); - BytesReference sourceRef = BytesReference.bytes(source); - hit.sourceRef(sourceRef); - QlIllegalArgumentException ex = expectThrows(QlIllegalArgumentException.class, () -> fe.extract(hit)); - assertThat(ex.getMessage(), is("Objects (returned by [" + fieldName + "]) are not supported")); + DocumentField field = new DocumentField("a", asList(valueA, valueB)); + SearchHit hit = new SearchHit(1, null, null, singletonMap("a", field), null); + assertEquals(valueA, fe.extract(hit)); } public void testGeoShapeExtraction() { String fieldName = randomAlphaOfLength(5); - FieldHitExtractor fe = new FieldHitExtractor(fieldName, randomBoolean() ? GEO_SHAPE : SHAPE, UTC, false); - Map map = new HashMap<>(); - map.put(fieldName, "POINT (1 2)"); - assertEquals(new GeoShape(1, 2), fe.extractFromSource(map)); + FieldHitExtractor fe = new FieldHitExtractor(fieldName, randomBoolean() ? GEO_SHAPE : SHAPE, UTC, FAIL_IF_MULTIVALUE); - map = new HashMap<>(); - assertNull(fe.extractFromSource(map)); - } + Map map = new HashMap<>(2); + map.put("coordinates", asList(1d, 2d)); + map.put("type", "Point"); + DocumentField field = new DocumentField(fieldName, singletonList(map)); + SearchHit hit = new SearchHit(1, null, null, singletonMap(fieldName, field), null); - - public void testMultipleGeoShapeExtraction() { - String fieldName = randomAlphaOfLength(5); - FieldHitExtractor fe = new FieldHitExtractor(fieldName, randomBoolean() ? GEO_SHAPE : SHAPE, UTC, false); - Map map = new HashMap<>(); - map.put(fieldName, "POINT (1 2)"); - assertEquals(new GeoShape(1, 2), fe.extractFromSource(map)); - - map = new HashMap<>(); - assertNull(fe.extractFromSource(map)); - - Map map2 = new HashMap<>(); - map2.put(fieldName, asList("POINT (1 2)", "POINT (3 4)")); - QlIllegalArgumentException ex = expectThrows(QlIllegalArgumentException.class, () -> fe.extractFromSource(map2)); - assertThat(ex.getMessage(), is("Cannot return multiple values for field [" + fieldName + "]; " + - "use ARRAY(" + fieldName + ") instead")); - - FieldHitExtractor lenientFe = new FieldHitExtractor(fieldName, - randomBoolean() ? GEO_SHAPE : SHAPE, UTC, false, EXTRACT_ONE); - assertEquals(new GeoShape(1, 2), lenientFe.extractFromSource(map2)); - - FieldHitExtractor arrayFe = getArrayFieldHitExtractor(fieldName, randomBoolean() ? GEO_SHAPE : SHAPE); - assertEquals(asList(new GeoShape(1, 2), new GeoShape(3, 4)), arrayFe.extractFromSource(map2)); - } - - public void testGeoPointExtractionFromSource() throws IOException { - int layers = randomIntBetween(1, 3); - String pathCombined = ""; - double lat = randomDoubleBetween(-90, 90, true); - double lon = randomDoubleBetween(-180, 180, true); - SearchHit hit = new SearchHit(1); - XContentBuilder source = JsonXContent.contentBuilder(); - boolean[] arrayWrap = new boolean[layers - 1]; - source.startObject(); { - for (int i = 0; i < layers - 1; i++) { - arrayWrap[i] = randomBoolean(); - String name = randomAlphaOfLength(10); - source.field(name); - if (arrayWrap[i]) { - source.startArray(); - } - source.startObject(); - pathCombined = pathCombined + name + "."; - } - String name = randomAlphaOfLength(10); - pathCombined = pathCombined + name; - source.field(name, randomSpecPoint(lat, lon)); - for (int i = layers - 2; i >= 0; i--) { - source.endObject(); - if (arrayWrap[i]) { - source.endArray(); - } - } - } - source.endObject(); - BytesReference sourceRef = BytesReference.bytes(source); - hit.sourceRef(sourceRef); - - FieldHitExtractor fe = new FieldHitExtractor(pathCombined, GEO_POINT, UTC, false); - assertEquals(new GeoShape(lon, lat), fe.extract(hit)); - } - - public void testMultipleGeoPointExtractionFromSource() throws IOException { - SearchHit hit = new SearchHit(1); - String fieldName = randomAlphaOfLength(5); - int arraySize = randomIntBetween(2, 4); - List geoShapes = new ArrayList<>(arraySize); - XContentBuilder source = JsonXContent.contentBuilder(); - source.startObject(); { - source.startArray(fieldName); - for (int i = 1; i <= arraySize; i++) { - double lat = randomDoubleBetween(-90, 90, true); - double lon = randomDoubleBetween(-180, 180, true); - source.value(randomSpecPoint(lat, lon)); - geoShapes.add(new GeoShape(lon, lat)); - } - source.endArray(); - } - source.endObject(); - BytesReference sourceRef = BytesReference.bytes(source); - hit.sourceRef(sourceRef); - - FieldHitExtractor fe = new FieldHitExtractor(fieldName, GEO_POINT, UTC, false); - QlIllegalArgumentException ex = expectThrows(QlIllegalArgumentException.class, () -> fe.extract(hit)); - assertThat(ex.getMessage(), is("Cannot return multiple values for field [" + fieldName + "]; " + - "use ARRAY(" + fieldName + ") instead")); - - FieldHitExtractor lenientFe = new FieldHitExtractor(fieldName, GEO_POINT, UTC, false, EXTRACT_ONE); - assertEquals(geoShapes.get(0), lenientFe.extract(hit)); - - FieldHitExtractor arrayFe = getArrayFieldHitExtractor(fieldName, GEO_POINT); - assertEquals(geoShapes, arrayFe.extract(hit)); - } - - public void testGeoPointExtractionFromDocValues() { - String fieldName = randomAlphaOfLength(5); - FieldHitExtractor fe = new FieldHitExtractor(fieldName, GEO_POINT, UTC, true); - DocumentField field = new DocumentField(fieldName, singletonList("2, 1")); - SearchHit hit = new SearchHit(1, null, singletonMap(fieldName, field), null); assertEquals(new GeoShape(1, 2), fe.extract(hit)); - hit = new SearchHit(1); - assertNull(fe.extract(hit)); } - public void testGeoPointExtractionFromMultipleDocValues() { + public void testMultipleGeoShapeExtraction() { String fieldName = randomAlphaOfLength(5); - SearchHit hit = new SearchHit(1, null, singletonMap(fieldName, - new DocumentField(fieldName, asList("2,1", "3,4"))), null); + FieldHitExtractor fe = new FieldHitExtractor(fieldName, randomBoolean() ? GEO_SHAPE : SHAPE, UTC, FAIL_IF_MULTIVALUE); + + Map map1 = new HashMap<>(2); + map1.put("coordinates", asList(1d, 2d)); + map1.put("type", "Point"); + Map map2 = new HashMap<>(2); + map2.put("coordinates", asList(3d, 4d)); + map2.put("type", "Point"); + DocumentField field = new DocumentField(fieldName, asList(map1, map2)); + SearchHit hit = new SearchHit(1, null, singletonMap(fieldName, field), null); - FieldHitExtractor fe = new FieldHitExtractor(fieldName, GEO_POINT, UTC, true); QlIllegalArgumentException ex = expectThrows(QlIllegalArgumentException.class, () -> fe.extract(hit)); - assertThat(ex.getMessage(), is("Cannot return multiple values for field [" + fieldName + "]; " + - "use ARRAY(" + fieldName + ") instead")); - - FieldHitExtractor lenientFe = new FieldHitExtractor(fieldName, GEO_POINT, UTC, true, EXTRACT_ONE); - assertEquals(new GeoShape(1, 2), lenientFe.extract(hit)); + assertThat(ex.getMessage(), is("Arrays (returned by [" + fieldName + "]) are not supported")); - FieldHitExtractor arrayFe = new FieldHitExtractor(fieldName, GEO_POINT, UTC, true, EXTRACT_ARRAY); - assertEquals(asList(new GeoShape(1, 2), new GeoShape(4, 3)), arrayFe.extract(hit)); + FieldHitExtractor lenientFe = new FieldHitExtractor(fieldName, randomBoolean() ? GEO_SHAPE : SHAPE, UTC, EXTRACT_ONE); + assertEquals(new GeoShape(3, 4), lenientFe.extract(new SearchHit(1, null, null, singletonMap(fieldName, + new DocumentField(fieldName, singletonList(map2))), null))); } - - - private FieldHitExtractor getFieldHitExtractor(String fieldName, boolean useDocValue) { - return new FieldHitExtractor(fieldName, null, UTC, useDocValue); + private FieldHitExtractor getFieldHitExtractor(String fieldName) { + return new FieldHitExtractor(fieldName, null, UTC); } private static FieldHitExtractor getArrayFieldHitExtractor(String fieldName, DataType dataType) { - return new FieldHitExtractor(fieldName, dataType, UTC, false, EXTRACT_ARRAY); + return new FieldHitExtractor(fieldName, dataType, UTC, EXTRACT_ARRAY); } private Object randomValue() { @@ -883,39 +237,4 @@ private Object randomValue() { () -> null)); return value.get(); } - - private Object randomNonNullValue() { - Supplier value = randomFrom(asList( - () -> randomAlphaOfLength(10), - ESTestCase::randomLong, - ESTestCase::randomDouble, - ESTestCase::randomInt, - () -> BigInteger.valueOf(Long.MAX_VALUE).add(BigInteger.ONE), - () -> new BigDecimal("20012312345621343256123456254.20012312345621343256123456254"))); - return value.get(); - } - - private void assertFieldHitEquals(Object expected, Object actual) { - if (expected instanceof BigDecimal) { - // parsing will, by default, build a Double even if the initial value is BigDecimal - // Elasticsearch does this the same when returning the results - assertEquals(((BigDecimal) expected).doubleValue(), actual); - } else { - assertEquals(expected, actual); - } - } - - private Object randomSpecPoint(double lat, double lon) { - Supplier value = randomFrom(asList( - () -> lat + "," + lon, - () -> asList(lon, lat), - () -> { - Map map1 = new HashMap<>(); - map1.put("lat", lat); - map1.put("lon", lon); - return map1; - } - )); - return value.get(); - } } diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/sql/translate.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/sql/translate.yml index 3e61e2ed0e9eb..6439b6f1be92b 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/sql/translate.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/sql/translate.yml @@ -17,11 +17,8 @@ - match: $body: size: 1000 - _source: - includes: - - int - - str - excludes: [] + _source: false + fields: [ {"field" : "int" }, {"field" : "str" } ] sort: - int: order: asc