diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersQueryIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersQueryIT.java index 6cfc42523007e..1a99fd9d0383d 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersQueryIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersQueryIT.java @@ -220,21 +220,13 @@ public void testSearchesAgainstNonMatchingIndicesWithLocalOnly() { } { String q = "FROM nomatch"; - VerificationException e = expectThrows(VerificationException.class, () -> runQuery(q, false)); - assertThat(e.getDetailedMessage(), containsString("Unknown index [nomatch]")); - - String limit0 = q + " | LIMIT 0"; - e = expectThrows(VerificationException.class, () -> runQuery(limit0, false)); - assertThat(e.getDetailedMessage(), containsString("Unknown index [nomatch]")); + String expectedError = "Unknown index [nomatch]"; + expectVerificationExceptionForQuery(q, expectedError, false); } { String q = "FROM nomatch*"; - VerificationException e = expectThrows(VerificationException.class, () -> runQuery(q, false)); - assertThat(e.getDetailedMessage(), containsString("Unknown index [nomatch*]")); - - String limit0 = q + " | LIMIT 0"; - e = expectThrows(VerificationException.class, () -> runQuery(limit0, false)); - assertThat(e.getDetailedMessage(), containsString("Unknown index [nomatch*]")); + String expectedError = "Unknown index [nomatch*]"; + expectVerificationExceptionForQuery(q, expectedError, false); } } @@ -296,554 +288,181 @@ public void testSearchesAgainstIndicesWithNoMappingsSkipUnavailableTrue() { } } - public void testSearchesAgainstNonMatchingIndicesWithSkipUnavailableTrue() { + public void testSearchesAgainstNonMatchingIndices() { int numClusters = 3; Map testClusterInfo = setupClusters(numClusters); int localNumShards = (Integer) testClusterInfo.get("local.num_shards"); int remote1NumShards = (Integer) testClusterInfo.get("remote.num_shards"); - int remote2NumShards = (Integer) testClusterInfo.get("remote2.num_shards"); String localIndex = (String) testClusterInfo.get("local.index"); String remote1Index = (String) testClusterInfo.get("remote.index"); String remote2Index = (String) testClusterInfo.get("remote2.index"); createIndexAliases(numClusters); - setSkipUnavailable(REMOTE_CLUSTER_1, true); - setSkipUnavailable(REMOTE_CLUSTER_2, true); Tuple includeCCSMetadata = randomIncludeCCSMetadata(); Boolean requestIncludeMeta = includeCCSMetadata.v1(); boolean responseExpectMeta = includeCCSMetadata.v2(); - try { - // missing concrete local index is fatal - { - String q = "FROM nomatch,cluster-a:" + randomFrom(remote1Index, IDX_ALIAS, FILTERED_IDX_ALIAS); - VerificationException e = expectThrows(VerificationException.class, () -> runQuery(q, requestIncludeMeta)); - assertThat(e.getDetailedMessage(), containsString("Unknown index [nomatch]")); - - String limit0 = q + " | LIMIT 0"; - e = expectThrows(VerificationException.class, () -> runQuery(limit0, requestIncludeMeta)); - assertThat(e.getDetailedMessage(), containsString("Unknown index [nomatch]")); - } - - // missing concrete remote index is not fatal when skip_unavailable=true (as long as an index matches on another cluster) - { - String localIndexName = randomFrom(localIndex, IDX_ALIAS, FILTERED_IDX_ALIAS); - String q = Strings.format("FROM %s,cluster-a:nomatch", localIndexName); - try (EsqlQueryResponse resp = runQuery(q, requestIncludeMeta)) { - assertThat(getValuesList(resp).size(), greaterThanOrEqualTo(1)); - EsqlExecutionInfo executionInfo = resp.getExecutionInfo(); - assertThat(executionInfo.isCrossClusterSearch(), is(true)); - assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta)); - assertExpectedClustersForMissingIndicesTests( - executionInfo, - List.of( - new ExpectedCluster(LOCAL_CLUSTER, localIndexName, EsqlExecutionInfo.Cluster.Status.SUCCESSFUL, localNumShards), - new ExpectedCluster(REMOTE_CLUSTER_1, "nomatch", EsqlExecutionInfo.Cluster.Status.SKIPPED, 0) - ) - ); - } - - String limit0 = q + " | LIMIT 0"; - try (EsqlQueryResponse resp = runQuery(limit0, requestIncludeMeta)) { - assertThat(resp.columns().size(), greaterThan(0)); - assertThat(getValuesList(resp).size(), greaterThanOrEqualTo(0)); - EsqlExecutionInfo executionInfo = resp.getExecutionInfo(); - assertThat(executionInfo.isCrossClusterSearch(), is(true)); - assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta)); - assertExpectedClustersForMissingIndicesTests( - executionInfo, - List.of( - new ExpectedCluster(LOCAL_CLUSTER, localIndexName, EsqlExecutionInfo.Cluster.Status.SUCCESSFUL, 0), - new ExpectedCluster(REMOTE_CLUSTER_1, "nomatch", EsqlExecutionInfo.Cluster.Status.SKIPPED, 0) - ) - ); - } - } - - // since there is at least one matching index in the query, the missing wildcarded local index is not an error - { - String remoteIndexName = randomFrom(remote1Index, IDX_ALIAS, FILTERED_IDX_ALIAS); - String q = "FROM nomatch*,cluster-a:" + remoteIndexName; - try (EsqlQueryResponse resp = runQuery(q, requestIncludeMeta)) { - assertThat(getValuesList(resp).size(), greaterThanOrEqualTo(1)); - EsqlExecutionInfo executionInfo = resp.getExecutionInfo(); - assertThat(executionInfo.isCrossClusterSearch(), is(true)); - assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta)); - assertExpectedClustersForMissingIndicesTests( - executionInfo, - List.of( - // local cluster is never marked as SKIPPED even when no matching indices - just marked as 0 shards searched - new ExpectedCluster(LOCAL_CLUSTER, "nomatch*", EsqlExecutionInfo.Cluster.Status.SUCCESSFUL, 0), - new ExpectedCluster( - REMOTE_CLUSTER_1, - remoteIndexName, - EsqlExecutionInfo.Cluster.Status.SUCCESSFUL, - remote1NumShards - ) - ) - ); - } - - String limit0 = q + " | LIMIT 0"; - try (EsqlQueryResponse resp = runQuery(limit0, requestIncludeMeta)) { - assertThat(getValuesList(resp).size(), equalTo(0)); - assertThat(resp.columns().size(), greaterThan(0)); - EsqlExecutionInfo executionInfo = resp.getExecutionInfo(); - assertThat(executionInfo.isCrossClusterSearch(), is(true)); - assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta)); - assertExpectedClustersForMissingIndicesTests( - executionInfo, - List.of( - // local cluster is never marked as SKIPPED even when no matching indices - just marked as 0 shards searched - new ExpectedCluster(LOCAL_CLUSTER, "nomatch*", EsqlExecutionInfo.Cluster.Status.SUCCESSFUL, 0), - // LIMIT 0 searches always have total shards = 0 - new ExpectedCluster(REMOTE_CLUSTER_1, remoteIndexName, EsqlExecutionInfo.Cluster.Status.SUCCESSFUL, 0) - ) - ); - } - } + // missing concrete local index is an error + { + String q = "FROM nomatch,cluster-a:" + remote1Index; + String expectedError = "Unknown index [nomatch]"; + expectVerificationExceptionForQuery(q, expectedError, requestIncludeMeta); + } - // since at least one index of the query matches on some cluster, a wildcarded index on skip_un=true is not an error - { - String localIndexName = randomFrom(localIndex, IDX_ALIAS, FILTERED_IDX_ALIAS); - String q = Strings.format("FROM %s,cluster-a:nomatch*", localIndexName); - try (EsqlQueryResponse resp = runQuery(q, requestIncludeMeta)) { - assertThat(getValuesList(resp).size(), greaterThanOrEqualTo(1)); - EsqlExecutionInfo executionInfo = resp.getExecutionInfo(); - assertThat(executionInfo.isCrossClusterSearch(), is(true)); - assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta)); - assertExpectedClustersForMissingIndicesTests( - executionInfo, - List.of( - new ExpectedCluster(LOCAL_CLUSTER, localIndexName, EsqlExecutionInfo.Cluster.Status.SUCCESSFUL, localNumShards), - new ExpectedCluster(REMOTE_CLUSTER_1, "nomatch*", EsqlExecutionInfo.Cluster.Status.SKIPPED, 0) - ) - ); - } + // missing concrete remote index is fatal + { + String q = "FROM logs*,cluster-a:nomatch"; + String expectedError = "Unknown index [cluster-a:nomatch]"; + expectVerificationExceptionForQuery(q, expectedError, requestIncludeMeta); + } - String limit0 = q + " | LIMIT 0"; - try (EsqlQueryResponse resp = runQuery(limit0, requestIncludeMeta)) { - assertThat(resp.columns().size(), greaterThan(0)); - assertThat(getValuesList(resp).size(), greaterThanOrEqualTo(0)); - EsqlExecutionInfo executionInfo = resp.getExecutionInfo(); - assertThat(executionInfo.isCrossClusterSearch(), is(true)); - assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta)); - assertExpectedClustersForMissingIndicesTests( - executionInfo, - List.of( - new ExpectedCluster(LOCAL_CLUSTER, localIndexName, EsqlExecutionInfo.Cluster.Status.SUCCESSFUL, 0), - new ExpectedCluster(REMOTE_CLUSTER_1, "nomatch*", EsqlExecutionInfo.Cluster.Status.SKIPPED, 0) + // No error since local non-matching index has wildcard and the remote cluster index expression matches + { + String remote1IndexName = randomFrom(remote1Index, IDX_ALIAS, FILTERED_IDX_ALIAS); + String q = Strings.format("FROM nomatch*,%s:%s", REMOTE_CLUSTER_1, remote1IndexName); + try (EsqlQueryResponse resp = runQuery(q, requestIncludeMeta)) { + assertThat(getValuesList(resp).size(), greaterThanOrEqualTo(1)); + EsqlExecutionInfo executionInfo = resp.getExecutionInfo(); + assertThat(executionInfo.isCrossClusterSearch(), is(true)); + assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta)); + assertExpectedClustersForMissingIndicesTests( + executionInfo, + List.of( + // local cluster is never marked as SKIPPED even when no matcing indices - just marked as 0 shards searched + new ExpectedCluster(LOCAL_CLUSTER, "nomatch*", EsqlExecutionInfo.Cluster.Status.SUCCESSFUL, 0), + new ExpectedCluster( + REMOTE_CLUSTER_1, + remote1IndexName, + EsqlExecutionInfo.Cluster.Status.SUCCESSFUL, + remote1NumShards ) - ); - } + ) + ); } - // an error is thrown if there are no matching indices at all, even when the cluster is skip_unavailable=true - { - // with non-matching concrete index - String q = "FROM cluster-a:nomatch"; - VerificationException e = expectThrows(VerificationException.class, () -> runQuery(q, requestIncludeMeta)); - assertThat(e.getDetailedMessage(), containsString("Unknown index [cluster-a:nomatch]")); - - String limit0 = q + " | LIMIT 0"; - e = expectThrows(VerificationException.class, () -> runQuery(limit0, requestIncludeMeta)); - assertThat(e.getDetailedMessage(), containsString("Unknown index [cluster-a:nomatch]")); - } - - // an error is thrown if there are no matching indices at all, even when the cluster is skip_unavailable=true and the - // index was wildcarded - { - // with non-matching wildcard index - String q = "FROM cluster-a:nomatch*"; - VerificationException e = expectThrows(VerificationException.class, () -> runQuery(q, requestIncludeMeta)); - assertThat(e.getDetailedMessage(), containsString("Unknown index [cluster-a:nomatch*]")); - - String limit0 = q + " | LIMIT 0"; - e = expectThrows(VerificationException.class, () -> runQuery(limit0, requestIncludeMeta)); - assertThat(e.getDetailedMessage(), containsString("Unknown index [cluster-a:nomatch*]")); - } - - // an error is thrown if there are no matching indices at all - local with wildcard, remote with concrete - { - String q = "FROM nomatch*,cluster-a:nomatch"; - VerificationException e = expectThrows(VerificationException.class, () -> runQuery(q, requestIncludeMeta)); - assertThat(e.getDetailedMessage(), containsString("Unknown index [cluster-a:nomatch,nomatch*]")); - - String limit0 = q + " | LIMIT 0"; - e = expectThrows(VerificationException.class, () -> runQuery(limit0, requestIncludeMeta)); - assertThat(e.getDetailedMessage(), containsString("Unknown index [cluster-a:nomatch,nomatch*]")); - } - - // an error is thrown if there are no matching indices at all - local with wildcard, remote with wildcard - { - String q = "FROM nomatch*,cluster-a:nomatch*"; - VerificationException e = expectThrows(VerificationException.class, () -> runQuery(q, requestIncludeMeta)); - assertThat(e.getDetailedMessage(), containsString("Unknown index [cluster-a:nomatch*,nomatch*]")); - - String limit0 = q + " | LIMIT 0"; - e = expectThrows(VerificationException.class, () -> runQuery(limit0, requestIncludeMeta)); - assertThat(e.getDetailedMessage(), containsString("Unknown index [cluster-a:nomatch*,nomatch*]")); + String limit0 = q + " | LIMIT 0"; + try (EsqlQueryResponse resp = runQuery(limit0, requestIncludeMeta)) { + assertThat(getValuesList(resp).size(), equalTo(0)); + assertThat(resp.columns().size(), greaterThan(0)); + EsqlExecutionInfo executionInfo = resp.getExecutionInfo(); + assertThat(executionInfo.isCrossClusterSearch(), is(true)); + assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta)); + assertExpectedClustersForMissingIndicesTests( + executionInfo, + List.of( + // local cluster is never marked as SKIPPED even when no matching indices - just marked as 0 shards searched + new ExpectedCluster(LOCAL_CLUSTER, "nomatch*", EsqlExecutionInfo.Cluster.Status.SUCCESSFUL, 0), + // LIMIT 0 searches always have total shards = 0 + new ExpectedCluster(REMOTE_CLUSTER_1, remote1IndexName, EsqlExecutionInfo.Cluster.Status.SUCCESSFUL, 0) + ) + ); } + } - // an error is thrown if there are no matching indices at all - local with concrete, remote with concrete - { - String q = "FROM nomatch,cluster-a:nomatch"; - VerificationException e = expectThrows(VerificationException.class, () -> runQuery(q, requestIncludeMeta)); - assertThat(e.getDetailedMessage(), containsString("Unknown index [cluster-a:nomatch,nomatch]")); + // No error since remote non-matching index has wildcard and the local cluster index expression matches + { + String indexLoc = randomFrom(localIndex, IDX_ALIAS, FILTERED_IDX_ALIAS); + String q = Strings.format("FROM %s,cluster-a:nomatch*", indexLoc); - String limit0 = q + " | LIMIT 0"; - e = expectThrows(VerificationException.class, () -> runQuery(limit0, requestIncludeMeta)); - assertThat(e.getDetailedMessage(), containsString("Unknown index [cluster-a:nomatch,nomatch]")); + try (EsqlQueryResponse resp = runQuery(q, requestIncludeMeta)) { + assertThat(getValuesList(resp).size(), greaterThanOrEqualTo(1)); + EsqlExecutionInfo executionInfo = resp.getExecutionInfo(); + assertThat(executionInfo.isCrossClusterSearch(), is(true)); + assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta)); + assertExpectedClustersForMissingIndicesTests( + executionInfo, + List.of( + // local cluster is never marked as SKIPPED even when no matching indices - just marked as 0 shards searched + new ExpectedCluster(LOCAL_CLUSTER, indexLoc, EsqlExecutionInfo.Cluster.Status.SUCCESSFUL, localNumShards), + new ExpectedCluster(REMOTE_CLUSTER_1, "nomatch*", EsqlExecutionInfo.Cluster.Status.SKIPPED, 0) + ) + ); } - // an error is thrown if there are no matching indices at all - local with concrete, remote with wildcard - { - String q = "FROM nomatch,cluster-a:nomatch*"; - VerificationException e = expectThrows(VerificationException.class, () -> runQuery(q, requestIncludeMeta)); - assertThat(e.getDetailedMessage(), containsString("Unknown index [cluster-a:nomatch*,nomatch]")); - - String limit0 = q + " | LIMIT 0"; - e = expectThrows(VerificationException.class, () -> runQuery(limit0, requestIncludeMeta)); - assertThat(e.getDetailedMessage(), containsString("Unknown index [cluster-a:nomatch*,nomatch]")); + String limit0 = q + " | LIMIT 0"; + try (EsqlQueryResponse resp = runQuery(limit0, requestIncludeMeta)) { + assertThat(getValuesList(resp).size(), equalTo(0)); + assertThat(resp.columns().size(), greaterThan(0)); + EsqlExecutionInfo executionInfo = resp.getExecutionInfo(); + assertThat(executionInfo.isCrossClusterSearch(), is(true)); + assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta)); + assertExpectedClustersForMissingIndicesTests( + executionInfo, + List.of( + // LIMIT 0 searches always have total shards = 0 + new ExpectedCluster(LOCAL_CLUSTER, indexLoc, EsqlExecutionInfo.Cluster.Status.SUCCESSFUL, 0), + new ExpectedCluster(REMOTE_CLUSTER_1, "nomatch*", EsqlExecutionInfo.Cluster.Status.SKIPPED, 0) + ) + ); } + } - // since cluster-a is skip_unavailable=true and at least one cluster has a matching indices, no error is thrown - { - // TODO solve in follow-on PR which does skip_unavailable handling at execution time - // String q = Strings.format("FROM %s,cluster-a:nomatch,cluster-a:%s*", localIndex, remote1Index); - // try (EsqlQueryResponse resp = runQuery(q, requestIncludeMeta)) { - // assertThat(getValuesList(resp).size(), greaterThanOrEqualTo(1)); - // EsqlExecutionInfo executionInfo = resp.getExecutionInfo(); - // assertThat(executionInfo.isCrossClusterSearch(), is(true)); - // assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta)); - // assertExpectedClustersForMissingIndicesTests(executionInfo, List.of( - // // local cluster is never marked as SKIPPED even when no matching indices - just marked as 0 shards searched - // new ExpectedCluster(REMOTE_CLUSTER_1, "nomatch", EsqlExecutionInfo.Cluster.Status.SKIPPED, 0), - // new ExpectedCluster(REMOTE_CLUSTER_1, "*", EsqlExecutionInfo.Cluster.Status.SUCCESSFUL, remote2NumShards) - // )); - // } - - // TODO: handle LIMIT 0 for this case in follow-on PR - // String limit0 = q + " | LIMIT 0"; - // try (EsqlQueryResponse resp = runQuery(limit0, requestIncludeMeta)) { - // assertThat(resp.columns().size(), greaterThanOrEqualTo(1)); - // assertThat(getValuesList(resp).size(), greaterThanOrEqualTo(0)); - // EsqlExecutionInfo executionInfo = resp.getExecutionInfo(); - // assertThat(executionInfo.isCrossClusterSearch(), is(true)); - // assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta)); - // assertExpectedClustersForMissingIndicesTests(executionInfo, List.of( - // // local cluster is never marked as SKIPPED even when no matching indices - just marked as 0 shards searched - // new ExpectedCluster(LOCAL_CLUSTER, localIndex, EsqlExecutionInfo.Cluster.Status.SUCCESSFUL, 0), - // new ExpectedCluster(REMOTE_CLUSTER_1, "nomatch," + remote1Index + "*", EsqlExecutionInfo.Cluster.Status.SKIPPED, 0) - // )); - // } - } + // an error is thrown if there is a concrete index that does not match + { + String q = "FROM cluster-a:nomatch"; + String expectedError = "Unknown index [cluster-a:nomatch]"; + expectVerificationExceptionForQuery(q, expectedError, requestIncludeMeta); + } - // tests with three clusters --- + // an error is thrown if there are no matching indices at all - single remote cluster with wildcard index expression + { + String q = "FROM cluster-a:nomatch*"; + String expectedError = "Unknown index [cluster-a:nomatch*]"; + expectVerificationExceptionForQuery(q, expectedError, requestIncludeMeta); + } - // since cluster-a is skip_unavailable=true and at least one cluster has a matching indices, no error is thrown - // cluster-a should be marked as SKIPPED with VerificationException - { - String remote2IndexName = randomFrom(remote2Index, IDX_ALIAS, FILTERED_IDX_ALIAS); - String q = Strings.format("FROM nomatch*,cluster-a:nomatch,%s:%s", REMOTE_CLUSTER_2, remote2IndexName); - try (EsqlQueryResponse resp = runQuery(q, requestIncludeMeta)) { - assertThat(getValuesList(resp).size(), greaterThanOrEqualTo(1)); - EsqlExecutionInfo executionInfo = resp.getExecutionInfo(); - assertThat(executionInfo.isCrossClusterSearch(), is(true)); - assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta)); - assertExpectedClustersForMissingIndicesTests( - executionInfo, - List.of( - // local cluster is never marked as SKIPPED even when no matching indices - just marked as 0 shards searched - new ExpectedCluster(LOCAL_CLUSTER, "nomatch*", EsqlExecutionInfo.Cluster.Status.SUCCESSFUL, 0), - new ExpectedCluster(REMOTE_CLUSTER_1, "nomatch", EsqlExecutionInfo.Cluster.Status.SKIPPED, 0), - new ExpectedCluster( - REMOTE_CLUSTER_2, - remote2IndexName, - EsqlExecutionInfo.Cluster.Status.SUCCESSFUL, - remote2NumShards - ) - ) - ); - } + // an error is thrown if there is a concrete index that does not match + { + String q = "FROM nomatch*,cluster-a:nomatch"; + String expectedError = "Unknown index [cluster-a:nomatch,nomatch*]"; + expectVerificationExceptionForQuery(q, expectedError, requestIncludeMeta); + } - String limit0 = q + " | LIMIT 0"; - try (EsqlQueryResponse resp = runQuery(limit0, requestIncludeMeta)) { - assertThat(resp.columns().size(), greaterThanOrEqualTo(1)); - assertThat(getValuesList(resp).size(), greaterThanOrEqualTo(0)); - EsqlExecutionInfo executionInfo = resp.getExecutionInfo(); - assertThat(executionInfo.isCrossClusterSearch(), is(true)); - assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta)); - assertExpectedClustersForMissingIndicesTests( - executionInfo, - List.of( - // local cluster is never marked as SKIPPED even when no matching indices - just marked as 0 shards searched - new ExpectedCluster(LOCAL_CLUSTER, "nomatch*", EsqlExecutionInfo.Cluster.Status.SUCCESSFUL, 0), - new ExpectedCluster(REMOTE_CLUSTER_1, "nomatch", EsqlExecutionInfo.Cluster.Status.SKIPPED, 0), - new ExpectedCluster(REMOTE_CLUSTER_2, remote2IndexName, EsqlExecutionInfo.Cluster.Status.SUCCESSFUL, 0) - ) - ); - } - } + // an error is thrown if there are no matching indices at all - local with wildcard, remote with wildcard + { + String q = "FROM nomatch*,cluster-a:nomatch*"; + String expectedError = "Unknown index [cluster-a:nomatch*,nomatch*]"; + expectVerificationExceptionForQuery(q, expectedError, requestIncludeMeta); + } + { + String q = "FROM nomatch,cluster-a:nomatch"; + String expectedError = "Unknown index [cluster-a:nomatch,nomatch]"; + expectVerificationExceptionForQuery(q, expectedError, requestIncludeMeta); + } + { + String q = "FROM nomatch,cluster-a:nomatch*"; + String expectedError = "Unknown index [cluster-a:nomatch*,nomatch]"; + expectVerificationExceptionForQuery(q, expectedError, requestIncludeMeta); + } - // since cluster-a is skip_unavailable=true and at least one cluster has a matching indices, no error is thrown - // cluster-a should be marked as SKIPPED with a "NoMatchingIndicesException" since a wildcard index was requested - { - String remote2IndexName = randomFrom(remote2Index, IDX_ALIAS, FILTERED_IDX_ALIAS); - String q = Strings.format("FROM nomatch*,cluster-a:nomatch*,%s:%s", REMOTE_CLUSTER_2, remote2IndexName); - try (EsqlQueryResponse resp = runQuery(q, requestIncludeMeta)) { - assertThat(getValuesList(resp).size(), greaterThanOrEqualTo(1)); - EsqlExecutionInfo executionInfo = resp.getExecutionInfo(); - assertThat(executionInfo.isCrossClusterSearch(), is(true)); - assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta)); - assertExpectedClustersForMissingIndicesTests( - executionInfo, - List.of( - // local cluster is never marked as SKIPPED even when no matching indices - just marked as 0 shards searched - new ExpectedCluster(LOCAL_CLUSTER, "nomatch*", EsqlExecutionInfo.Cluster.Status.SUCCESSFUL, 0), - new ExpectedCluster(REMOTE_CLUSTER_1, "nomatch*", EsqlExecutionInfo.Cluster.Status.SKIPPED, 0), - new ExpectedCluster( - REMOTE_CLUSTER_2, - remote2IndexName, - EsqlExecutionInfo.Cluster.Status.SUCCESSFUL, - remote2NumShards - ) - ) - ); - } + // --- test against 3 clusters - String limit0 = q + " | LIMIT 0"; - try (EsqlQueryResponse resp = runQuery(limit0, requestIncludeMeta)) { - assertThat(resp.columns().size(), greaterThanOrEqualTo(1)); - assertThat(getValuesList(resp).size(), greaterThanOrEqualTo(0)); - EsqlExecutionInfo executionInfo = resp.getExecutionInfo(); - assertThat(executionInfo.isCrossClusterSearch(), is(true)); - assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta)); - assertExpectedClustersForMissingIndicesTests( - executionInfo, - List.of( - // local cluster is never marked as SKIPPED even when no matching indices - just marked as 0 shards searched - new ExpectedCluster(LOCAL_CLUSTER, "nomatch*", EsqlExecutionInfo.Cluster.Status.SUCCESSFUL, 0), - new ExpectedCluster(REMOTE_CLUSTER_1, "nomatch*", EsqlExecutionInfo.Cluster.Status.SKIPPED, 0), - new ExpectedCluster(REMOTE_CLUSTER_2, remote2IndexName, EsqlExecutionInfo.Cluster.Status.SUCCESSFUL, 0) - ) - ); - } - } - } finally { - clearSkipUnavailable(); + // missing concrete index (on remote) is error + { + String localIndexName = randomFrom(localIndex, IDX_ALIAS, FILTERED_IDX_ALIAS); + String remote2IndexName = randomFrom(remote2Index, IDX_ALIAS, FILTERED_IDX_ALIAS); + String q = Strings.format("FROM %s*,cluster-a:nomatch,%s:%s*", localIndexName, REMOTE_CLUSTER_2, remote2IndexName); + String expectedError = "Unknown index [cluster-a:nomatch]"; + expectVerificationExceptionForQuery(q, expectedError, requestIncludeMeta); } } - public void testSearchesAgainstNonMatchingIndicesWithSkipUnavailableFalse() { - int numClusters = 3; - Map testClusterInfo = setupClusters(numClusters); - int remote1NumShards = (Integer) testClusterInfo.get("remote.num_shards"); - String localIndex = (String) testClusterInfo.get("local.index"); - String remote1Index = (String) testClusterInfo.get("remote.index"); - String remote2Index = (String) testClusterInfo.get("remote2.index"); - - createIndexAliases(numClusters); - setSkipUnavailable(REMOTE_CLUSTER_1, false); - setSkipUnavailable(REMOTE_CLUSTER_2, false); - - Tuple includeCCSMetadata = randomIncludeCCSMetadata(); - Boolean requestIncludeMeta = includeCCSMetadata.v1(); - boolean responseExpectMeta = includeCCSMetadata.v2(); - - try { - // missing concrete local index is an error - { - String q = "FROM nomatch,cluster-a:" + remote1Index; - VerificationException e = expectThrows(VerificationException.class, () -> runQuery(q, requestIncludeMeta)); - assertThat(e.getDetailedMessage(), containsString("Unknown index [nomatch]")); - - String limit0 = q + " | LIMIT 0"; - e = expectThrows(VerificationException.class, () -> runQuery(limit0, requestIncludeMeta)); - assertThat(e.getDetailedMessage(), containsString("Unknown index [nomatch]")); - } - - // missing concrete remote index is fatal when skip_unavailable=false - { - String q = "FROM logs*,cluster-a:nomatch"; - VerificationException e = expectThrows(VerificationException.class, () -> runQuery(q, requestIncludeMeta)); - assertThat(e.getDetailedMessage(), containsString("Unknown index [cluster-a:nomatch]")); - - String limit0 = q + " | LIMIT 0"; - e = expectThrows(VerificationException.class, () -> runQuery(limit0, requestIncludeMeta)); - assertThat(e.getDetailedMessage(), containsString("Unknown index [cluster-a:nomatch]")); - } - - // No error since local non-matching has wildcard and the remote cluster matches - { - String remote1IndexName = randomFrom(remote1Index, IDX_ALIAS, FILTERED_IDX_ALIAS); - String q = Strings.format("FROM nomatch*,%s:%s", REMOTE_CLUSTER_1, remote1IndexName); - try (EsqlQueryResponse resp = runQuery(q, requestIncludeMeta)) { - assertThat(getValuesList(resp).size(), greaterThanOrEqualTo(1)); - EsqlExecutionInfo executionInfo = resp.getExecutionInfo(); - assertThat(executionInfo.isCrossClusterSearch(), is(true)); - assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta)); - assertExpectedClustersForMissingIndicesTests( - executionInfo, - List.of( - // local cluster is never marked as SKIPPED even when no matcing indices - just marked as 0 shards searched - new ExpectedCluster(LOCAL_CLUSTER, "nomatch*", EsqlExecutionInfo.Cluster.Status.SUCCESSFUL, 0), - new ExpectedCluster( - REMOTE_CLUSTER_1, - remote1IndexName, - EsqlExecutionInfo.Cluster.Status.SUCCESSFUL, - remote1NumShards - ) - ) - ); - } - - String limit0 = q + " | LIMIT 0"; - try (EsqlQueryResponse resp = runQuery(limit0, requestIncludeMeta)) { - assertThat(getValuesList(resp).size(), equalTo(0)); - assertThat(resp.columns().size(), greaterThan(0)); - EsqlExecutionInfo executionInfo = resp.getExecutionInfo(); - assertThat(executionInfo.isCrossClusterSearch(), is(true)); - assertThat(executionInfo.includeCCSMetadata(), equalTo(responseExpectMeta)); - assertExpectedClustersForMissingIndicesTests( - executionInfo, - List.of( - // local cluster is never marked as SKIPPED even when no matcing indices - just marked as 0 shards searched - new ExpectedCluster(LOCAL_CLUSTER, "nomatch*", EsqlExecutionInfo.Cluster.Status.SUCCESSFUL, 0), - // LIMIT 0 searches always have total shards = 0 - new ExpectedCluster(REMOTE_CLUSTER_1, remote1IndexName, EsqlExecutionInfo.Cluster.Status.SUCCESSFUL, 0) - ) - ); - } - } - - // query is fatal since cluster-a has skip_unavailable=false and has no matching indices - { - String q = Strings.format("FROM %s,cluster-a:nomatch*", randomFrom(localIndex, IDX_ALIAS, FILTERED_IDX_ALIAS)); - VerificationException e = expectThrows(VerificationException.class, () -> runQuery(q, requestIncludeMeta)); - assertThat(e.getDetailedMessage(), containsString("Unknown index [cluster-a:nomatch*]")); - - String limit0 = q + " | LIMIT 0"; - e = expectThrows(VerificationException.class, () -> runQuery(limit0, requestIncludeMeta)); - assertThat(e.getDetailedMessage(), containsString("Unknown index [cluster-a:nomatch*]")); - } - - // an error is thrown if there are no matching indices at all - single remote cluster with concrete index expression - { - String q = "FROM cluster-a:nomatch"; - VerificationException e = expectThrows(VerificationException.class, () -> runQuery(q, requestIncludeMeta)); - assertThat(e.getDetailedMessage(), containsString("Unknown index [cluster-a:nomatch]")); - - String limit0 = q + " | LIMIT 0"; - e = expectThrows(VerificationException.class, () -> runQuery(limit0, requestIncludeMeta)); - assertThat(e.getDetailedMessage(), containsString("Unknown index [cluster-a:nomatch]")); - } - - // an error is thrown if there are no matching indices at all - single remote cluster with wildcard index expression - { - String q = "FROM cluster-a:nomatch*"; - VerificationException e = expectThrows(VerificationException.class, () -> runQuery(q, requestIncludeMeta)); - assertThat(e.getDetailedMessage(), containsString("Unknown index [cluster-a:nomatch*]")); - - String limit0 = q + " | LIMIT 0"; - e = expectThrows(VerificationException.class, () -> runQuery(limit0, requestIncludeMeta)); - assertThat(e.getDetailedMessage(), containsString("Unknown index [cluster-a:nomatch*]")); - } - - // an error is thrown if there are no matching indices at all - local with wildcard, remote with concrete - { - String q = "FROM nomatch*,cluster-a:nomatch"; - VerificationException e = expectThrows(VerificationException.class, () -> runQuery(q, requestIncludeMeta)); - assertThat(e.getDetailedMessage(), containsString("Unknown index [cluster-a:nomatch,nomatch*]")); - - String limit0 = q + " | LIMIT 0"; - e = expectThrows(VerificationException.class, () -> runQuery(limit0, requestIncludeMeta)); - assertThat(e.getDetailedMessage(), containsString("Unknown index [cluster-a:nomatch,nomatch*]")); - } - - // an error is thrown if there are no matching indices at all - local with wildcard, remote with wildcard - { - String q = "FROM nomatch*,cluster-a:nomatch*"; - VerificationException e = expectThrows(VerificationException.class, () -> runQuery(q, requestIncludeMeta)); - assertThat(e.getDetailedMessage(), containsString("Unknown index [cluster-a:nomatch*,nomatch*]")); - - String limit0 = q + " | LIMIT 0"; - e = expectThrows(VerificationException.class, () -> runQuery(limit0, requestIncludeMeta)); - assertThat(e.getDetailedMessage(), containsString("Unknown index [cluster-a:nomatch*,nomatch*]")); - } - - // an error is thrown if there are no matching indices at all - local with concrete, remote with concrete - { - String q = "FROM nomatch,cluster-a:nomatch"; - VerificationException e = expectThrows(VerificationException.class, () -> runQuery(q, requestIncludeMeta)); - assertThat(e.getDetailedMessage(), containsString("Unknown index [cluster-a:nomatch,nomatch]")); - - String limit0 = q + " | LIMIT 0"; - e = expectThrows(VerificationException.class, () -> runQuery(limit0, requestIncludeMeta)); - assertThat(e.getDetailedMessage(), containsString("Unknown index [cluster-a:nomatch,nomatch]")); - } - - // an error is thrown if there are no matching indices at all - local with concrete, remote with wildcard - { - String q = "FROM nomatch,cluster-a:nomatch*"; - VerificationException e = expectThrows(VerificationException.class, () -> runQuery(q, requestIncludeMeta)); - assertThat(e.getDetailedMessage(), containsString("Unknown index [cluster-a:nomatch*,nomatch]")); - - String limit0 = q + " | LIMIT 0"; - e = expectThrows(VerificationException.class, () -> runQuery(limit0, requestIncludeMeta)); - assertThat(e.getDetailedMessage(), containsString("Unknown index [cluster-a:nomatch*,nomatch]")); - } - - // Missing concrete index on skip_unavailable=false cluster is a fatal error, even when another index expression - // against that cluster matches - { - String remote2IndexName = randomFrom(remote2Index, IDX_ALIAS, FILTERED_IDX_ALIAS); - String q = Strings.format("FROM %s,cluster-a:nomatch,cluster-a:%s*", localIndex, remote2IndexName); - IndexNotFoundException e = expectThrows(IndexNotFoundException.class, () -> runQuery(q, requestIncludeMeta)); - assertThat(e.getDetailedMessage(), containsString("no such index [nomatch]")); - - // TODO: in follow on PR, add support for throwing a VerificationException from this scenario - // String limit0 = q + " | LIMIT 0"; - // VerificationException e = expectThrows(VerificationException.class, () -> runQuery(limit0, requestIncludeMeta)); - // assertThat(e.getDetailedMessage(), containsString("Unknown index [cluster-a:nomatch*,nomatch]")); - } - - // --- test against 3 clusters - - // skip_unavailable=false cluster having no matching indices is a fatal error. This error - // is fatal at plan time, so it throws VerificationException, not IndexNotFoundException (thrown at execution time) - { - String localIndexName = randomFrom(localIndex, IDX_ALIAS, FILTERED_IDX_ALIAS); - String remote2IndexName = randomFrom(remote2Index, IDX_ALIAS, FILTERED_IDX_ALIAS); - String q = Strings.format("FROM %s*,cluster-a:nomatch,%s:%s*", localIndexName, REMOTE_CLUSTER_2, remote2IndexName); - VerificationException e = expectThrows(VerificationException.class, () -> runQuery(q, requestIncludeMeta)); - assertThat(e.getDetailedMessage(), containsString("Unknown index [cluster-a:nomatch]")); - - String limit0 = q + " | LIMIT 0"; - e = expectThrows(VerificationException.class, () -> runQuery(limit0, requestIncludeMeta)); - assertThat(e.getDetailedMessage(), containsString("Unknown index [cluster-a:nomatch]")); - } + record ExpectedCluster(String clusterAlias, String indexExpression, EsqlExecutionInfo.Cluster.Status status, Integer totalShards) {} - // skip_unavailable=false cluster having no matching indices is a fatal error (even if wildcarded) - { - String localIndexName = randomFrom(localIndex, IDX_ALIAS, FILTERED_IDX_ALIAS); - String remote2IndexName = randomFrom(remote2Index, IDX_ALIAS, FILTERED_IDX_ALIAS); - String q = Strings.format("FROM %s*,cluster-a:nomatch*,%s:%s*", localIndexName, REMOTE_CLUSTER_2, remote2IndexName); - VerificationException e = expectThrows(VerificationException.class, () -> runQuery(q, requestIncludeMeta)); - assertThat(e.getDetailedMessage(), containsString("Unknown index [cluster-a:nomatch*]")); + /** + * Runs the provided query, expecting a VerificationError. It then runs the same query with a "| LIMIT 0" + * extra processing step to ensure that ESQL coordinator-only operations throw the same VerificationError. + */ + private void expectVerificationExceptionForQuery(String query, String error, Boolean requestIncludeMeta) { + VerificationException e = expectThrows(VerificationException.class, () -> runQuery(query, requestIncludeMeta)); + assertThat(e.getDetailedMessage(), containsString(error)); - String limit0 = q + " | LIMIT 0"; - e = expectThrows(VerificationException.class, () -> runQuery(limit0, requestIncludeMeta)); - assertThat(e.getDetailedMessage(), containsString("Unknown index [cluster-a:nomatch*]")); - } - } finally { - clearSkipUnavailable(); - } + String limit0 = query + " | LIMIT 0"; + e = expectThrows(VerificationException.class, () -> runQuery(limit0, requestIncludeMeta)); + assertThat(e.getDetailedMessage(), containsString(error)); } - record ExpectedCluster(String clusterAlias, String indexExpression, EsqlExecutionInfo.Cluster.Status status, Integer totalShards) {} - public void assertExpectedClustersForMissingIndicesTests(EsqlExecutionInfo executionInfo, List expected) { long overallTookMillis = executionInfo.overallTook().millis(); assertThat(overallTookMillis, greaterThanOrEqualTo(0L)); diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersUsageTelemetryIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersUsageTelemetryIT.java index 33d868e7a69eb..cd30ab02676fc 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersUsageTelemetryIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/CrossClustersUsageTelemetryIT.java @@ -91,35 +91,32 @@ public void testFailed() throws Exception { assertThat(telemetry.getSuccessCount(), equalTo(0L)); assertThat(telemetry.getByRemoteCluster().size(), equalTo(0)); - // One remote is skipped, one is not + // Errors from both remotes telemetry = getTelemetryFromFailedQuery("from logs-*,c*:no_such_index | stats sum (v)"); assertThat(telemetry.getTotalCount(), equalTo(1L)); assertThat(telemetry.getSuccessCount(), equalTo(0L)); - assertThat(telemetry.getByRemoteCluster().size(), equalTo(1)); + assertThat(telemetry.getByRemoteCluster().size(), equalTo(0)); assertThat(telemetry.getRemotesPerSearchAvg(), equalTo(2.0)); assertThat(telemetry.getRemotesPerSearchMax(), equalTo(2L)); - assertThat(telemetry.getSearchCountWithSkippedRemotes(), equalTo(1L)); + assertThat(telemetry.getSearchCountWithSkippedRemotes(), equalTo(0L)); Map expectedFailure = Map.of(CCSUsageTelemetry.Result.NOT_FOUND.getName(), 1L); assertThat(telemetry.getFailureReasons(), equalTo(expectedFailure)); - // cluster-b should be skipped - assertThat(telemetry.getByRemoteCluster().get(REMOTE2).getCount(), equalTo(0L)); - assertThat(telemetry.getByRemoteCluster().get(REMOTE2).getSkippedCount(), equalTo(1L)); // this is only for cluster-a so no skipped remotes telemetry = getTelemetryFromFailedQuery("from logs-*,cluster-a:no_such_index | stats sum (v)"); assertThat(telemetry.getTotalCount(), equalTo(2L)); assertThat(telemetry.getSuccessCount(), equalTo(0L)); - assertThat(telemetry.getByRemoteCluster().size(), equalTo(1)); + assertThat(telemetry.getByRemoteCluster().size(), equalTo(0)); assertThat(telemetry.getRemotesPerSearchAvg(), equalTo(2.0)); assertThat(telemetry.getRemotesPerSearchMax(), equalTo(2L)); - assertThat(telemetry.getSearchCountWithSkippedRemotes(), equalTo(1L)); + assertThat(telemetry.getSearchCountWithSkippedRemotes(), equalTo(0L)); expectedFailure = Map.of(CCSUsageTelemetry.Result.NOT_FOUND.getName(), 2L); assertThat(telemetry.getFailureReasons(), equalTo(expectedFailure)); - assertThat(telemetry.getByRemoteCluster().size(), equalTo(1)); + assertThat(telemetry.getByRemoteCluster().size(), equalTo(0)); } - // TODO: enable when skip-up patch is merged + // TODO: enable when skip-un patch is merged // public void testSkipAllRemotes() throws Exception { // var telemetry = getTelemetryFromQuery("from logs-*,c*:no_such_index | stats sum (v)", "unknown"); // diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSessionCCSUtils.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSessionCCSUtils.java index f8670a8e6d053..304a54741d44b 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSessionCCSUtils.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSessionCCSUtils.java @@ -199,15 +199,13 @@ static void updateExecutionInfoWithClustersWithNoMatchingIndices(EsqlExecutionIn /** * Rules enforced at planning time around non-matching indices - * P1. fail query if no matching indices on any cluster (VerificationException) - that is handled elsewhere (TODO: document where) - * P2. fail query if a skip_unavailable:false cluster has no matching indices (the local cluster already has this rule - * enforced at planning time) - * P3. fail query if the local cluster has no matching indices and a concrete index was specified + * 1. fail query if no matching indices on any cluster (VerificationException) - that is handled elsewhere + * 2. fail query if a cluster has no matching indices *and* a concrete index was specified - handled here */ String fatalErrorMessage = null; /* * These are clusters in the original request that are not present in the field-caps response. They were - * specified with an index expression matched no indices, so the search on that cluster is done. + * specified with an index expression that matched no indices, so the search on that cluster is done. * Mark it as SKIPPED with 0 shards searched and took=0. */ for (String c : clustersWithNoMatchingIndices) { @@ -216,7 +214,7 @@ static void updateExecutionInfoWithClustersWithNoMatchingIndices(EsqlExecutionIn continue; } final String indexExpression = executionInfo.getCluster(c).getIndexExpression(); - if (missingIndicesIsFatal(c, executionInfo)) { + if (concreteIndexRequested(executionInfo.getCluster(c).getIndexExpression())) { String error = Strings.format( "Unknown index [%s]", (c.equals(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY) ? indexExpression : c + ":" + indexExpression) @@ -227,10 +225,11 @@ static void updateExecutionInfoWithClustersWithNoMatchingIndices(EsqlExecutionIn fatalErrorMessage += "; " + error; } } else { - // handles local cluster (when no concrete indices requested) and skip_unavailable=true clusters + // no matching indices and no concrete index requested - just skip it, no error EsqlExecutionInfo.Cluster.Status status; ShardSearchFailure failure; if (c.equals(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY)) { + // never mark local cluster as SKIPPED status = EsqlExecutionInfo.Cluster.Status.SUCCESSFUL; failure = null; } else { @@ -257,15 +256,7 @@ static void updateExecutionInfoWithClustersWithNoMatchingIndices(EsqlExecutionIn } // visible for testing - static boolean missingIndicesIsFatal(String clusterAlias, EsqlExecutionInfo executionInfo) { - // missing indices on local cluster is fatal only if a concrete index requested - if (clusterAlias.equals(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY)) { - return concreteIndexRequested(executionInfo.getCluster(clusterAlias).getIndexExpression()); - } - return executionInfo.getCluster(clusterAlias).isSkipUnavailable() == false; - } - - private static boolean concreteIndexRequested(String indexExpression) { + static boolean concreteIndexRequested(String indexExpression) { if (Strings.isNullOrBlank(indexExpression)) { return false; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java index d000b2765e2b1..b11a8580a1e18 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java @@ -94,7 +94,6 @@ public void resolveAsMergedMapping( public IndexResolution mergedMappings(String indexPattern, FieldCapabilitiesResponse fieldCapsResponse) { assert ThreadPool.assertCurrentThreadPool(ThreadPool.Names.SEARCH_COORDINATION); // too expensive to run this on a transport worker if (fieldCapsResponse.getIndexResponses().isEmpty()) { - // TODO in follow-on PR, handle the case where remotes were specified with non-existent indices, according to skip_unavailable return IndexResolution.notFound(indexPattern); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/EsqlSessionCCSUtilsTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/EsqlSessionCCSUtilsTests.java index 6b01010ffa5f4..05d04ff1315e6 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/EsqlSessionCCSUtilsTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/EsqlSessionCCSUtilsTests.java @@ -220,17 +220,22 @@ public void testUpdateExecutionInfoWithClustersWithNoMatchingIndices() { final String localClusterAlias = RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY; final String remote1Alias = "remote1"; final String remote2Alias = "remote2"; - // all clusters present in EsIndex, so no updates to EsqlExecutionInfo should happen + + // all clusters had matching indices from field-caps call, so no updates to EsqlExecutionInfo should happen { EsqlExecutionInfo executionInfo = new EsqlExecutionInfo(true); executionInfo.swapCluster(localClusterAlias, (k, v) -> new EsqlExecutionInfo.Cluster(localClusterAlias, "logs*", false)); - executionInfo.swapCluster(remote1Alias, (k, v) -> new EsqlExecutionInfo.Cluster(remote1Alias, "*", true)); - executionInfo.swapCluster(remote2Alias, (k, v) -> new EsqlExecutionInfo.Cluster(remote2Alias, "mylogs1,mylogs2,logs*", false)); + executionInfo.swapCluster(remote1Alias, (k, v) -> new EsqlExecutionInfo.Cluster(remote1Alias, "*", randomBoolean())); + executionInfo.swapCluster( + remote2Alias, + (k, v) -> new EsqlExecutionInfo.Cluster(remote2Alias, "mylogs1,mylogs2,logs*", randomBoolean()) + ); EsIndex esIndex = new EsIndex( - "logs*,remote1:*,remote2:mylogs1,remote2:mylogs2,remote2:logs*", + "logs*,remote1:*,remote2:mylogs1,remote2:mylogs2,remote2:logs*", // original user-provided index expression randomMapping(), Map.of( + // resolved indices from field-caps (all clusters represented) "logs-a", IndexMode.STANDARD, "remote1:logs-a", @@ -261,17 +266,22 @@ public void testUpdateExecutionInfoWithClustersWithNoMatchingIndices() { assertClusterStatusAndShardCounts(remote2Cluster, EsqlExecutionInfo.Cluster.Status.RUNNING); } - // remote1 is missing from EsIndex info, so it should be updated and marked as SKIPPED with 0 total shards, 0 took time, etc. + // remote1 had no matching indices from field-caps call, it was not marked as unavailable, so it should be updated and + // marked as SKIPPED with 0 total shards, 0 took time, etc. { EsqlExecutionInfo executionInfo = new EsqlExecutionInfo(true); executionInfo.swapCluster(localClusterAlias, (k, v) -> new EsqlExecutionInfo.Cluster(localClusterAlias, "logs*", false)); - executionInfo.swapCluster(remote1Alias, (k, v) -> new EsqlExecutionInfo.Cluster(remote1Alias, "*", true)); - executionInfo.swapCluster(remote2Alias, (k, v) -> new EsqlExecutionInfo.Cluster(remote2Alias, "mylogs1,mylogs2,logs*", false)); + executionInfo.swapCluster(remote1Alias, (k, v) -> new EsqlExecutionInfo.Cluster(remote1Alias, "*", randomBoolean())); + executionInfo.swapCluster( + remote2Alias, + (k, v) -> new EsqlExecutionInfo.Cluster(remote2Alias, "mylogs1,mylogs2,logs*", randomBoolean()) + ); EsIndex esIndex = new EsIndex( - "logs*,remote2:mylogs1,remote2:mylogs2,remote2:logs*", + "logs*,remote2:mylogs1,remote2:mylogs2,remote2:logs*", // original user-provided index expression randomMapping(), Map.of( + // resolved indices from field-caps (none from remote1) "logs-a", IndexMode.STANDARD, "remote2:mylogs1", @@ -282,7 +292,8 @@ public void testUpdateExecutionInfoWithClustersWithNoMatchingIndices() { IndexMode.STANDARD ) ); - IndexResolution indexResolution = IndexResolution.valid(esIndex, esIndex.concreteIndices(), Map.of()); + Map unavailableClusters = Map.of(); + IndexResolution indexResolution = IndexResolution.valid(esIndex, esIndex.concreteIndices(), unavailableClusters); EsqlSessionCCSUtils.updateExecutionInfoWithClustersWithNoMatchingIndices(executionInfo, indexResolution); @@ -304,21 +315,27 @@ public void testUpdateExecutionInfoWithClustersWithNoMatchingIndices() { assertClusterStatusAndShardCounts(remote2Cluster, EsqlExecutionInfo.Cluster.Status.RUNNING); } - // all remotes are missing from EsIndex info, so they should be updated and marked as SKIPPED with 0 total shards, 0 took time, etc. + // No remotes had matching indices from field-caps call: 1) remote1 because it was unavailable, 2) remote2 was available, + // but had no matching indices and since no concrete indices were requested, no VerificationException is thrown and is just + // marked as SKIPPED { EsqlExecutionInfo executionInfo = new EsqlExecutionInfo(true); executionInfo.swapCluster(localClusterAlias, (k, v) -> new EsqlExecutionInfo.Cluster(localClusterAlias, "logs*", false)); - executionInfo.swapCluster(remote1Alias, (k, v) -> new EsqlExecutionInfo.Cluster(remote1Alias, "*", true)); - executionInfo.swapCluster(remote2Alias, (k, v) -> new EsqlExecutionInfo.Cluster(remote2Alias, "mylogs1,mylogs2,logs*", true)); + executionInfo.swapCluster(remote1Alias, (k, v) -> new EsqlExecutionInfo.Cluster(remote1Alias, "*", randomBoolean())); + executionInfo.swapCluster( + remote2Alias, + (k, v) -> new EsqlExecutionInfo.Cluster(remote2Alias, "mylogs1*,mylogs2*,logs*", randomBoolean()) + ); EsIndex esIndex = new EsIndex( - "logs*,remote2:mylogs1,remote2:mylogs2,remote2:logs*", + "logs*,remote2:mylogs1*,remote2:mylogs2*,remote2:logs*", // original user-provided index expression randomMapping(), - Map.of("logs-a", IndexMode.STANDARD) + Map.of("logs-a", IndexMode.STANDARD) // resolved indices from field-caps (none from either remote) ); // remote1 is unavailable var failure = new FieldCapabilitiesFailure(new String[] { "logs-a" }, new NoSeedNodeLeftException("unable to connect")); - IndexResolution indexResolution = IndexResolution.valid(esIndex, esIndex.concreteIndices(), Map.of(remote1Alias, failure)); + Map unavailableClusters = Map.of(remote1Alias, failure); + IndexResolution indexResolution = IndexResolution.valid(esIndex, esIndex.concreteIndices(), unavailableClusters); EsqlSessionCCSUtils.updateExecutionInfoWithClustersWithNoMatchingIndices(executionInfo, indexResolution); @@ -333,7 +350,7 @@ public void testUpdateExecutionInfoWithClustersWithNoMatchingIndices() { assertThat(remote1Cluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.RUNNING)); EsqlExecutionInfo.Cluster remote2Cluster = executionInfo.getCluster(remote2Alias); - assertThat(remote2Cluster.getIndexExpression(), equalTo("mylogs1,mylogs2,logs*")); + assertThat(remote2Cluster.getIndexExpression(), equalTo("mylogs1*,mylogs2*,logs*")); assertThat(remote2Cluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SKIPPED)); assertThat(remote2Cluster.getTook().millis(), equalTo(0L)); assertThat(remote2Cluster.getTotalShards(), equalTo(0)); @@ -342,28 +359,77 @@ public void testUpdateExecutionInfoWithClustersWithNoMatchingIndices() { assertThat(remote2Cluster.getFailedShards(), equalTo(0)); } - // all remotes are missing from EsIndex info. Since one is configured with skip_unavailable=false, - // an exception should be thrown + // No remotes had matching indices from field-caps call: 1) remote1 because it was unavailable, 2) remote2 was available, + // but had no matching indices and since a concrete index was requested, a VerificationException is thrown { EsqlExecutionInfo executionInfo = new EsqlExecutionInfo(true); executionInfo.swapCluster(localClusterAlias, (k, v) -> new EsqlExecutionInfo.Cluster(localClusterAlias, "logs*")); - executionInfo.swapCluster(remote1Alias, (k, v) -> new EsqlExecutionInfo.Cluster(remote1Alias, "*", true)); - executionInfo.swapCluster(remote2Alias, (k, v) -> new EsqlExecutionInfo.Cluster(remote2Alias, "mylogs1,mylogs2,logs*", false)); + executionInfo.swapCluster(remote1Alias, (k, v) -> new EsqlExecutionInfo.Cluster(remote1Alias, "*", randomBoolean())); + executionInfo.swapCluster( + remote2Alias, + (k, v) -> new EsqlExecutionInfo.Cluster(remote2Alias, "mylogs1,mylogs2,logs*", randomBoolean()) + ); EsIndex esIndex = new EsIndex( - "logs*,remote2:mylogs1,remote2:mylogs2,remote2:logs*", + "logs*,remote2:mylogs1,remote2:mylogs2,remote2:logs*", // original user-provided index expression randomMapping(), - Map.of("logs-a", IndexMode.STANDARD) + Map.of("logs-a", IndexMode.STANDARD) // resolved indices from field-caps (none from either remote) ); var failure = new FieldCapabilitiesFailure(new String[] { "logs-a" }, new NoSeedNodeLeftException("unable to connect")); - IndexResolution indexResolution = IndexResolution.valid(esIndex, esIndex.concreteIndices(), Map.of(remote1Alias, failure)); + Map unavailableClusters = Map.of(remote1Alias, failure); + IndexResolution indexResolution = IndexResolution.valid(esIndex, esIndex.concreteIndices(), unavailableClusters); VerificationException ve = expectThrows( VerificationException.class, () -> EsqlSessionCCSUtils.updateExecutionInfoWithClustersWithNoMatchingIndices(executionInfo, indexResolution) ); assertThat(ve.getDetailedMessage(), containsString("Unknown index [remote2:mylogs1,mylogs2,logs*]")); } + + // test where remote2 is already marked as SKIPPED so no modifications or exceptions should be thrown + // (the EsqlSessionCCSUtils.updateExecutionInfoWithUnavailableClusters() method handles that case not the one tested here) + { + EsqlExecutionInfo executionInfo = new EsqlExecutionInfo(true); + executionInfo.swapCluster(localClusterAlias, (k, v) -> new EsqlExecutionInfo.Cluster(localClusterAlias, "logs*")); + executionInfo.swapCluster(remote1Alias, (k, v) -> new EsqlExecutionInfo.Cluster(remote1Alias, "*", randomBoolean())); + // remote2 is already marked as SKIPPED (simulating failed enrich policy lookup due to unavailable cluster) + executionInfo.swapCluster( + remote2Alias, + (k, v) -> new EsqlExecutionInfo.Cluster( + remote2Alias, + "mylogs1*,mylogs2*,logs*", + randomBoolean(), + EsqlExecutionInfo.Cluster.Status.SKIPPED + ) + ); + + EsIndex esIndex = new EsIndex( + "logs*,remote2:mylogs1,remote2:mylogs2,remote2:logs*", // original user-provided index expression + randomMapping(), + Map.of("logs-a", IndexMode.STANDARD) // resolved indices from field-caps (none from either remote) + ); + + // remote1 is unavailable + var failure = new FieldCapabilitiesFailure(new String[] { "logs-a" }, new NoSeedNodeLeftException("unable to connect")); + Map unavailableClusters = Map.of(remote1Alias, failure); + IndexResolution indexResolution = IndexResolution.valid(esIndex, esIndex.concreteIndices(), unavailableClusters); + + EsqlSessionCCSUtils.updateExecutionInfoWithClustersWithNoMatchingIndices(executionInfo, indexResolution); + + EsqlExecutionInfo.Cluster localCluster = executionInfo.getCluster(localClusterAlias); + assertThat(localCluster.getIndexExpression(), equalTo("logs*")); + assertClusterStatusAndShardCounts(localCluster, EsqlExecutionInfo.Cluster.Status.RUNNING); + + EsqlExecutionInfo.Cluster remote1Cluster = executionInfo.getCluster(remote1Alias); + assertThat(remote1Cluster.getIndexExpression(), equalTo("*")); + // since remote1 is in the unavailable Map (passed to IndexResolution.valid), it's status will not be changed + // by updateExecutionInfoWithClustersWithNoMatchingIndices (it is handled in updateExecutionInfoWithUnavailableClusters) + assertThat(remote1Cluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.RUNNING)); + + EsqlExecutionInfo.Cluster remote2Cluster = executionInfo.getCluster(remote2Alias); + assertThat(remote2Cluster.getIndexExpression(), equalTo("mylogs1*,mylogs2*,logs*")); + assertThat(remote2Cluster.getStatus(), equalTo(EsqlExecutionInfo.Cluster.Status.SKIPPED)); + } } public void testDetermineUnavailableRemoteClusters() { @@ -600,46 +666,12 @@ public void testUpdateExecutionInfoToReturnEmptyResult() { } } - public void testMissingIndicesIsFatal() { - String localClusterAlias = RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY; - String remote1Alias = "remote1"; - String remote2Alias = "remote2"; - String remote3Alias = "remote3"; - - // scenario 1: cluster is skip_unavailable=true - not fatal - { - EsqlExecutionInfo executionInfo = new EsqlExecutionInfo(true); - executionInfo.swapCluster(localClusterAlias, (k, v) -> new EsqlExecutionInfo.Cluster(localClusterAlias, "logs*", false)); - executionInfo.swapCluster(remote1Alias, (k, v) -> new EsqlExecutionInfo.Cluster(remote1Alias, "mylogs1,mylogs2,logs*", true)); - assertThat(EsqlSessionCCSUtils.missingIndicesIsFatal(remote1Alias, executionInfo), equalTo(false)); - } - - // scenario 2: cluster is local cluster and had no concrete indices - not fatal - { - EsqlExecutionInfo executionInfo = new EsqlExecutionInfo(true); - executionInfo.swapCluster(localClusterAlias, (k, v) -> new EsqlExecutionInfo.Cluster(localClusterAlias, "logs*", false)); - executionInfo.swapCluster(remote1Alias, (k, v) -> new EsqlExecutionInfo.Cluster(remote1Alias, "mylogs1,mylogs2,logs*", true)); - assertThat(EsqlSessionCCSUtils.missingIndicesIsFatal(localClusterAlias, executionInfo), equalTo(false)); - } - - // scenario 3: cluster is local cluster and user specified a concrete index - fatal - { - EsqlExecutionInfo executionInfo = new EsqlExecutionInfo(true); - String localIndexExpr = randomFrom("foo*,logs", "logs", "logs,metrics", "bar*,x*,logs", "logs-1,*x*"); - executionInfo.swapCluster(localClusterAlias, (k, v) -> new EsqlExecutionInfo.Cluster(localClusterAlias, localIndexExpr, false)); - executionInfo.swapCluster(remote1Alias, (k, v) -> new EsqlExecutionInfo.Cluster(remote1Alias, "mylogs1,mylogs2,logs*", true)); - assertThat(EsqlSessionCCSUtils.missingIndicesIsFatal(localClusterAlias, executionInfo), equalTo(true)); - } - - // scenario 4: cluster is skip_unavailable=false - always fatal - { - EsqlExecutionInfo executionInfo = new EsqlExecutionInfo(true); - executionInfo.swapCluster(localClusterAlias, (k, v) -> new EsqlExecutionInfo.Cluster(localClusterAlias, "*", false)); - String indexExpr = randomFrom("foo*,logs", "logs", "bar*,x*,logs", "logs-1,*x*", "*"); - executionInfo.swapCluster(remote1Alias, (k, v) -> new EsqlExecutionInfo.Cluster(remote1Alias, indexExpr, false)); - assertThat(EsqlSessionCCSUtils.missingIndicesIsFatal(remote1Alias, executionInfo), equalTo(true)); - } - + public void testConcreteIndexRequested() { + assertThat(EsqlSessionCCSUtils.concreteIndexRequested("logs*"), equalTo(false)); + assertThat(EsqlSessionCCSUtils.concreteIndexRequested("mylogs1,mylogs2,logs*"), equalTo(true)); + assertThat(EsqlSessionCCSUtils.concreteIndexRequested("x*,logs"), equalTo(true)); + assertThat(EsqlSessionCCSUtils.concreteIndexRequested("logs,metrics"), equalTo(true)); + assertThat(EsqlSessionCCSUtils.concreteIndexRequested("*"), equalTo(false)); } public void testCheckForCcsLicense() { diff --git a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/CrossClusterEsqlRCS1MissingIndicesIT.java b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/CrossClusterEsqlRCS1MissingIndicesIT.java index 8bccc2e3c5c23..8d8629db96fc6 100644 --- a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/CrossClusterEsqlRCS1MissingIndicesIT.java +++ b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/CrossClusterEsqlRCS1MissingIndicesIT.java @@ -14,7 +14,6 @@ import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.json.JsonXContent; -import org.hamcrest.Matchers; import org.junit.ClassRule; import org.junit.rules.RuleChain; import org.junit.rules.TestRule; @@ -125,228 +124,11 @@ void assertExpectedClustersForMissingIndicesTests(Map responseMa } @SuppressWarnings("unchecked") - public void testSearchesAgainstNonMatchingIndicesWithSkipUnavailableTrue() throws Exception { + public void testSearchesAgainstNonMatchingIndices() throws Exception { setupRolesAndPrivileges(); setupIndex(); - configureRemoteCluster(REMOTE_CLUSTER_ALIAS, fulfillingCluster, true, randomBoolean(), true); - - // missing concrete local index is an error - { - String q = Strings.format("FROM nomatch,%s:%s | STATS count(*)", REMOTE_CLUSTER_ALIAS, INDEX2); - - String limit1 = q + " | LIMIT 1"; - ResponseException e = expectThrows(ResponseException.class, () -> client().performRequest(esqlRequest(limit1))); - assertThat(e.getMessage(), containsString("Unknown index [nomatch]")); - - String limit0 = q + " | LIMIT 0"; - e = expectThrows(ResponseException.class, () -> client().performRequest(esqlRequest(limit0))); - assertThat(e.getMessage(), Matchers.containsString("Unknown index [nomatch]")); - } - - // missing concrete remote index is not fatal when skip_unavailable=true (as long as an index matches on another cluster) - { - String q = Strings.format("FROM %s,%s:nomatch | STATS count(*)", INDEX1, REMOTE_CLUSTER_ALIAS); - - String limit1 = q + " | LIMIT 1"; - Response response = client().performRequest(esqlRequest(limit1)); - assertOK(response); - - Map map = responseAsMap(response); - assertThat(((ArrayList) map.get("columns")).size(), greaterThanOrEqualTo(1)); - assertThat(((ArrayList) map.get("values")).size(), greaterThanOrEqualTo(1)); - - assertExpectedClustersForMissingIndicesTests( - map, - List.of( - new ExpectedCluster("(local)", INDEX1, "successful", null), - new ExpectedCluster(REMOTE_CLUSTER_ALIAS, "nomatch", "skipped", 0) - ) - ); - - String limit0 = q + " | LIMIT 0"; - response = client().performRequest(esqlRequest(limit0)); - assertOK(response); - - map = responseAsMap(response); - assertThat(((ArrayList) map.get("columns")).size(), greaterThanOrEqualTo(1)); - assertThat(((ArrayList) map.get("values")).size(), is(0)); - - assertExpectedClustersForMissingIndicesTests( - map, - List.of( - new ExpectedCluster("(local)", INDEX1, "successful", 0), - new ExpectedCluster(REMOTE_CLUSTER_ALIAS, "nomatch", "skipped", 0) - ) - ); - } - - // since there is at least one matching index in the query, the missing wildcarded local index is not an error - { - String q = Strings.format("FROM nomatch*,%s:%s", REMOTE_CLUSTER_ALIAS, INDEX2); - - String limit1 = q + " | LIMIT 1"; - Response response = client().performRequest(esqlRequest(limit1)); - assertOK(response); - - Map map = responseAsMap(response); - assertThat(((ArrayList) map.get("columns")).size(), greaterThanOrEqualTo(1)); - assertThat(((ArrayList) map.get("values")).size(), greaterThanOrEqualTo(1)); - - assertExpectedClustersForMissingIndicesTests( - map, - List.of( - // local cluster is never marked as SKIPPED even when no matching indices - just marked as 0 shards searched - new ExpectedCluster("(local)", "nomatch*", "successful", 0), - new ExpectedCluster(REMOTE_CLUSTER_ALIAS, INDEX2, "successful", null) - ) - ); - - String limit0 = q + " | LIMIT 0"; - response = client().performRequest(esqlRequest(limit0)); - assertOK(response); - - map = responseAsMap(response); - assertThat(((ArrayList) map.get("columns")).size(), greaterThanOrEqualTo(1)); - assertThat(((ArrayList) map.get("values")).size(), is(0)); - - assertExpectedClustersForMissingIndicesTests( - map, - List.of( - // local cluster is never marked as SKIPPED even when no matching indices - just marked as 0 shards searched - new ExpectedCluster("(local)", "nomatch*", "successful", 0), - new ExpectedCluster(REMOTE_CLUSTER_ALIAS, INDEX2, "successful", 0) - ) - ); - } - - // since at least one index of the query matches on some cluster, a missing wildcarded index on skip_un=true is not an error - { - String q = Strings.format("FROM %s,%s:nomatch*", INDEX1, REMOTE_CLUSTER_ALIAS); - - String limit1 = q + " | LIMIT 1"; - Response response = client().performRequest(esqlRequest(limit1)); - assertOK(response); - - Map map = responseAsMap(response); - assertThat(((ArrayList) map.get("columns")).size(), greaterThanOrEqualTo(1)); - assertThat(((ArrayList) map.get("values")).size(), greaterThanOrEqualTo(1)); - - assertExpectedClustersForMissingIndicesTests( - map, - List.of( - new ExpectedCluster("(local)", INDEX1, "successful", null), - new ExpectedCluster(REMOTE_CLUSTER_ALIAS, "nomatch*", "skipped", 0) - ) - ); - - String limit0 = q + " | LIMIT 0"; - response = client().performRequest(esqlRequest(limit0)); - assertOK(response); - - map = responseAsMap(response); - assertThat(((ArrayList) map.get("columns")).size(), greaterThanOrEqualTo(1)); - assertThat(((ArrayList) map.get("values")).size(), is(0)); - - assertExpectedClustersForMissingIndicesTests( - map, - List.of( - new ExpectedCluster("(local)", INDEX1, "successful", 0), - new ExpectedCluster(REMOTE_CLUSTER_ALIAS, "nomatch*", "skipped", 0) - ) - ); - } - - // an error is thrown if there are no matching indices at all, even when the cluster is skip_unavailable=true - { - // with non-matching concrete index - String q = Strings.format("FROM %s:nomatch", REMOTE_CLUSTER_ALIAS); - - String limit1 = q + " | LIMIT 1"; - ResponseException e = expectThrows(ResponseException.class, () -> client().performRequest(esqlRequest(limit1))); - assertThat(e.getMessage(), containsString(Strings.format("Unknown index [%s:nomatch]", REMOTE_CLUSTER_ALIAS))); - - String limit0 = q + " | LIMIT 0"; - e = expectThrows(ResponseException.class, () -> client().performRequest(esqlRequest(limit0))); - assertThat(e.getMessage(), containsString(Strings.format("Unknown index [%s:nomatch]", REMOTE_CLUSTER_ALIAS))); - } - - // an error is thrown if there are no matching indices at all, even when the cluster is skip_unavailable=true and the - // index was wildcarded - { - String q = Strings.format("FROM %s:nomatch*", REMOTE_CLUSTER_ALIAS); - - String limit1 = q + " | LIMIT 1"; - ResponseException e = expectThrows(ResponseException.class, () -> client().performRequest(esqlRequest(limit1))); - assertThat(e.getMessage(), containsString(Strings.format("Unknown index [%s:nomatch*]", REMOTE_CLUSTER_ALIAS))); - - String limit0 = q + " | LIMIT 0"; - e = expectThrows(ResponseException.class, () -> client().performRequest(esqlRequest(limit0))); - assertThat(e.getMessage(), containsString(Strings.format("Unknown index [%s:nomatch*]", REMOTE_CLUSTER_ALIAS))); - } - - // an error is thrown if there are no matching indices at all - { - String localExpr = randomFrom("nomatch", "nomatch*"); - String remoteExpr = randomFrom("nomatch", "nomatch*"); - String q = Strings.format("FROM %s,%s:%s", localExpr, REMOTE_CLUSTER_ALIAS, remoteExpr); - - String limit1 = q + " | LIMIT 1"; - ResponseException e = expectThrows(ResponseException.class, () -> client().performRequest(esqlRequest(limit1))); - assertThat(e.getMessage(), containsString("Unknown index")); - assertThat(e.getMessage(), containsString(Strings.format("%s:%s", REMOTE_CLUSTER_ALIAS, remoteExpr))); - - String limit0 = q + " | LIMIT 0"; - e = expectThrows(ResponseException.class, () -> client().performRequest(esqlRequest(limit0))); - assertThat(e.getMessage(), containsString("Unknown index")); - assertThat(e.getMessage(), containsString(Strings.format("%s:%s", REMOTE_CLUSTER_ALIAS, remoteExpr))); - } - - // TODO uncomment and test in follow-on PR which does skip_unavailable handling at execution time - // { - // String q = Strings.format("FROM %s,%s:nomatch,%s:%s*", INDEX1, REMOTE_CLUSTER_ALIAS, REMOTE_CLUSTER_ALIAS, INDEX2); - // - // String limit1 = q + " | LIMIT 1"; - // Response response = client().performRequest(esqlRequest(limit1)); - // assertOK(response); - // - // Map map = responseAsMap(response); - // assertThat(((ArrayList) map.get("columns")).size(), greaterThanOrEqualTo(1)); - // assertThat(((ArrayList) map.get("values")).size(), greaterThanOrEqualTo(1)); - // - // assertExpectedClustersForMissingIndicesTests(map, - // List.of( - // new ExpectedCluster("(local)", INDEX1, "successful", null), - // new ExpectedCluster(REMOTE_CLUSTER_ALIAS, "nomatch," + INDEX2 + "*", "skipped", 0) - // ) - // ); - // - // String limit0 = q + " | LIMIT 0"; - // response = client().performRequest(esqlRequest(limit0)); - // assertOK(response); - // - // map = responseAsMap(response); - // assertThat(((ArrayList) map.get("columns")).size(), greaterThanOrEqualTo(1)); - // assertThat(((ArrayList) map.get("values")).size(), is(0)); - // - // assertExpectedClustersForMissingIndicesTests(map, - // List.of( - // new ExpectedCluster("(local)", INDEX1, "successful", 0), - // new ExpectedCluster(REMOTE_CLUSTER_ALIAS, "nomatch," + INDEX2 + "*", "skipped", 0) - // ) - // ); - // } - } - - @SuppressWarnings("unchecked") - public void testSearchesAgainstNonMatchingIndicesWithSkipUnavailableFalse() throws Exception { - // Remote cluster is closed and skip_unavailable is set to false. - // Although the other cluster is open, we expect an Exception. - - setupRolesAndPrivileges(); - setupIndex(); - - configureRemoteCluster(REMOTE_CLUSTER_ALIAS, fulfillingCluster, true, randomBoolean(), false); + configureRemoteCluster(REMOTE_CLUSTER_ALIAS, fulfillingCluster, true, randomBoolean(), randomBoolean()); // missing concrete local index is an error { @@ -361,9 +143,9 @@ public void testSearchesAgainstNonMatchingIndicesWithSkipUnavailableFalse() thro assertThat(e.getMessage(), containsString("Unknown index [nomatch]")); } - // missing concrete remote index is not fatal when skip_unavailable=true (as long as an index matches on another cluster) + // missing concrete index on remote cluster is an error { - String q = Strings.format("FROM %s,%s:nomatch | STATS count(*)", INDEX1, REMOTE_CLUSTER_ALIAS); + String q = Strings.format("FROM %s:nomatch", REMOTE_CLUSTER_ALIAS); String limit1 = q + " | LIMIT 1"; ResponseException e = expectThrows(ResponseException.class, () -> client().performRequest(esqlRequest(limit1))); @@ -413,37 +195,23 @@ public void testSearchesAgainstNonMatchingIndicesWithSkipUnavailableFalse() thro ); } - // query is fatal since the remote cluster has skip_unavailable=false and has no matching indices - { - String q = Strings.format("FROM %s,%s:nomatch*", INDEX1, REMOTE_CLUSTER_ALIAS); - - String limit1 = q + " | LIMIT 1"; - ResponseException e = expectThrows(ResponseException.class, () -> client().performRequest(esqlRequest(limit1))); - assertThat(e.getMessage(), containsString(Strings.format("Unknown index [%s:nomatch*]", REMOTE_CLUSTER_ALIAS))); - - String limit0 = q + " | LIMIT 0"; - e = expectThrows(ResponseException.class, () -> client().performRequest(esqlRequest(limit0))); - assertThat(e.getMessage(), Matchers.containsString(Strings.format("Unknown index [%s:nomatch*]", REMOTE_CLUSTER_ALIAS))); - } - // an error is thrown if there are no matching indices at all { - // with non-matching concrete index - String q = Strings.format("FROM %s:nomatch", REMOTE_CLUSTER_ALIAS); + String q = Strings.format("FROM %s:nomatch*", REMOTE_CLUSTER_ALIAS); String limit1 = q + " | LIMIT 1"; ResponseException e = expectThrows(ResponseException.class, () -> client().performRequest(esqlRequest(limit1))); - assertThat(e.getMessage(), containsString(Strings.format("Unknown index [%s:nomatch]", REMOTE_CLUSTER_ALIAS))); + assertThat(e.getMessage(), containsString(Strings.format("Unknown index [%s:nomatch*]", REMOTE_CLUSTER_ALIAS))); String limit0 = q + " | LIMIT 0"; e = expectThrows(ResponseException.class, () -> client().performRequest(esqlRequest(limit0))); - assertThat(e.getMessage(), containsString(Strings.format("Unknown index [%s:nomatch]", REMOTE_CLUSTER_ALIAS))); + assertThat(e.getMessage(), containsString(Strings.format("Unknown index [%s:nomatch*]", REMOTE_CLUSTER_ALIAS))); } - // an error is thrown if there are no matching indices at all + // an error is thrown if there are no matching indices at all (2 clusters) { - String localExpr = randomFrom("nomatch", "nomatch*"); - String remoteExpr = randomFrom("nomatch", "nomatch*"); + String localExpr = "nomatch*"; + String remoteExpr = "nomatch*"; String q = Strings.format("FROM %s,%s:%s", localExpr, REMOTE_CLUSTER_ALIAS, remoteExpr); String limit1 = q + " | LIMIT 1"; @@ -457,7 +225,7 @@ public void testSearchesAgainstNonMatchingIndicesWithSkipUnavailableFalse() thro assertThat(e.getMessage(), containsString(Strings.format("%s:%s", REMOTE_CLUSTER_ALIAS, remoteExpr))); } - // error since the remote cluster with skip_unavailable=false specified a concrete index that is not found + // error since the remote cluster specified a concrete index that is not found { String q = Strings.format("FROM %s,%s:nomatch,%s:%s*", INDEX1, REMOTE_CLUSTER_ALIAS, REMOTE_CLUSTER_ALIAS, INDEX2); diff --git a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityEsqlIT.java b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityEsqlIT.java index d6bad85161fd9..41f2eab6a00e8 100644 --- a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityEsqlIT.java +++ b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityEsqlIT.java @@ -56,6 +56,7 @@ import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.not; +// uses RCS 2.0 public class RemoteClusterSecurityEsqlIT extends AbstractRemoteClusterSecurityTestCase { private static final AtomicReference> API_KEY_MAP_REF = new AtomicReference<>(); private static final AtomicReference> REST_API_KEY_MAP_REF = new AtomicReference<>(); @@ -995,245 +996,9 @@ public void testAlias() throws Exception { } @SuppressWarnings("unchecked") - public void testSearchesAgainstNonMatchingIndicesWithSkipUnavailableTrue() throws Exception { - configureRemoteCluster(REMOTE_CLUSTER_ALIAS, fulfillingCluster, false, randomBoolean(), true); - populateData(); - { - final var putRoleRequest = new Request("PUT", "/_security/role/" + REMOTE_SEARCH_ROLE); - putRoleRequest.setJsonEntity(""" - { - "indices": [{"names": ["employees*"], "privileges": ["read","read_cross_cluster"]}], - "cluster": [ "manage_own_api_key" ], - "remote_indices": [ - { - "names": ["employees*"], - "privileges": ["read"], - "clusters": ["my_remote_cluster"] - } - ] - }"""); - Response response = adminClient().performRequest(putRoleRequest); - assertOK(response); - } - - String remoteSearchUserAPIKey = createRemoteSearchUserAPIKey(); - - // sanity check - init queries to ensure we can query employees on local and employees,employees2 on remote - { - Request request = esqlRequest(""" - FROM employees,my_remote_cluster:employees,my_remote_cluster:employees2 - | SORT emp_id ASC - | LIMIT 9 - | KEEP emp_id, department"""); - - CheckedConsumer verifier = resp -> { - assertOK(resp); - Map map = responseAsMap(resp); - assertThat(((ArrayList) map.get("columns")).size(), greaterThanOrEqualTo(1)); - assertThat(((ArrayList) map.get("values")).size(), greaterThanOrEqualTo(1)); - assertExpectedClustersForMissingIndicesTests( - map, - List.of( - // local cluster is never marked as SKIPPED even when no matching indices - just marked as 0 shards searched - new ExpectedCluster("(local)", "nomatch*", "successful", null), - new ExpectedCluster(REMOTE_CLUSTER_ALIAS, "employees,employees2", "successful", null) - ) - ); - }; - - verifier.accept(performRequestWithRemoteSearchUser(request)); - verifier.accept(performRequestWithRemoteSearchUserViaAPIKey(request, remoteSearchUserAPIKey)); - } - - // missing concrete local index is an error - { - String q = "FROM employees_nomatch,my_remote_cluster:employees"; - - Request limit1 = esqlRequest(q + " | LIMIT 1"); - ResponseException e = expectThrows(ResponseException.class, () -> performRequestWithRemoteSearchUser(limit1)); - assertThat(e.getMessage(), containsString("Unknown index [employees_nomatch]")); - e = expectThrows(ResponseException.class, () -> performRequestWithRemoteSearchUserViaAPIKey(limit1, remoteSearchUserAPIKey)); - assertThat(e.getMessage(), containsString("Unknown index [employees_nomatch]")); - - Request limit0 = esqlRequest(q + " | LIMIT 0"); - e = expectThrows(ResponseException.class, () -> performRequestWithRemoteSearchUser(limit0)); - assertThat(e.getMessage(), containsString("Unknown index [employees_nomatch]")); - e = expectThrows(ResponseException.class, () -> performRequestWithRemoteSearchUserViaAPIKey(limit0, remoteSearchUserAPIKey)); - assertThat(e.getMessage(), containsString("Unknown index [employees_nomatch]")); - } - - // missing concrete remote index is not fatal when skip_unavailable=true (as long as an index matches on another cluster) - { - String q = "FROM employees,my_remote_cluster:employees_nomatch"; - - CheckedBiConsumer verifier = new CheckedBiConsumer() { - @Override - public void accept(Response response, Boolean limit0) throws Exception { - assertOK(response); - Map map = responseAsMap(response); - assertThat(((List) map.get("columns")).size(), greaterThanOrEqualTo(1)); - if (limit0) { - assertThat(((List) map.get("values")).size(), equalTo(0)); - } else { - assertThat(((List) map.get("values")).size(), greaterThanOrEqualTo(1)); - } - assertExpectedClustersForMissingIndicesTests( - map, - List.of( - new ExpectedCluster("(local)", "employees", "successful", limit0 ? 0 : null), - new ExpectedCluster(REMOTE_CLUSTER_ALIAS, "employees_nomatch", "skipped", 0) - ) - ); - } - }; - Request limit1 = esqlRequest(q + " | LIMIT 1"); - verifier.accept(performRequestWithRemoteSearchUser(limit1), false); - verifier.accept(performRequestWithRemoteSearchUserViaAPIKey(limit1, remoteSearchUserAPIKey), false); - - Request limit0 = esqlRequest(q + " | LIMIT 0"); - verifier.accept(performRequestWithRemoteSearchUser(limit0), true); - verifier.accept(performRequestWithRemoteSearchUserViaAPIKey(limit0, remoteSearchUserAPIKey), true); - } - - // since there is at least one matching index in the query, the missing wildcarded local index is not an error - { - String q = "FROM employees_nomatch*,my_remote_cluster:employees"; - - CheckedBiConsumer verifier = (response, limit0) -> { - assertOK(response); - Map map = responseAsMap(response); - assertThat(((List) map.get("columns")).size(), greaterThanOrEqualTo(1)); - if (limit0) { - assertThat(((List) map.get("values")).size(), equalTo(0)); - } else { - assertThat(((List) map.get("values")).size(), greaterThanOrEqualTo(1)); - } - assertExpectedClustersForMissingIndicesTests( - map, - List.of( - // local cluster is never marked as SKIPPED even when no matching indices - just marked as 0 shards searched - new ExpectedCluster("(local)", "employees_nomatch*", "successful", 0), - new ExpectedCluster(REMOTE_CLUSTER_ALIAS, "employees", "successful", limit0 ? 0 : null) - ) - ); - }; - - Request limit1 = esqlRequest(q + " | LIMIT 1"); - verifier.accept(performRequestWithRemoteSearchUser(limit1), false); - verifier.accept(performRequestWithRemoteSearchUserViaAPIKey(limit1, remoteSearchUserAPIKey), false); - - Request limit0 = esqlRequest(q + " | LIMIT 0"); - verifier.accept(performRequestWithRemoteSearchUser(limit0), true); - verifier.accept(performRequestWithRemoteSearchUserViaAPIKey(limit0, remoteSearchUserAPIKey), true); - } - - // since at least one index of the query matches on some cluster, a missing wildcarded index on skip_un=true is not an error - { - String q = "FROM employees,my_remote_cluster:employees_nomatch*"; - - CheckedBiConsumer verifier = (response, limit0) -> { - assertOK(response); - Map map = responseAsMap(response); - assertThat(((List) map.get("columns")).size(), greaterThanOrEqualTo(1)); - if (limit0) { - assertThat(((List) map.get("values")).size(), equalTo(0)); - } else { - assertThat(((List) map.get("values")).size(), greaterThanOrEqualTo(1)); - } - assertExpectedClustersForMissingIndicesTests( - map, - List.of( - new ExpectedCluster("(local)", "employees", "successful", limit0 ? 0 : null), - new ExpectedCluster("my_remote_cluster", "employees_nomatch*", "skipped", 0) - ) - ); - }; - - Request limit1 = esqlRequest(q + " | LIMIT 1"); - verifier.accept(performRequestWithRemoteSearchUser(limit1), false); - verifier.accept(performRequestWithRemoteSearchUserViaAPIKey(limit1, remoteSearchUserAPIKey), false); - - Request limit0 = esqlRequest(q + " | LIMIT 0"); - verifier.accept(performRequestWithRemoteSearchUser(limit0), true); - verifier.accept(performRequestWithRemoteSearchUserViaAPIKey(limit0, remoteSearchUserAPIKey), true); - } - - // an error is thrown if there are no matching indices at all, even when the cluster is skip_unavailable=true - { - // with non-matching concrete index - String q = "FROM my_remote_cluster:employees_nomatch"; - - Request limit1 = esqlRequest(q + " | LIMIT 1"); - ResponseException e = expectThrows(ResponseException.class, () -> performRequestWithRemoteSearchUser(limit1)); - assertThat(e.getMessage(), containsString("Unknown index [my_remote_cluster:employees_nomatch]")); - e = expectThrows(ResponseException.class, () -> performRequestWithRemoteSearchUserViaAPIKey(limit1, remoteSearchUserAPIKey)); - assertThat(e.getMessage(), containsString("Unknown index [my_remote_cluster:employees_nomatch]")); - - Request limit0 = esqlRequest(q + " | LIMIT 0"); - e = expectThrows(ResponseException.class, () -> performRequestWithRemoteSearchUser(limit0)); - assertThat(e.getMessage(), containsString("Unknown index [my_remote_cluster:employees_nomatch]")); - e = expectThrows(ResponseException.class, () -> performRequestWithRemoteSearchUserViaAPIKey(limit0, remoteSearchUserAPIKey)); - assertThat(e.getMessage(), containsString("Unknown index [my_remote_cluster:employees_nomatch]")); - } - - // an error is thrown if there are no matching indices at all, even when the cluster is skip_unavailable=true and the - // index was wildcarded - { - String localExpr = randomFrom("nomatch", "nomatch*"); - String remoteExpr = randomFrom("nomatch", "nomatch*"); - String q = Strings.format("FROM %s,%s:%s", localExpr, REMOTE_CLUSTER_ALIAS, remoteExpr); - - Request limit1 = esqlRequest(q + " | LIMIT 1"); - ResponseException e = expectThrows(ResponseException.class, () -> performRequestWithRemoteSearchUser(limit1)); - assertThat(e.getMessage(), containsString("Unknown index")); - assertThat(e.getMessage(), containsString(Strings.format("%s:%s", REMOTE_CLUSTER_ALIAS, remoteExpr))); - e = expectThrows(ResponseException.class, () -> performRequestWithRemoteSearchUserViaAPIKey(limit1, remoteSearchUserAPIKey)); - assertThat(e.getMessage(), containsString("Unknown index")); - assertThat(e.getMessage(), containsString(Strings.format("%s:%s", REMOTE_CLUSTER_ALIAS, remoteExpr))); - - Request limit0 = esqlRequest(q + " | LIMIT 0"); - e = expectThrows(ResponseException.class, () -> performRequestWithRemoteSearchUser(limit0)); - assertThat(e.getMessage(), containsString("Unknown index")); - assertThat(e.getMessage(), containsString(Strings.format("%s:%s", REMOTE_CLUSTER_ALIAS, remoteExpr))); - e = expectThrows(ResponseException.class, () -> performRequestWithRemoteSearchUserViaAPIKey(limit0, remoteSearchUserAPIKey)); - assertThat(e.getMessage(), containsString("Unknown index")); - assertThat(e.getMessage(), containsString(Strings.format("%s:%s", REMOTE_CLUSTER_ALIAS, remoteExpr))); - } - - // missing concrete index on skip_unavailable=true cluster is not an error - { - String q = "FROM employees,my_remote_cluster:employees_nomatch,my_remote_cluster:employees*"; - - CheckedBiConsumer verifier = (response, limit0) -> { - assertOK(response); - Map map = responseAsMap(response); - assertThat(((List) map.get("columns")).size(), greaterThanOrEqualTo(1)); - if (limit0) { - assertThat(((List) map.get("values")).size(), equalTo(0)); - } else { - assertThat(((List) map.get("values")).size(), greaterThanOrEqualTo(1)); - } - final List expectedClusters = List.of( - new ExpectedCluster("(local)", "employees", "successful", limit0 ? 0 : null), - new ExpectedCluster(REMOTE_CLUSTER_ALIAS, "employees_nomatch,employees*", "successful", 0) - ); - assertExpectedClustersForMissingIndicesTests(map, expectedClusters); - }; - - // TODO: uncomment in follow on PR handling skip_unavailable errors at execution time - // Request limit1 = esqlRequest(q + " | LIMIT 1"); - // verifier.accept(performRequestWithRemoteSearchUser(limit1), false); - // verifier.accept(performRequestWithRemoteSearchUserViaAPIKey(limit1, remoteSearchUserAPIKey), false); - - Request limit0 = esqlRequest(q + " | LIMIT 0"); - verifier.accept(performRequestWithRemoteSearchUser(limit0), true); - verifier.accept(performRequestWithRemoteSearchUserViaAPIKey(limit0, remoteSearchUserAPIKey), true); - } - } - - @SuppressWarnings("unchecked") - public void testSearchesAgainstNonMatchingIndicesWithSkipUnavailableFalse() throws Exception { - configureRemoteCluster(REMOTE_CLUSTER_ALIAS, fulfillingCluster, false, randomBoolean(), false); + public void testSearchesAgainstNonMatchingIndices() throws Exception { + boolean skipUnavailable = randomBoolean(); + configureRemoteCluster(REMOTE_CLUSTER_ALIAS, fulfillingCluster, false, randomBoolean(), skipUnavailable); populateData(); { @@ -1351,23 +1116,6 @@ public void testSearchesAgainstNonMatchingIndicesWithSkipUnavailableFalse() thro verifier.accept(performRequestWithRemoteSearchUserViaAPIKey(limit0, remoteSearchUserAPIKey), true); } - // query is fatal since the remote cluster has skip_unavailable=false and has no matching indices - { - String q = "FROM employees,my_remote_cluster:employees_nomatch*"; - - Request limit1 = esqlRequest(q + " | LIMIT 1"); - ResponseException e = expectThrows(ResponseException.class, () -> performRequestWithRemoteSearchUser(limit1)); - assertThat(e.getMessage(), containsString("Unknown index [my_remote_cluster:employees_nomatch*]")); - e = expectThrows(ResponseException.class, () -> performRequestWithRemoteSearchUserViaAPIKey(limit1, remoteSearchUserAPIKey)); - assertThat(e.getMessage(), containsString("Unknown index [my_remote_cluster:employees_nomatch*]")); - - Request limit0 = esqlRequest(q + " | LIMIT 0"); - e = expectThrows(ResponseException.class, () -> performRequestWithRemoteSearchUser(limit0)); - assertThat(e.getMessage(), containsString("Unknown index [my_remote_cluster:employees_nomatch*]")); - e = expectThrows(ResponseException.class, () -> performRequestWithRemoteSearchUserViaAPIKey(limit0, remoteSearchUserAPIKey)); - assertThat(e.getMessage(), containsString("Unknown index [my_remote_cluster:employees_nomatch*]")); - } - // an error is thrown if there are no matching indices at all { // with non-matching concrete index @@ -1409,7 +1157,7 @@ public void testSearchesAgainstNonMatchingIndicesWithSkipUnavailableFalse() thro assertThat(e.getMessage(), containsString(Strings.format("%s:%s", REMOTE_CLUSTER_ALIAS, remoteExpr))); } - // error since the remote cluster with skip_unavailable=false specified a concrete index that is not found + // error since the remote cluster specified a concrete index that is not found { String q = "FROM employees,my_remote_cluster:employees_nomatch,my_remote_cluster:employees*";