From 32df62ae1554e7a98d65c2131bd0ca9179fc114a Mon Sep 17 00:00:00 2001 From: Przemyslaw Gomulka Date: Fri, 16 Nov 2018 08:58:13 +0100 Subject: [PATCH 1/4] Docs: HLRC: refactor bulk, migrate and reindex apis (#35413) fix typos and refactor to DRY up documentation for bulk, reindex and migration apis relates #35345 --- .../org/elasticsearch/client/ReindexIT.java | 12 ++++--- .../high-level/document/bulk.asciidoc | 25 ++++++++++++-- .../high-level/document/reindex.asciidoc | 17 ++++++++-- docs/java-rest/high-level/index.asciidoc | 1 + .../high-level/migration/upgrade.asciidoc | 34 +++++++++++-------- 5 files changed, 66 insertions(+), 23 deletions(-) diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ReindexIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ReindexIT.java index afc5e99b5f03a..f94cc41432c4c 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/ReindexIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ReindexIT.java @@ -106,15 +106,19 @@ public void testReindexTask() throws IOException, InterruptedException { ); } { - ReindexRequest reindexRequest = new ReindexRequest(); + // tag::submit-reindex-task + ReindexRequest reindexRequest = new ReindexRequest(); // <1> reindexRequest.setSourceIndices(sourceIndex); reindexRequest.setDestIndex(destinationIndex); - reindexRequest.setSourceQuery(new IdsQueryBuilder().addIds("1").types("type")); reindexRequest.setRefresh(true); - TaskSubmissionResponse reindexSubmission = highLevelClient().submitReindexTask(reindexRequest, RequestOptions.DEFAULT); + TaskSubmissionResponse reindexSubmission = highLevelClient() + .submitReindexTask(reindexRequest, RequestOptions.DEFAULT); // <2> + + String taskId = reindexSubmission.getTask(); // <3> + // end::submit-reindex-task - BooleanSupplier hasUpgradeCompleted = checkCompletionStatus(reindexSubmission.getTask()); + BooleanSupplier hasUpgradeCompleted = checkCompletionStatus(taskId); awaitBusy(hasUpgradeCompleted); } } diff --git a/docs/java-rest/high-level/document/bulk.asciidoc b/docs/java-rest/high-level/document/bulk.asciidoc index db9a3463135e8..c50a1f790583b 100644 --- a/docs/java-rest/high-level/document/bulk.asciidoc +++ b/docs/java-rest/high-level/document/bulk.asciidoc @@ -37,9 +37,9 @@ And different operation types can be added to the same +{request}+: -------------------------------------------------- include-tagged::{doc-tests-file}[{api}-request-with-mixed-operations] -------------------------------------------------- -<1> Adds a `DeleteRequest` to the `BulkRequest`. See <<{upid}-delete>> +<1> Adds a `DeleteRequest` to the +{request}+. See <<{upid}-delete>> for more information on how to build `DeleteRequest`. -<2> Adds an `UpdateRequest` to the `BulkRequest`. See <<{upid}-update>> +<2> Adds an `UpdateRequest` to the +{request}+. See <<{upid}-update>> for more information on how to build `UpdateRequest`. <3> Adds an `IndexRequest` using the SMILE format @@ -70,6 +70,25 @@ the index/update/delete operations. `ActiveShardCount.ALL`, `ActiveShardCount.ONE` or `ActiveShardCount.DEFAULT` (default) +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests-file}[{api}-request-pipeline] +-------------------------------------------------- +<1> Global pipelineId used on all sub requests, unless overridden on a sub request + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests-file}[{api}-request-routing] +-------------------------------------------------- +<1> Global routingId used on all sub requests, unless overridden on a sub request + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests-file}[{api}-request-index-type] +-------------------------------------------------- +<1> A bulk request with global index and type used on all sub requests, unless overridden on a sub request. +Both parameters are @Nullable and can only be set during +{request}+ creation. + include::../execution.asciidoc[] [id="{upid}-{api}-response"] @@ -148,7 +167,7 @@ actions currently added (defaults to 1000, use -1 to disable it) actions currently added (defaults to 5Mb, use -1 to disable it) <3> Set the number of concurrent requests allowed to be executed (default to 1, use 0 to only allow the execution of a single request) -<4> Set a flush interval flushing any `BulkRequest` pending if the +<4> Set a flush interval flushing any +{request}+ pending if the interval passes (defaults to not set) <5> Set a constant back off policy that initially waits for 1 second and retries up to 3 times. See `BackoffPolicy.noBackoff()`, diff --git a/docs/java-rest/high-level/document/reindex.asciidoc b/docs/java-rest/high-level/document/reindex.asciidoc index 2482467410c96..7d8876aa1269a 100644 --- a/docs/java-rest/high-level/document/reindex.asciidoc +++ b/docs/java-rest/high-level/document/reindex.asciidoc @@ -10,7 +10,7 @@ [id="{upid}-{api}-request"] ==== Reindex Request -A +{request} can be used to copy documents from one or more indexes into a +A +{request}+ can be used to copy documents from one or more indexes into a destination index. It requires an existing source index and a target index which may or may not exist pre-request. Reindex does not attempt @@ -100,7 +100,7 @@ include-tagged::{doc-tests-file}[{api}-request-sort] <1> add descending sort to`field1` <2> add ascending sort to `field2` -+{request} also supports a `script` that modifies the document. It allows you to ++{request}+ also supports a `script` that modifies the document. It allows you to also change the document's metadata. The following example illustrates that. ["source","java",subs="attributes,callouts,macros"] @@ -157,6 +157,19 @@ include-tagged::{doc-tests-file}[{api}-request-refresh] include::../execution.asciidoc[] +[id="{upid}-{api}-task-submission"] +==== Reindex task submission +It is also possible to submit a +{request}+ and not wait for it completion with the use of Task API. This is an equivalent of a REST request +with wait_for_completion flag set to false. + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{hlrc-tests}/ReindexIT.java[submit-reindex-task] +-------------------------------------------------- +<1> A +{request}+ is constructed the same way as for the synchronous method +<2> A submit method returns a `TaskSubmissionResponse` which contains a task identifier. +<3> The task identifier can be used to get `response` from a completed task. + [id="{upid}-{api}-response"] ==== Reindex Response diff --git a/docs/java-rest/high-level/index.asciidoc b/docs/java-rest/high-level/index.asciidoc index a15967e9ad717..2010c9c539a54 100644 --- a/docs/java-rest/high-level/index.asciidoc +++ b/docs/java-rest/high-level/index.asciidoc @@ -25,6 +25,7 @@ the same response objects. -- :doc-tests: {docdir}/../../client/rest-high-level/src/test/java/org/elasticsearch/client/documentation +:hlrc-tests: {docdir}/../../client/rest-high-level/src/test/java/org/elasticsearch/client include::getting-started.asciidoc[] include::supported-apis.asciidoc[] diff --git a/docs/java-rest/high-level/migration/upgrade.asciidoc b/docs/java-rest/high-level/migration/upgrade.asciidoc index 76eae0652d9bf..7497b74d38391 100644 --- a/docs/java-rest/high-level/migration/upgrade.asciidoc +++ b/docs/java-rest/high-level/migration/upgrade.asciidoc @@ -1,14 +1,22 @@ +-- +:api: upgrade +:request: IndexUpgradeRequest +:response: BulkByScrollResponse +:submit_response: IndexUpgradeSubmissionResponse +:doc-tests-file: {doc-tests}/MigrationClientDocumentationIT.java +-- + [[java-rest-high-migration-upgrade]] === Migration Upgrade [[java-rest-high-migraton-upgrade-request]] ==== Index Upgrade Request -An `IndexUpgradeRequest` requires an index argument. Only one index at the time should be upgraded: +An +{request}+ requires an index argument. Only one index at the time should be upgraded: ["source","java",subs="attributes,callouts,macros"] -------------------------------------------------- -include-tagged::{doc-tests}/MigrationClientDocumentationIT.java[upgrade-request] +include-tagged::{doc-tests-file}[{api}-request] -------------------------------------------------- <1> Create a new request instance @@ -17,39 +25,37 @@ include-tagged::{doc-tests}/MigrationClientDocumentationIT.java[upgrade-request] ["source","java",subs="attributes,callouts,macros"] -------------------------------------------------- -include-tagged::{doc-tests}/MigrationClientDocumentationIT.java[upgrade-execute] +include-tagged::{doc-tests-file}[{api}-execute] -------------------------------------------------- [[java-rest-high-migration-upgrade-response]] ==== Response -The returned `BulkByScrollResponse` contains information about the executed operation +The returned +{response}+ contains information about the executed operation [[java-rest-high-migraton-async-upgrade-request]] ==== Asynchronous Execution -The asynchronous execution of a upgrade request requires both the `IndexUpgradeRequest` +The asynchronous execution of an upgrade request requires both the +{request}+ instance and an `ActionListener` instance to be passed to the asynchronous method: -A typical listener for `BulkResponse` looks like: - ["source","java",subs="attributes,callouts,macros"] -------------------------------------------------- -include-tagged::{doc-tests}/MigrationClientDocumentationIT.java[upgrade-async-listener] +include-tagged::{doc-tests-file}[{api}-async-listener] -------------------------------------------------- <1> Called when the execution is successfully completed. The response is provided as an argument and contains a list of individual results for each operation that was executed. Note that one or more operations might have failed while the others have been successfully executed. -<2> Called when the whole `IndexUpgradeRequest` fails. In this case the raised +<2> Called when the whole +{request}+ fails. In this case the raised exception is provided as an argument and no operation has been executed. ["source","java",subs="attributes,callouts,macros"] -------------------------------------------------- -include-tagged::{doc-tests}/MigrationClientDocumentationIT.java[upgrade-async-execute] +include-tagged::{doc-tests-file}[{api}-async-execute] -------------------------------------------------- -<1> The `IndexUpgradeRequest` to execute and the `ActionListener` to use when +<1> The +{request}+ to execute and the `ActionListener` to use when the execution completes The asynchronous method does not block and returns immediately. Once it is @@ -59,11 +65,11 @@ it failed. === Migration Upgrade with Task API -Submission of upgrade request task will requires the `IndexUpgradeRequest` and will return -`IndexUpgradeSubmissionResponse`. The `IndexUpgradeSubmissionResponse` can later be use to fetch +Submission of upgrade request task will requires the +{request}+ and will return ++{submit_response}+. The +{submit_response}+ can later be use to fetch TaskId and query the Task API for results. ["source","java",subs="attributes,callouts,macros"] -------------------------------------------------- -include-tagged::{doc-tests}/MigrationClientDocumentationIT.java[upgrade-task-api] +include-tagged::{doc-tests-file}[{api}-task-api] -------------------------------------------------- From 9f6ea40f49502ad88bb8e5e8cfb4142e170d36db Mon Sep 17 00:00:00 2001 From: Przemyslaw Gomulka Date: Fri, 2 Nov 2018 15:01:43 +0100 Subject: [PATCH 2/4] ensure watcher started --- .../elasticsearch/upgrades/WatchBackwardsCompatibilityIT.java | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/WatchBackwardsCompatibilityIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/WatchBackwardsCompatibilityIT.java index be3b0525d06bf..2330f659230d5 100644 --- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/WatchBackwardsCompatibilityIT.java +++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/WatchBackwardsCompatibilityIT.java @@ -179,6 +179,7 @@ public void testWatcherStats() throws Exception { public void testWatcherRestart() throws Exception { executeUpgradeIfNeeded(); + ensureWatcherStarted(); executeAgainstRandomNode(client -> assertOK(client.performRequest("POST", "/_xpack/watcher/_stop"))); ensureWatcherStopped(); From 3693d1d11ca295fae5115381a9f67f99d7e5102b Mon Sep 17 00:00:00 2001 From: Przemyslaw Gomulka Date: Mon, 17 Dec 2018 14:44:05 +0100 Subject: [PATCH 3/4] Muting failing tests awaiting the fix only muting two failing test cases in a subclass where they fail. issue referencing the problem: #36598 --- .../index/query/MultiMatchQueryBuilderTests.java | 11 +++++++++++ .../org/elasticsearch/test/AbstractQueryTestCase.java | 2 -- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/index/query/MultiMatchQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/MultiMatchQueryBuilderTests.java index 4dce3fd53b71b..e6c1bade759e8 100644 --- a/server/src/test/java/org/elasticsearch/index/query/MultiMatchQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/MultiMatchQueryBuilderTests.java @@ -492,4 +492,15 @@ private static IndexMetaData newIndexMeta(String name, Settings oldIndexSettings .build(); return IndexMetaData.builder(name).settings(build).build(); } + + + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/36598") + @Override + public void testMustRewrite() throws IOException { + } + + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/36598") + @Override + public void testToQuery() throws IOException { + } } diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java index 16f52473f4254..7647a9e6d6546 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractQueryTestCase.java @@ -413,7 +413,6 @@ protected boolean builderGeneratesCacheableQueries() { * Test creates the {@link Query} from the {@link QueryBuilder} under test and delegates the * assertions being made on the result to the implementing subclass. */ - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/36598") public void testToQuery() throws IOException { for (int runs = 0; runs < NUMBER_OF_TESTQUERIES; runs++) { QueryShardContext context = createShardContext(); @@ -798,7 +797,6 @@ private static String msg(String left, String right) { * This test ensures that queries that need to be rewritten have dedicated tests. * These queries must override this method accordingly. */ - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/36598") public void testMustRewrite() throws IOException { QueryShardContext context = createShardContext(); context.setAllowUnmappedFields(true); From 49d2a2c4d37b5c2560fdd3110990418301de5711 Mon Sep 17 00:00:00 2001 From: Przemyslaw Gomulka Date: Mon, 7 Jan 2019 14:37:11 +0100 Subject: [PATCH 4/4] verify test after merge --- .../elasticsearch/upgrades/WatchBackwardsCompatibilityIT.java | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/WatchBackwardsCompatibilityIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/WatchBackwardsCompatibilityIT.java index aa92b273f68a0..2330f659230d5 100644 --- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/WatchBackwardsCompatibilityIT.java +++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/WatchBackwardsCompatibilityIT.java @@ -177,7 +177,6 @@ public void testWatcherStats() throws Exception { ); } - @AwaitsFix(bugUrl="https://github.com/elastic/elasticsearch/issues/33753") public void testWatcherRestart() throws Exception { executeUpgradeIfNeeded(); ensureWatcherStarted();