From 69f60866fb538c2dc96dcfd945d168dafa78fb52 Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Thu, 31 Jan 2019 15:22:10 +0100 Subject: [PATCH 01/19] wip --- docs/reference/docs/index_.asciidoc | 2 ++ docs/reference/docs/update.asciidoc | 2 ++ docs/reference/migration/migrate_6_7.asciidoc | 13 +++++++++++++ .../org/elasticsearch/action/DocWriteRequest.java | 14 ++++++++++++++ .../elasticsearch/action/delete/DeleteRequest.java | 5 +++++ .../elasticsearch/action/index/IndexRequest.java | 6 ++++++ .../elasticsearch/action/update/UpdateRequest.java | 3 +++ .../protocol/xpack/watcher/PutWatchRequest.java | 11 +++++++++++ 8 files changed, 56 insertions(+) diff --git a/docs/reference/docs/index_.asciidoc b/docs/reference/docs/index_.asciidoc index 678e9c69c6ec1..36028aa249256 100644 --- a/docs/reference/docs/index_.asciidoc +++ b/docs/reference/docs/index_.asciidoc @@ -369,6 +369,8 @@ the different version types and their semantics. `internal`:: only index the document if the given version is identical to the version of the stored document. +deprecated[6.7.0, Please use `if_seq_no` & `if_primary_term` instead. See <> for more details.] + `external` or `external_gt`:: only index the document if the given version is strictly higher than the version of the stored document *or* if there is no existing document. The given diff --git a/docs/reference/docs/update.asciidoc b/docs/reference/docs/update.asciidoc index 1cfc122bee402..b93c19f21195c 100644 --- a/docs/reference/docs/update.asciidoc +++ b/docs/reference/docs/update.asciidoc @@ -338,6 +338,8 @@ The update API uses the Elasticsearch's versioning support internally to make sure the document doesn't change during the update. You can use the `version` parameter to specify that the document should only be updated if its version matches the one specified. +deprecated[6.7.0, Please use `if_seq_no` & `if_primary_term` instead. See <> for more details.] + [NOTE] .The update API does not support versioning other than internal diff --git a/docs/reference/migration/migrate_6_7.asciidoc b/docs/reference/migration/migrate_6_7.asciidoc index 194c1d6f20970..986e0885ffad1 100644 --- a/docs/reference/migration/migrate_6_7.asciidoc +++ b/docs/reference/migration/migrate_6_7.asciidoc @@ -7,11 +7,24 @@ This section discusses the changes that you need to be aware of when migrating your application to Elasticsearch 6.7. +* <> * <> * <> See also <> and <>. +[float] +[[breaking_67_indexing_changes]] +=== Indexing changes + +==== Usage of `internal` versioning for optimistic concurrency control + +`internal` version may not uniquely identify a document's version if an indexed document +wasn't fully replicated when a primary fails. As such it is unsafe to use for +optimistic concurrency control, is deprecated and the option will no longer be available +in Elasticsearch 7.0.0. Please use the `if_seq_no` and `if_primary_term` parameters instead. +See <> for more details. + [float] [[breaking_67_plugin_changes]] === Plugin changes diff --git a/server/src/main/java/org/elasticsearch/action/DocWriteRequest.java b/server/src/main/java/org/elasticsearch/action/DocWriteRequest.java index f98a6883b46d4..1a829c852a944 100644 --- a/server/src/main/java/org/elasticsearch/action/DocWriteRequest.java +++ b/server/src/main/java/org/elasticsearch/action/DocWriteRequest.java @@ -24,6 +24,8 @@ import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.logging.DeprecationLogger; +import org.elasticsearch.common.lucene.uid.Versions; import org.elasticsearch.index.VersionType; import java.io.IOException; @@ -213,4 +215,16 @@ static void writeDocumentRequest(StreamOutput out, DocWriteRequest request) thr throw new IllegalStateException("invalid request [" + request.getClass().getSimpleName() + " ]"); } } + + static void logDeprecationWarnings(DocWriteRequest request, DeprecationLogger logger) { + if (request.versionType() == VersionType.INTERNAL && + request.version() != Versions.MATCH_ANY && + request.version() != Versions.MATCH_DELETED) { + logger.deprecatedAndMaybeLog("occ_internal_version", + "Usage of internal versioning for optimistic concurrency control is deprecated and will be removed. Please use" + + " the `if_seq_no` and `if_primary_term` parameters instead. (request for index [{}], type [{}], id [{}])", + request.index(), request.type(), request.id()); + } + + } } diff --git a/server/src/main/java/org/elasticsearch/action/delete/DeleteRequest.java b/server/src/main/java/org/elasticsearch/action/delete/DeleteRequest.java index c4e6014dfd260..d900d1fbfb8cf 100644 --- a/server/src/main/java/org/elasticsearch/action/delete/DeleteRequest.java +++ b/server/src/main/java/org/elasticsearch/action/delete/DeleteRequest.java @@ -19,6 +19,7 @@ package org.elasticsearch.action.delete; +import org.apache.logging.log4j.LogManager; import org.elasticsearch.Version; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.CompositeIndicesRequest; @@ -28,6 +29,7 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.lucene.uid.Versions; import org.elasticsearch.index.VersionType; import org.elasticsearch.index.shard.ShardId; @@ -51,6 +53,7 @@ */ public class DeleteRequest extends ReplicatedWriteRequest implements DocWriteRequest, CompositeIndicesRequest { + private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(LogManager.getLogger(DeleteRequest.class)); private String type; private String id; @@ -118,6 +121,8 @@ public ActionRequestValidationException validate() { addValidationError("ifSeqNo is unassigned, but primary term is [" + ifPrimaryTerm + "]", validationException); } + DocWriteRequest.logDeprecationWarnings(this, DEPRECATION_LOGGER); + return validationException; } diff --git a/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java b/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java index d17756aa2ed1d..385ab8b65a300 100644 --- a/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java +++ b/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java @@ -19,6 +19,7 @@ package org.elasticsearch.action.index; +import org.apache.logging.log4j.LogManager; import org.elasticsearch.ElasticsearchGenerationException; import org.elasticsearch.Version; import org.elasticsearch.action.ActionRequestValidationException; @@ -36,6 +37,7 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.lucene.uid.Versions; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -73,6 +75,7 @@ * @see org.elasticsearch.client.Client#index(IndexRequest) */ public class IndexRequest extends ReplicatedWriteRequest implements DocWriteRequest, CompositeIndicesRequest { + private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(LogManager.getLogger(IndexRequest.class)); /** * Max length of the source document to include into string() @@ -217,6 +220,9 @@ public ActionRequestValidationException validate() { addValidationError("ifSeqNo is unassigned, but primary term is [" + ifPrimaryTerm + "]", validationException); } + DocWriteRequest.logDeprecationWarnings(this, DEPRECATION_LOGGER); + + return validationException; } diff --git a/server/src/main/java/org/elasticsearch/action/update/UpdateRequest.java b/server/src/main/java/org/elasticsearch/action/update/UpdateRequest.java index c95ab817430ab..7b8639b1f7d4f 100644 --- a/server/src/main/java/org/elasticsearch/action/update/UpdateRequest.java +++ b/server/src/main/java/org/elasticsearch/action/update/UpdateRequest.java @@ -144,6 +144,9 @@ public ActionRequestValidationException validate() { if (doc == null && docAsUpsert) { validationException = addValidationError("doc must be specified if doc_as_upsert is enabled", validationException); } + + DocWriteRequest.logDeprecationWarnings(this, DEPRECATION_LOGGER); + return validationException; } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/protocol/xpack/watcher/PutWatchRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/protocol/xpack/watcher/PutWatchRequest.java index abc42b149194b..e6c4627916bc8 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/protocol/xpack/watcher/PutWatchRequest.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/protocol/xpack/watcher/PutWatchRequest.java @@ -5,6 +5,7 @@ */ package org.elasticsearch.protocol.xpack.watcher; +import org.apache.logging.log4j.LogManager; import org.elasticsearch.Version; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.ValidateActions; @@ -13,6 +14,7 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.lucene.uid.Versions; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentHelper; @@ -27,6 +29,8 @@ */ public class PutWatchRequest extends MasterNodeRequest { + private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(LogManager.getLogger(PutWatchRequest.class)); + private static final TimeValue DEFAULT_TIMEOUT = TimeValue.timeValueSeconds(10); private static final Pattern NO_WS_PATTERN = Pattern.compile("\\S+"); @@ -121,6 +125,13 @@ public ActionRequestValidationException validate() { if (xContentType == null) { validationException = ValidateActions.addValidationError("request body is missing", validationException); } + + if (version != Versions.MATCH_ANY) { + DEPRECATION_LOGGER.deprecated( + "Usage of internal versioning for optimistic concurrency control is deprecated and will be removed. Please use" + + " the `if_seq_no` and `if_primary_term` parameters instead."); + } + return validationException; } From 8678df51b067c66a37e8939ff8f9cd561e93d58d Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Tue, 5 Feb 2019 09:48:57 +0100 Subject: [PATCH 02/19] add delete cas rest test --- .../rest-api-spec/test/delete/21_cas.yml | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/delete/21_cas.yml diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/delete/21_cas.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/delete/21_cas.yml new file mode 100644 index 0000000000000..dc28867b7cefd --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/delete/21_cas.yml @@ -0,0 +1,33 @@ +--- +"Compare and set with sequence numbers": + - skip: + version: " - 6.5.99" + reason: sequence numbers can be used for cas as of 6.6.0 + + - do: + index: + index: test_1 + type: test + id: 1 + body: { foo: bar } + + - match: { _seq_no: 0 } + + - do: + catch: conflict + delete: + index: test_1 + type: test + id: 1 + if_seq_no: 2 + if_primary_term: 1 + + - do: + delete: + index: test_1 + type: test + id: 1 + if_seq_no: 0 + if_primary_term: 1 + + - match: { _seq_no: 1 } \ No newline at end of file From 69bfe53fd577983ad98c7d1b85ed832a6e28047f Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Tue, 5 Feb 2019 10:04:34 +0100 Subject: [PATCH 03/19] add some deprecation warnings --- .../rest-api-spec/test/bulk/60_deprecated.yml | 2 ++ .../rest-api-spec/test/delete/20_internal_version.yml | 6 ++++++ .../resources/rest-api-spec/test/delete/21_cas.yml | 6 +++--- .../rest-api-spec/test/index/30_internal_version.yml | 10 ++++++++++ .../rest-api-spec/test/update/30_internal_version.yml | 8 +++++++- .../elasticsearch/action/bulk/BulkRequestTests.java | 2 ++ .../action/update/UpdateRequestTests.java | 2 ++ 7 files changed, 32 insertions(+), 4 deletions(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/bulk/60_deprecated.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/bulk/60_deprecated.yml index 17418893d6892..5fa95968c4ffb 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/bulk/60_deprecated.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/bulk/60_deprecated.yml @@ -16,6 +16,8 @@ { "doc": { "f1": "v2" } } warnings: - "Deprecated field [_version] used, expected [version] instead" + - "Usage of internal versioning for optimistic concurrency control is deprecated and will be removed. Please use the `if_seq_no` and `if_primary_term` parameters instead. (request for index [test_index], type [test_type], id [test_id_1]" + - "Usage of internal versioning for optimistic concurrency control is deprecated and will be removed. Please use the `if_seq_no` and `if_primary_term` parameters instead. (request for index [test_index], type [test_type], id [test_id_2]" - do: bulk: diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/delete/20_internal_version.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/delete/20_internal_version.yml index 3d9ddb79366f7..07b711aa74201 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/delete/20_internal_version.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/delete/20_internal_version.yml @@ -1,5 +1,7 @@ --- "Internal version": + - skip: + features: warnings - do: index: @@ -17,6 +19,8 @@ type: test id: 1 version: 2 + warnings: + - "Usage of internal versioning for optimistic concurrency control is deprecated and will be removed. Please use the `if_seq_no` and `if_primary_term` parameters instead. (request for index [test_1], type [test], id [1]" - do: delete: @@ -24,5 +28,7 @@ type: test id: 1 version: 1 + warnings: + - "Usage of internal versioning for optimistic concurrency control is deprecated and will be removed. Please use the `if_seq_no` and `if_primary_term` parameters instead. (request for index [test_1], type [test], id [1]" - match: { _version: 2 } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/delete/21_cas.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/delete/21_cas.yml index dc28867b7cefd..4269723b442e7 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/delete/21_cas.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/delete/21_cas.yml @@ -1,8 +1,8 @@ --- "Compare and set with sequence numbers": - skip: - version: " - 6.5.99" - reason: sequence numbers can be used for cas as of 6.6.0 + version: " - 6.5.99" + reason: sequence numbers can be used for cas as of 6.6.0 - do: index: @@ -30,4 +30,4 @@ if_seq_no: 0 if_primary_term: 1 - - match: { _seq_no: 1 } \ No newline at end of file + - match: { _seq_no: 1 } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/index/30_internal_version.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/index/30_internal_version.yml index 1767fbebbf966..c43c20cc2f4a9 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/index/30_internal_version.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/index/30_internal_version.yml @@ -1,5 +1,7 @@ --- "Internal version": + - skip: + features: warnings - do: index: @@ -15,6 +17,9 @@ type: test id: 1 body: { foo: bar } + warnings: + - "Usage of internal versioning for optimistic concurrency control is deprecated and will be removed. Please use the `if_seq_no` and `if_primary_term` parameters instead. (request for index [test_1], type [test], id [1]" + - match: { _version: 2} - do: @@ -25,6 +30,9 @@ id: 1 body: { foo: bar } version: 1 + warnings: + - "Usage of internal versioning for optimistic concurrency control is deprecated and will be removed. Please use the `if_seq_no` and `if_primary_term` parameters instead. (request for index [test_1], type [test], id [1]" + - do: index: index: test_1 @@ -32,5 +40,7 @@ id: 1 body: { foo: bar } version: 2 + warnings: + - "Usage of internal versioning for optimistic concurrency control is deprecated and will be removed. Please use the `if_seq_no` and `if_primary_term` parameters instead. (request for index [test_1], type [test], id [1]" - match: { _version: 3 } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/update/30_internal_version.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/update/30_internal_version.yml index 17c4806c693ac..d09fa72c94098 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/update/30_internal_version.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/update/30_internal_version.yml @@ -1,7 +1,9 @@ --- "Internal version": + - skip: + features: warnings - - do: + - do: catch: missing update: index: test_1 @@ -10,6 +12,8 @@ version: 1 body: doc: { foo: baz } + warnings: + - "Usage of internal versioning for optimistic concurrency control is deprecated and will be removed. Please use the `if_seq_no` and `if_primary_term` parameters instead. (request for index [test_1], type [test], id [1]" - do: index: @@ -28,3 +32,5 @@ version: 2 body: doc: { foo: baz } + warnings: + - "Usage of internal versioning for optimistic concurrency control is deprecated and will be removed. Please use the `if_seq_no` and `if_primary_term` parameters instead. (request for index [test_1], type [test], id [1]" diff --git a/server/src/test/java/org/elasticsearch/action/bulk/BulkRequestTests.java b/server/src/test/java/org/elasticsearch/action/bulk/BulkRequestTests.java index dc1532b8301eb..2bb64d62e739c 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/BulkRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/BulkRequestTests.java @@ -316,6 +316,8 @@ public void testToValidateUpsertRequestAndVersionInBulkRequest() throws IOExcept bulkRequest.add(data, null, null, xContentType); assertThat(bulkRequest.validate().validationErrors(), contains("can't provide both upsert request and a version", "can't provide version in upsert request")); + assertWarnings("Usage of internal versioning for optimistic concurrency control is deprecated and will be removed. " + + "Please use the `if_seq_no` and `if_primary_term` parameters instead. (request for index [index], type [type], id [id])"); } public void testBulkTerminatedByNewline() throws Exception { diff --git a/server/src/test/java/org/elasticsearch/action/update/UpdateRequestTests.java b/server/src/test/java/org/elasticsearch/action/update/UpdateRequestTests.java index b7a1084683da6..133e20ae824c6 100644 --- a/server/src/test/java/org/elasticsearch/action/update/UpdateRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/update/UpdateRequestTests.java @@ -557,6 +557,8 @@ public void testToValidateUpsertRequestAndVersion() { updateRequest.doc("{}", XContentType.JSON); updateRequest.upsert(new IndexRequest("index","type", "id")); assertThat(updateRequest.validate().validationErrors(), contains("can't provide both upsert request and a version")); + assertWarnings("Usage of internal versioning for optimistic concurrency control is deprecated and will be removed. " + + "Please use the `if_seq_no` and `if_primary_term` parameters instead. (request for index [index], type [type], id [id])"); } public void testToValidateUpsertRequestWithVersion() { From cd4c85e0b3f5810dc418320cc43423c82fc64b5f Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Tue, 5 Feb 2019 12:08:53 +0100 Subject: [PATCH 04/19] Wire if_seq_no and if_primary_term in rest client bulk --- .../client/RequestConverters.java | 5 +++ .../java/org/elasticsearch/client/CrudIT.java | 23 +++++++--- .../client/RequestConvertersTests.java | 19 +++++--- .../rest-api-spec/test/bulk/80_cas.yml | 45 +++++++++++++++++++ 4 files changed, 80 insertions(+), 12 deletions(-) create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/bulk/80_cas.yml diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java index 3ad1ecfc106f8..1ba3d22f70d0f 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java @@ -192,6 +192,11 @@ static Request bulk(BulkRequest bulkRequest) throws IOException { } } + if (action.ifSeqNo() != SequenceNumbers.UNASSIGNED_SEQ_NO) { + metadata.field("if_seq_no", action.ifSeqNo()); + metadata.field("if_primary_term", action.ifPrimaryTerm()); + } + if (opType == DocWriteRequest.OpType.INDEX || opType == DocWriteRequest.OpType.CREATE) { IndexRequest indexRequest = (IndexRequest) action; if (Strings.hasLength(indexRequest.getPipeline())) { diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/CrudIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/CrudIT.java index e88ce713c686b..68385efee9a0b 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/CrudIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/CrudIT.java @@ -98,11 +98,14 @@ public void testDelete() throws IOException { { // Testing deletion String docId = "id"; - highLevelClient().index( + IndexResponse indexResponse = highLevelClient().index( new IndexRequest("index", "type", docId).source(Collections.singletonMap("foo", "bar")), RequestOptions.DEFAULT); DeleteRequest deleteRequest = new DeleteRequest("index", "type", docId); if (randomBoolean()) { - deleteRequest.version(1L); + deleteRequest.setIfSeqNo(indexResponse.getSeqNo()); + deleteRequest.setIfPrimaryTerm(indexResponse.getPrimaryTerm()); + } else { + deleteRequest.version(indexResponse.getVersion()); } DeleteResponse deleteResponse = execute(deleteRequest, highLevelClient()::delete, highLevelClient()::deleteAsync, highLevelClient()::delete, highLevelClient()::deleteAsync); @@ -127,7 +130,13 @@ public void testDelete() throws IOException { String docId = "version_conflict"; highLevelClient().index( new IndexRequest("index", "type", docId).source(Collections.singletonMap("foo", "bar")), RequestOptions.DEFAULT); - DeleteRequest deleteRequest = new DeleteRequest("index", "type", docId).version(2); + DeleteRequest deleteRequest = new DeleteRequest("index", "type", docId); + if (randomBoolean()) { + deleteRequest.version(2); + } else { + deleteRequest.setIfSeqNo(2).setIfPrimaryTerm(2); + } + ElasticsearchException exception = expectThrows(ElasticsearchException.class, () -> execute(deleteRequest, highLevelClient()::delete, highLevelClient()::deleteAsync, highLevelClient()::delete, highLevelClient()::deleteAsync)); @@ -457,7 +466,7 @@ public void testIndex() throws IOException { ElasticsearchStatusException exception = expectThrows(ElasticsearchStatusException.class, () -> { IndexRequest wrongRequest = new IndexRequest("index", "type", "id"); wrongRequest.source(XContentBuilder.builder(xContentType.xContent()).startObject().field("field", "test").endObject()); - wrongRequest.version(5L); + wrongRequest.setIfSeqNo(1L).setIfPrimaryTerm(5L); execute(wrongRequest, highLevelClient()::index, highLevelClient()::indexAsync, highLevelClient()::index, highLevelClient()::indexAsync); @@ -763,7 +772,8 @@ public void testBulk() throws IOException { if (opType == DocWriteRequest.OpType.INDEX) { IndexRequest indexRequest = new IndexRequest("index", "test", id).source(source, xContentType); if (erroneous) { - indexRequest.version(12L); + indexRequest.setIfSeqNo(12L); + indexRequest.setIfPrimaryTerm(12L); } bulkRequest.add(indexRequest); @@ -1075,7 +1085,8 @@ public void afterBulk(long executionId, BulkRequest request, Throwable failure) if (opType == DocWriteRequest.OpType.INDEX) { IndexRequest indexRequest = new IndexRequest("index", "test", id).source(xContentType, "id", i); if (erroneous) { - indexRequest.version(12L); + indexRequest.setIfSeqNo(12L); + indexRequest.setIfPrimaryTerm(12L); } processor.add(indexRequest); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java index 78a16cbc1517c..284702eab0e21 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java @@ -53,11 +53,11 @@ import org.elasticsearch.action.support.master.MasterNodeReadRequest; import org.elasticsearch.action.support.master.MasterNodeRequest; import org.elasticsearch.action.support.replication.ReplicationRequest; -import org.elasticsearch.client.core.MultiTermVectorsRequest; -import org.elasticsearch.client.core.TermVectorsRequest; import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.client.RequestConverters.EndpointBuilder; import org.elasticsearch.client.core.CountRequest; +import org.elasticsearch.client.core.MultiTermVectorsRequest; +import org.elasticsearch.client.core.TermVectorsRequest; import org.elasticsearch.common.CheckedBiConsumer; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesArray; @@ -767,10 +767,15 @@ public void testBulk() throws IOException { docWriteRequest.routing(randomAlphaOfLength(10)); } if (randomBoolean()) { - docWriteRequest.version(randomNonNegativeLong()); - } - if (randomBoolean()) { - docWriteRequest.versionType(randomFrom(VersionType.values())); + if (randomBoolean()) { + docWriteRequest.version(randomNonNegativeLong()); + } + if (randomBoolean()) { + docWriteRequest.versionType(randomFrom(VersionType.values())); + } + } else if (randomBoolean()) { + docWriteRequest.setIfSeqNo(randomNonNegativeLong()); + docWriteRequest.setIfPrimaryTerm(randomLongBetween(1, 200)); } bulkRequest.add(docWriteRequest); } @@ -801,6 +806,8 @@ public void testBulk() throws IOException { assertEquals(originalRequest.parent(), parsedRequest.parent()); assertEquals(originalRequest.version(), parsedRequest.version()); assertEquals(originalRequest.versionType(), parsedRequest.versionType()); + assertEquals(originalRequest.ifSeqNo(), parsedRequest.ifSeqNo()); + assertEquals(originalRequest.ifPrimaryTerm(), parsedRequest.ifPrimaryTerm()); DocWriteRequest.OpType opType = originalRequest.opType(); if (opType == DocWriteRequest.OpType.INDEX) { diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/bulk/80_cas.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/bulk/80_cas.yml new file mode 100644 index 0000000000000..14237a1b9dc57 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/bulk/80_cas.yml @@ -0,0 +1,45 @@ +--- +"Compare And Swap Sequence Numbers": + + - skip: + version: " - 6.5.99" + reason: cas operations with sequence numbers was added in 6.6 + + - do: + index: + index: test_1 + type: _doc + id: 1 + body: { foo: bar } + - match: { _version: 1} + - set: { _seq_no: seqno } + - set: { _primary_term: primary_term } + + - do: + bulk: + body: + - index: + _index: test_1 + _type: _doc + _id: 1 + if_seq_no: 10000 + if_primary_term: $primary_term + - foo: bar2 + + - match: { errors: true } + - match: { items.0.index.status: 409 } + - match: { items.0.index.error.type: version_conflict_engine_exception } + + - do: + bulk: + body: + - index: + _index: test_1 + _type: _doc + _id: 1 + if_seq_no: $seqno + if_primary_term: $primary_term + - foo: bar2 + + - match: { errors: false} + - match: { items.0.index.status: 200 } From 898d509a1a6efc967107979bc86928d204dcffec Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Tue, 5 Feb 2019 13:24:18 +0100 Subject: [PATCH 05/19] wire index and delete --- .../client/RequestConverters.java | 4 ++ .../java/org/elasticsearch/client/CrudIT.java | 38 +++++++++++++++---- .../client/RequestConvertersTests.java | 20 ++++++++++ 3 files changed, 54 insertions(+), 8 deletions(-) diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java index 1ba3d22f70d0f..4e3628a181035 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java @@ -108,6 +108,8 @@ static Request delete(DeleteRequest deleteRequest) { parameters.withTimeout(deleteRequest.timeout()); parameters.withVersion(deleteRequest.version()); parameters.withVersionType(deleteRequest.versionType()); + parameters.withIfSeqNo(deleteRequest.ifSeqNo()); + parameters.withIfPrimaryTerm(deleteRequest.ifPrimaryTerm()); parameters.withRefreshPolicy(deleteRequest.getRefreshPolicy()); parameters.withWaitForActiveShards(deleteRequest.waitForActiveShards(), ActiveShardCount.DEFAULT); return request; @@ -313,6 +315,8 @@ static Request index(IndexRequest indexRequest) { parameters.withTimeout(indexRequest.timeout()); parameters.withVersion(indexRequest.version()); parameters.withVersionType(indexRequest.versionType()); + parameters.withIfSeqNo(indexRequest.ifSeqNo()); + parameters.withIfPrimaryTerm(indexRequest.ifPrimaryTerm()); parameters.withPipeline(indexRequest.getPipeline()); parameters.withRefreshPolicy(indexRequest.getRefreshPolicy()); parameters.withWaitForActiveShards(indexRequest.waitForActiveShards(), ActiveShardCount.DEFAULT); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/CrudIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/CrudIT.java index 68385efee9a0b..27e6221f0ec84 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/CrudIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/CrudIT.java @@ -48,6 +48,7 @@ import org.elasticsearch.client.core.MultiTermVectorsResponse; import org.elasticsearch.client.core.TermVectorsRequest; import org.elasticsearch.client.core.TermVectorsResponse; +import org.elasticsearch.client.indices.CreateIndexRequest; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; @@ -95,6 +96,9 @@ public class CrudIT extends ESRestHighLevelClientTestCase { public void testDelete() throws IOException { + highLevelClient().indices().create( + new CreateIndexRequest("index").settings(Collections.singletonMap("index.number_of_shards", "1")), + RequestOptions.DEFAULT); { // Testing deletion String docId = "id"; @@ -131,18 +135,25 @@ public void testDelete() throws IOException { highLevelClient().index( new IndexRequest("index", "type", docId).source(Collections.singletonMap("foo", "bar")), RequestOptions.DEFAULT); DeleteRequest deleteRequest = new DeleteRequest("index", "type", docId); - if (randomBoolean()) { - deleteRequest.version(2); - } else { + final boolean seqNos = randomBoolean(); + if (seqNos) { deleteRequest.setIfSeqNo(2).setIfPrimaryTerm(2); + } else { + deleteRequest.version(2); } ElasticsearchException exception = expectThrows(ElasticsearchException.class, () -> execute(deleteRequest, highLevelClient()::delete, highLevelClient()::deleteAsync, highLevelClient()::delete, highLevelClient()::deleteAsync)); assertEquals(RestStatus.CONFLICT, exception.status()); - assertEquals("Elasticsearch exception [type=version_conflict_engine_exception, reason=[type][" + docId + "]: " + - "version conflict, current version [1] is different than the one provided [2]]", exception.getMessage()); + if (seqNos) { + assertEquals("Elasticsearch exception [type=version_conflict_engine_exception, reason=[type][" + docId + "]: " + + "version conflict, required seqNo [2], primary term [2]. current document has seqNo [3] and primary term [1]]", + exception.getMessage()); + } else { + assertEquals("Elasticsearch exception [type=version_conflict_engine_exception, reason=[type][" + docId + "]: " + + "version conflict, current version [1] is different than the one provided [2]]", exception.getMessage()); + } assertEquals("index", exception.getMetadata("es.index").get(0)); } { @@ -462,18 +473,29 @@ public void testIndex() throws IOException { assertEquals("type", indexResponse.getType()); assertEquals("id", indexResponse.getId()); assertEquals(2L, indexResponse.getVersion()); + final boolean seqNosForConflict = randomBoolean(); ElasticsearchStatusException exception = expectThrows(ElasticsearchStatusException.class, () -> { IndexRequest wrongRequest = new IndexRequest("index", "type", "id"); wrongRequest.source(XContentBuilder.builder(xContentType.xContent()).startObject().field("field", "test").endObject()); - wrongRequest.setIfSeqNo(1L).setIfPrimaryTerm(5L); + if (seqNosForConflict) { + wrongRequest.setIfSeqNo(2).setIfPrimaryTerm(2); + } else { + wrongRequest.version(5); + } execute(wrongRequest, highLevelClient()::index, highLevelClient()::indexAsync, highLevelClient()::index, highLevelClient()::indexAsync); }); assertEquals(RestStatus.CONFLICT, exception.status()); - assertEquals("Elasticsearch exception [type=version_conflict_engine_exception, reason=[type][id]: " + - "version conflict, current version [2] is different than the one provided [5]]", exception.getMessage()); + if (seqNosForConflict) { + assertEquals("Elasticsearch exception [type=version_conflict_engine_exception, reason=[type][id]: " + + "version conflict, required seqNo [1], primary term [5]. current document has seqNo [2] and primary term [1]]", + exception.getMessage()); + } else { + assertEquals("Elasticsearch exception [type=version_conflict_engine_exception, reason=[type][id]: " + + "version conflict, current version [2] is different than the one provided [5]]", exception.getMessage()); + } assertEquals("index", exception.getMetadata("es.index").get(0)); } { diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java index 284702eab0e21..ca4821003ef06 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java @@ -216,6 +216,7 @@ public void testDelete() { setRandomRefreshPolicy(deleteRequest::setRefreshPolicy, expectedParams); setRandomVersion(deleteRequest, expectedParams); setRandomVersionType(deleteRequest::versionType, expectedParams); + setRandomIfSeqNoAndTerm(deleteRequest, expectedParams); if (frequently()) { if (randomBoolean()) { @@ -545,6 +546,7 @@ public void testIndex() throws IOException { } else { setRandomVersion(indexRequest, expectedParams); setRandomVersionType(indexRequest::versionType, expectedParams); + setRandomIfSeqNoAndTerm(indexRequest, expectedParams); } if (frequently()) { @@ -650,6 +652,7 @@ public void testUpdate() throws IOException { setRandomWaitForActiveShards(updateRequest::waitForActiveShards, ActiveShardCount.DEFAULT, expectedParams); setRandomVersion(updateRequest, expectedParams); setRandomVersionType(updateRequest::versionType, expectedParams); + setRandomIfSeqNoAndTerm(updateRequest, expectedParams); if (randomBoolean()) { int retryOnConflict = randomIntBetween(0, 5); updateRequest.retryOnConflict(retryOnConflict); @@ -680,6 +683,7 @@ public void testUpdate() throws IOException { assertEquals(updateRequest.docAsUpsert(), parsedUpdateRequest.docAsUpsert()); assertEquals(updateRequest.detectNoop(), parsedUpdateRequest.detectNoop()); assertEquals(updateRequest.fetchSource(), parsedUpdateRequest.fetchSource()); + assertIfSeqNoAndTerm(updateRequest, parsedUpdateRequest); assertEquals(updateRequest.script(), parsedUpdateRequest.script()); if (updateRequest.doc() != null) { assertToXContentEquivalent(updateRequest.doc().source(), parsedUpdateRequest.doc().source(), xContentType); @@ -693,6 +697,22 @@ public void testUpdate() throws IOException { } } + private void assertIfSeqNoAndTerm(DocWriteRequestrequest, DocWriteRequest parsedRequest) { + assertEquals(request.ifSeqNo(), parsedRequest.ifSeqNo()); + assertEquals(request.ifPrimaryTerm(), parsedRequest.ifPrimaryTerm()); + } + + private void setRandomIfSeqNoAndTerm(DocWriteRequest request, Map expectedParams) { + if (randomBoolean()) { + final long seqNo = randomNonNegativeLong(); + request.setIfSeqNo(seqNo); + expectedParams.put("if_seq_no", Long.toString(seqNo)); + final long primaryTerm = randomLongBetween(1, 200); + request.setIfPrimaryTerm(primaryTerm); + expectedParams.put("if_primary_term", Long.toString(primaryTerm)); + } + } + public void testUpdateWithDifferentContentTypes() { IllegalStateException exception = expectThrows(IllegalStateException.class, () -> { UpdateRequest updateRequest = new UpdateRequest(); From 7a0e29ca208ed4c8b6b8902d1892099212e1e53d Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Tue, 5 Feb 2019 13:36:14 +0100 Subject: [PATCH 06/19] feedback --- .../java/org/elasticsearch/client/RequestConvertersTests.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java index ca4821003ef06..41d0a0a59f420 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java @@ -697,12 +697,12 @@ public void testUpdate() throws IOException { } } - private void assertIfSeqNoAndTerm(DocWriteRequestrequest, DocWriteRequest parsedRequest) { + private static void assertIfSeqNoAndTerm(DocWriteRequestrequest, DocWriteRequest parsedRequest) { assertEquals(request.ifSeqNo(), parsedRequest.ifSeqNo()); assertEquals(request.ifPrimaryTerm(), parsedRequest.ifPrimaryTerm()); } - private void setRandomIfSeqNoAndTerm(DocWriteRequest request, Map expectedParams) { + private static void setRandomIfSeqNoAndTerm(DocWriteRequest request, Map expectedParams) { if (randomBoolean()) { final long seqNo = randomNonNegativeLong(); request.setIfSeqNo(seqNo); From e4384dab1c409ad9babe4c9b3c7c40927c7b9be2 Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Tue, 5 Feb 2019 14:58:38 +0100 Subject: [PATCH 07/19] update requests don't have parameters in url --- .../java/org/elasticsearch/client/RequestConvertersTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java index 41d0a0a59f420..4b15774568a6a 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java @@ -652,7 +652,7 @@ public void testUpdate() throws IOException { setRandomWaitForActiveShards(updateRequest::waitForActiveShards, ActiveShardCount.DEFAULT, expectedParams); setRandomVersion(updateRequest, expectedParams); setRandomVersionType(updateRequest::versionType, expectedParams); - setRandomIfSeqNoAndTerm(updateRequest, expectedParams); + setRandomIfSeqNoAndTerm(updateRequest, new HashMap<>()); // if* params are passed in the body if (randomBoolean()) { int retryOnConflict = randomIntBetween(0, 5); updateRequest.retryOnConflict(retryOnConflict); From 3be42abf4d8a8f83633540d91a296026b48028b4 Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Tue, 5 Feb 2019 15:07:56 +0100 Subject: [PATCH 08/19] indenting --- .../rest-api-spec/test/update/30_internal_version.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/update/30_internal_version.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/update/30_internal_version.yml index d09fa72c94098..deb1fd24574fa 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/update/30_internal_version.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/update/30_internal_version.yml @@ -1,9 +1,9 @@ --- "Internal version": - - skip: - features: warnings + - skip: + features: warnings - - do: + - do: catch: missing update: index: test_1 From ce44cdca96e7b43a63ca14f3a4f45881c96eb376 Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Tue, 5 Feb 2019 16:10:49 +0100 Subject: [PATCH 09/19] fix warning strings in rest --- .../resources/rest-api-spec/test/bulk/60_deprecated.yml | 4 ++-- .../rest-api-spec/test/delete/20_internal_version.yml | 4 ++-- .../rest-api-spec/test/index/30_internal_version.yml | 6 ++---- .../rest-api-spec/test/update/30_internal_version.yml | 4 ++-- 4 files changed, 8 insertions(+), 10 deletions(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/bulk/60_deprecated.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/bulk/60_deprecated.yml index 5fa95968c4ffb..b9e2d6af5b4b2 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/bulk/60_deprecated.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/bulk/60_deprecated.yml @@ -16,8 +16,8 @@ { "doc": { "f1": "v2" } } warnings: - "Deprecated field [_version] used, expected [version] instead" - - "Usage of internal versioning for optimistic concurrency control is deprecated and will be removed. Please use the `if_seq_no` and `if_primary_term` parameters instead. (request for index [test_index], type [test_type], id [test_id_1]" - - "Usage of internal versioning for optimistic concurrency control is deprecated and will be removed. Please use the `if_seq_no` and `if_primary_term` parameters instead. (request for index [test_index], type [test_type], id [test_id_2]" + - "Usage of internal versioning for optimistic concurrency control is deprecated and will be removed. Please use the `if_seq_no` and `if_primary_term` parameters instead. (request for index [test_index], type [test_type], id [test_id_1])" + - "Usage of internal versioning for optimistic concurrency control is deprecated and will be removed. Please use the `if_seq_no` and `if_primary_term` parameters instead. (request for index [test_index], type [test_type], id [test_id_2])" - do: bulk: diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/delete/20_internal_version.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/delete/20_internal_version.yml index 07b711aa74201..54c233c4731f5 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/delete/20_internal_version.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/delete/20_internal_version.yml @@ -20,7 +20,7 @@ id: 1 version: 2 warnings: - - "Usage of internal versioning for optimistic concurrency control is deprecated and will be removed. Please use the `if_seq_no` and `if_primary_term` parameters instead. (request for index [test_1], type [test], id [1]" + - "Usage of internal versioning for optimistic concurrency control is deprecated and will be removed. Please use the `if_seq_no` and `if_primary_term` parameters instead. (request for index [test_1], type [test], id [1])" - do: delete: @@ -29,6 +29,6 @@ id: 1 version: 1 warnings: - - "Usage of internal versioning for optimistic concurrency control is deprecated and will be removed. Please use the `if_seq_no` and `if_primary_term` parameters instead. (request for index [test_1], type [test], id [1]" + - "Usage of internal versioning for optimistic concurrency control is deprecated and will be removed. Please use the `if_seq_no` and `if_primary_term` parameters instead. (request for index [test_1], type [test], id [1])" - match: { _version: 2 } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/index/30_internal_version.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/index/30_internal_version.yml index c43c20cc2f4a9..a08c79be57b3a 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/index/30_internal_version.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/index/30_internal_version.yml @@ -17,8 +17,6 @@ type: test id: 1 body: { foo: bar } - warnings: - - "Usage of internal versioning for optimistic concurrency control is deprecated and will be removed. Please use the `if_seq_no` and `if_primary_term` parameters instead. (request for index [test_1], type [test], id [1]" - match: { _version: 2} @@ -31,7 +29,7 @@ body: { foo: bar } version: 1 warnings: - - "Usage of internal versioning for optimistic concurrency control is deprecated and will be removed. Please use the `if_seq_no` and `if_primary_term` parameters instead. (request for index [test_1], type [test], id [1]" + - "Usage of internal versioning for optimistic concurrency control is deprecated and will be removed. Please use the `if_seq_no` and `if_primary_term` parameters instead. (request for index [test_1], type [test], id [1])" - do: index: @@ -41,6 +39,6 @@ body: { foo: bar } version: 2 warnings: - - "Usage of internal versioning for optimistic concurrency control is deprecated and will be removed. Please use the `if_seq_no` and `if_primary_term` parameters instead. (request for index [test_1], type [test], id [1]" + - "Usage of internal versioning for optimistic concurrency control is deprecated and will be removed. Please use the `if_seq_no` and `if_primary_term` parameters instead. (request for index [test_1], type [test], id [1])" - match: { _version: 3 } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/update/30_internal_version.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/update/30_internal_version.yml index deb1fd24574fa..dee672bbd241a 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/update/30_internal_version.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/update/30_internal_version.yml @@ -13,7 +13,7 @@ body: doc: { foo: baz } warnings: - - "Usage of internal versioning for optimistic concurrency control is deprecated and will be removed. Please use the `if_seq_no` and `if_primary_term` parameters instead. (request for index [test_1], type [test], id [1]" + - "Usage of internal versioning for optimistic concurrency control is deprecated and will be removed. Please use the `if_seq_no` and `if_primary_term` parameters instead. (request for index [test_1], type [test], id [1])" - do: index: @@ -33,4 +33,4 @@ body: doc: { foo: baz } warnings: - - "Usage of internal versioning for optimistic concurrency control is deprecated and will be removed. Please use the `if_seq_no` and `if_primary_term` parameters instead. (request for index [test_1], type [test], id [1]" + - "Usage of internal versioning for optimistic concurrency control is deprecated and will be removed. Please use the `if_seq_no` and `if_primary_term` parameters instead. (request for index [test_1], type [test], id [1])" From 9a42490773d11aca0098247d6198ac759b0d42bd Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Tue, 5 Feb 2019 17:34:55 +0100 Subject: [PATCH 10/19] doc tweak --- docs/reference/migration/migrate_6_7.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/migration/migrate_6_7.asciidoc b/docs/reference/migration/migrate_6_7.asciidoc index 986e0885ffad1..f665f6bf489dc 100644 --- a/docs/reference/migration/migrate_6_7.asciidoc +++ b/docs/reference/migration/migrate_6_7.asciidoc @@ -17,7 +17,7 @@ See also <> and <>. [[breaking_67_indexing_changes]] === Indexing changes -==== Usage of `internal` versioning for optimistic concurrency control +==== UDeprecated usage of `internal` versioning for optimistic concurrency control `internal` version may not uniquely identify a document's version if an indexed document wasn't fully replicated when a primary fails. As such it is unsafe to use for From 812cbf3cc891c57ab7cacef09e571c371cf0f713 Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Tue, 5 Feb 2019 17:59:15 +0100 Subject: [PATCH 11/19] fix header warnings CRUDDocumentationIT --- .../client/documentation/CRUDDocumentationIT.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/CRUDDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/CRUDDocumentationIT.java index 426e9bd397cc4..c5663c35ab7cd 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/CRUDDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/CRUDDocumentationIT.java @@ -639,8 +639,8 @@ public void testDelete() throws Exception { // tag::delete-conflict try { DeleteResponse deleteResponse = client.delete( - new DeleteRequest("posts", "doc", "1").version(2), - RequestOptions.DEFAULT); + new DeleteRequest("posts", "doc", "1").setIfSeqNo(100L).setIfPrimaryTerm(2L), + RequestOptions.DEFAULT); } catch (ElasticsearchException exception) { if (exception.status() == RestStatus.CONFLICT) { // <1> From e0193f7b598d7edb5c0d24e8949095e8fc7eec2f Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Tue, 5 Feb 2019 18:33:13 +0100 Subject: [PATCH 12/19] line length --- .../client/documentation/CRUDDocumentationIT.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/CRUDDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/CRUDDocumentationIT.java index c5663c35ab7cd..0dee79d701900 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/CRUDDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/CRUDDocumentationIT.java @@ -639,7 +639,8 @@ public void testDelete() throws Exception { // tag::delete-conflict try { DeleteResponse deleteResponse = client.delete( - new DeleteRequest("posts", "doc", "1").setIfSeqNo(100L).setIfPrimaryTerm(2L), + new DeleteRequest("posts", "doc", "1") + .setIfSeqNo(100L).setIfPrimaryTerm(2L), RequestOptions.DEFAULT); } catch (ElasticsearchException exception) { if (exception.status() == RestStatus.CONFLICT) { From 356a14ab416f8d393e7787745bd6dd101917938b Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Tue, 5 Feb 2019 18:37:01 +0100 Subject: [PATCH 13/19] white space --- .../main/java/org/elasticsearch/action/index/IndexRequest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java b/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java index 3ff5139cd451c..ba715102cf001 100644 --- a/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java +++ b/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java @@ -203,7 +203,6 @@ public ActionRequestValidationException validate() { DocWriteRequest.logDeprecationWarnings(this, DEPRECATION_LOGGER); - return validationException; } From 94f2d4d4524b729dd708a8098e4e5d77c7f6c832 Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Tue, 5 Feb 2019 19:43:26 +0100 Subject: [PATCH 14/19] typos --- docs/reference/migration/migrate_6_7.asciidoc | 2 +- .../src/main/resources/rest-api-spec/test/delete/21_cas.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/reference/migration/migrate_6_7.asciidoc b/docs/reference/migration/migrate_6_7.asciidoc index f665f6bf489dc..1103a0ca0b479 100644 --- a/docs/reference/migration/migrate_6_7.asciidoc +++ b/docs/reference/migration/migrate_6_7.asciidoc @@ -17,7 +17,7 @@ See also <> and <>. [[breaking_67_indexing_changes]] === Indexing changes -==== UDeprecated usage of `internal` versioning for optimistic concurrency control +==== Deprecated usage of `internal` versioning for optimistic concurrency control `internal` version may not uniquely identify a document's version if an indexed document wasn't fully replicated when a primary fails. As such it is unsafe to use for diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/delete/21_cas.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/delete/21_cas.yml index 4269723b442e7..3baa81090b0a0 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/delete/21_cas.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/delete/21_cas.yml @@ -2,7 +2,7 @@ "Compare and set with sequence numbers": - skip: version: " - 6.5.99" - reason: sequence numbers can be used for cas as of 6.6.0 + reason: sequence numbers can be used for CAS as of 6.6.0 - do: index: From 7a29205679d9820fbd02a083f8868dddfa3bd45c Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Tue, 5 Feb 2019 20:02:59 +0100 Subject: [PATCH 15/19] fix warning headers --- .../java/org/elasticsearch/client/CrudIT.java | 21 ++++++++++++++++--- .../documentation/CRUDDocumentationIT.java | 6 ++++-- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/CrudIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/CrudIT.java index 27e6221f0ec84..71a8c2014122e 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/CrudIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/CrudIT.java @@ -105,7 +105,8 @@ public void testDelete() throws IOException { IndexResponse indexResponse = highLevelClient().index( new IndexRequest("index", "type", docId).source(Collections.singletonMap("foo", "bar")), RequestOptions.DEFAULT); DeleteRequest deleteRequest = new DeleteRequest("index", "type", docId); - if (randomBoolean()) { + final boolean useSeqNo = randomBoolean(); + if (useSeqNo) { deleteRequest.setIfSeqNo(indexResponse.getSeqNo()); deleteRequest.setIfPrimaryTerm(indexResponse.getPrimaryTerm()); } else { @@ -117,6 +118,11 @@ public void testDelete() throws IOException { assertEquals("type", deleteResponse.getType()); assertEquals(docId, deleteResponse.getId()); assertEquals(DocWriteResponse.Result.DELETED, deleteResponse.getResult()); + if (useSeqNo == false) { + assertWarnings("Usage of internal versioning for optimistic concurrency control is deprecated and will be removed." + + " Please use the `if_seq_no` and `if_primary_term` parameters instead." + + " (request for index [index], type [type], id [id])"); + } } { // Testing non existing document @@ -153,6 +159,9 @@ public void testDelete() throws IOException { } else { assertEquals("Elasticsearch exception [type=version_conflict_engine_exception, reason=[type][" + docId + "]: " + "version conflict, current version [1] is different than the one provided [2]]", exception.getMessage()); + assertWarnings("Usage of internal versioning for optimistic concurrency control is deprecated and will be removed." + + " Please use the `if_seq_no` and `if_primary_term` parameters instead." + + " (request for index [index], type [type], id [version_conflict])"); } assertEquals("index", exception.getMetadata("es.index").get(0)); } @@ -429,6 +438,9 @@ public void testMultiGet() throws IOException { public void testIndex() throws IOException { final XContentType xContentType = randomFrom(XContentType.values()); + highLevelClient().indices().create( + new CreateIndexRequest("index").settings(Collections.singletonMap("index.number_of_shards", "1")), + RequestOptions.DEFAULT); { IndexRequest indexRequest = new IndexRequest("index", "type"); indexRequest.source(XContentBuilder.builder(xContentType.xContent()).startObject().field("test", "test").endObject()); @@ -479,7 +491,7 @@ public void testIndex() throws IOException { IndexRequest wrongRequest = new IndexRequest("index", "type", "id"); wrongRequest.source(XContentBuilder.builder(xContentType.xContent()).startObject().field("field", "test").endObject()); if (seqNosForConflict) { - wrongRequest.setIfSeqNo(2).setIfPrimaryTerm(2); + wrongRequest.setIfSeqNo(5).setIfPrimaryTerm(2); } else { wrongRequest.version(5); } @@ -490,7 +502,7 @@ public void testIndex() throws IOException { assertEquals(RestStatus.CONFLICT, exception.status()); if (seqNosForConflict) { assertEquals("Elasticsearch exception [type=version_conflict_engine_exception, reason=[type][id]: " + - "version conflict, required seqNo [1], primary term [5]. current document has seqNo [2] and primary term [1]]", + "version conflict, required seqNo [5], primary term [2]. current document has seqNo [2] and primary term [1]]", exception.getMessage()); } else { assertEquals("Elasticsearch exception [type=version_conflict_engine_exception, reason=[type][id]: " + @@ -563,6 +575,9 @@ public void testIndex() throws IOException { } public void testUpdate() throws IOException { + highLevelClient().indices().create( + new CreateIndexRequest("index").settings(Collections.singletonMap("index.number_of_shards", "1")), + RequestOptions.DEFAULT); { UpdateRequest updateRequest = new UpdateRequest("index", "type", "does_not_exist"); updateRequest.doc(singletonMap("field", "value"), randomFrom(XContentType.values())); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/CRUDDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/CRUDDocumentationIT.java index 0dee79d701900..8b119915ae165 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/CRUDDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/CRUDDocumentationIT.java @@ -224,7 +224,8 @@ public void testIndex() throws Exception { // tag::index-conflict IndexRequest request = new IndexRequest("posts", "doc", "1") .source("field", "value") - .version(1); + .setIfSeqNo(100L) + .setIfPrimaryTerm(1L); try { IndexResponse response = client.index(request, RequestOptions.DEFAULT); } catch(ElasticsearchException e) { @@ -434,7 +435,8 @@ public void testUpdate() throws Exception { // tag::update-conflict UpdateRequest request = new UpdateRequest("posts", "doc", "1") .doc("field", "value") - .version(1); + .setIfSeqNo(100L) + .setIfPrimaryTerm(1L); try { UpdateResponse updateResponse = client.update( request, RequestOptions.DEFAULT); From 3d7d569ee73642c0b835daff2b90d58e168ec3fc Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Tue, 5 Feb 2019 20:18:33 +0100 Subject: [PATCH 16/19] add float to doc --- docs/reference/migration/migrate_6_7.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/reference/migration/migrate_6_7.asciidoc b/docs/reference/migration/migrate_6_7.asciidoc index 1103a0ca0b479..13b7d4b2cbc91 100644 --- a/docs/reference/migration/migrate_6_7.asciidoc +++ b/docs/reference/migration/migrate_6_7.asciidoc @@ -17,6 +17,7 @@ See also <> and <>. [[breaking_67_indexing_changes]] === Indexing changes +[float] ==== Deprecated usage of `internal` versioning for optimistic concurrency control `internal` version may not uniquely identify a document's version if an indexed document From de31f5d13484d27616c10022f075ed5f3614224c Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Tue, 5 Feb 2019 20:26:51 +0100 Subject: [PATCH 17/19] add warnings to watcher test --- .../watcher/put_watch/80_put_get_watch_with_passwords.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/watcher/put_watch/80_put_get_watch_with_passwords.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/watcher/put_watch/80_put_get_watch_with_passwords.yml index 357425f547480..a4b1a934d510a 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/watcher/put_watch/80_put_get_watch_with_passwords.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/watcher/put_watch/80_put_get_watch_with_passwords.yml @@ -392,7 +392,8 @@ setup: --- "Test putting a watch with a redacted password with current version works": - + - skip: + features: "warnings" - do: xpack.watcher.put_watch: id: "my_watch_with_version" @@ -462,6 +463,8 @@ setup: } } } + warnings: + - "Usage of internal versioning for optimistic concurrency control is deprecated and will be removed. Please use the `if_seq_no` and `if_primary_term` parameters instead." - match: { _id: "my_watch_with_version" } - match: { _version: 2 } From de91a2768d3180ea194913ee5d9858a6b8301aff Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Tue, 5 Feb 2019 21:22:48 +0100 Subject: [PATCH 18/19] fix crudIT --- .../src/test/java/org/elasticsearch/client/CrudIT.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/CrudIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/CrudIT.java index 71a8c2014122e..569cc9c5f4f2c 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/CrudIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/CrudIT.java @@ -507,6 +507,9 @@ public void testIndex() throws IOException { } else { assertEquals("Elasticsearch exception [type=version_conflict_engine_exception, reason=[type][id]: " + "version conflict, current version [2] is different than the one provided [5]]", exception.getMessage()); + assertWarnings("Usage of internal versioning for optimistic concurrency control is deprecated and will be removed. " + + "Please use the `if_seq_no` and `if_primary_term` parameters instead. " + + "(request for index [index], type [type], id [id])"); } assertEquals("index", exception.getMetadata("es.index").get(0)); } @@ -575,9 +578,6 @@ public void testIndex() throws IOException { } public void testUpdate() throws IOException { - highLevelClient().indices().create( - new CreateIndexRequest("index").settings(Collections.singletonMap("index.number_of_shards", "1")), - RequestOptions.DEFAULT); { UpdateRequest updateRequest = new UpdateRequest("index", "type", "does_not_exist"); updateRequest.doc(singletonMap("field", "value"), randomFrom(XContentType.values())); From edbf803a640d4b711712b706c614354e5f983a46 Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Tue, 5 Feb 2019 22:16:28 +0100 Subject: [PATCH 19/19] skip versioned tests in mix clusters until we find a better solutions for headers --- .../main/resources/rest-api-spec/test/bulk/60_deprecated.yml | 4 ++-- .../rest-api-spec/test/delete/20_internal_version.yml | 2 ++ .../rest-api-spec/test/index/30_internal_version.yml | 4 +++- .../rest-api-spec/test/update/30_internal_version.yml | 2 ++ 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/bulk/60_deprecated.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/bulk/60_deprecated.yml index b9e2d6af5b4b2..06d1dd1ceb92b 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/bulk/60_deprecated.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/bulk/60_deprecated.yml @@ -3,8 +3,8 @@ "Deprecated parameters should produce warning in Bulk query": - skip: - version: " - 6.0.99" - reason: some parameters are deprecated starting from 6.1, their equivalents without underscore are used instead + version: " - 6.6.99" + reason: versioned operations were deprecated in 6.7 features: "warnings" - do: diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/delete/20_internal_version.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/delete/20_internal_version.yml index 54c233c4731f5..a36dc52880af1 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/delete/20_internal_version.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/delete/20_internal_version.yml @@ -1,6 +1,8 @@ --- "Internal version": - skip: + version: " - 6.6.99" + reason: versioned operations were deprecated in 6.7 features: warnings - do: diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/index/30_internal_version.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/index/30_internal_version.yml index a08c79be57b3a..650f0bd6fa573 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/index/30_internal_version.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/index/30_internal_version.yml @@ -1,7 +1,9 @@ --- "Internal version": - skip: - features: warnings + version: " - 6.6.99" + reason: versioned operations were deprecated in 6.7 + features: warnings - do: index: diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/update/30_internal_version.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/update/30_internal_version.yml index dee672bbd241a..4ea5bb0e9cd90 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/update/30_internal_version.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/update/30_internal_version.yml @@ -1,6 +1,8 @@ --- "Internal version": - skip: + version: " - 6.6.99" + reason: versioned operations were deprecated in 6.7 features: warnings - do: