From b68478813fa09be32e4dfa6e9fea92250a54f782 Mon Sep 17 00:00:00 2001 From: pmpailis Date: Thu, 28 Sep 2023 21:09:16 +0300 Subject: [PATCH 1/4] updating test cases for sparse_vector exists queries --- .../test/search.vectors/90_sparse_vector.yml | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/90_sparse_vector.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/90_sparse_vector.yml index 27aa0e6e9a20b..3d99656372f09 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/90_sparse_vector.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/90_sparse_vector.yml @@ -145,8 +145,8 @@ "Sparse vector in 7.x": - skip: features: allowed_warnings - version: "all" - reason: "AwaitsFix https://github.com/elastic/elasticsearch/issues/100003" + version: "8.0.0 - " + reason: "sparse_vector field type supported in 7.x" - do: allowed_warnings: - "The [sparse_vector] field type is deprecated and will be removed in 8.0." @@ -163,16 +163,6 @@ - match: { acknowledged: true } - - do: - catch: /\[sparse_vector\] fields do not support \[exists\] queries/ - search: - rest_total_hits_as_int: true - index: test - body: - query: - exists: - field: ml.tokens - --- "Sparse vector in 8.x": - skip: @@ -189,3 +179,13 @@ type: text ml.tokens: type: sparse_vector + + - do: + catch: /\[sparse_vector\] fields do not support \[exists\] queries/ + search: + rest_total_hits_as_int: true + index: test + body: + query: + exists: + field: ml.tokens From 22de3bab8b0ac55c69751a6fb4d75bf50243102d Mon Sep 17 00:00:00 2001 From: pmpailis Date: Thu, 28 Sep 2023 21:57:39 +0300 Subject: [PATCH 2/4] updating --- .../test/search.vectors/90_sparse_vector.yml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/90_sparse_vector.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/90_sparse_vector.yml index 3d99656372f09..8f9e1a281525d 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/90_sparse_vector.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/90_sparse_vector.yml @@ -179,13 +179,3 @@ type: text ml.tokens: type: sparse_vector - - - do: - catch: /\[sparse_vector\] fields do not support \[exists\] queries/ - search: - rest_total_hits_as_int: true - index: test - body: - query: - exists: - field: ml.tokens From 2043c5ac89e155c599b2ea34c593239d85efa54e Mon Sep 17 00:00:00 2001 From: pmpailis Date: Tue, 3 Oct 2023 11:28:34 +0300 Subject: [PATCH 3/4] addressing PR comments - adding new explicit test for 8.x and adding a deprecation warning for 7.x versions --- .../test/search.vectors/90_sparse_vector.yml | 38 ++++++++++++++++--- .../vectors/SparseVectorFieldMapper.java | 8 +++- 2 files changed, 38 insertions(+), 8 deletions(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/90_sparse_vector.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/90_sparse_vector.yml index 8f9e1a281525d..8e15cf215cc4a 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/90_sparse_vector.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/90_sparse_vector.yml @@ -1,5 +1,5 @@ --- -"Indexing and searching sparse vectors": +"Indexing and searching sparse vectors in >=8.11": - skip: version: " - 8.10.99" @@ -77,16 +77,24 @@ index: test id: "3" body: - text: "doing nothing will result in nothing" + text: "empty array with no nested values - should not be retrieved in exists queries" ml: - tokens: {} + tokens: [ ] + - do: + index: + index: test + id: "4" + body: + text: "should still respond to exists queries if when empty" + ml: + tokens: { } - match: { result: "created" } - do: index: index: test - id: "4" + id: "5" body: text: "other embeddings available only" embeddings: @@ -144,7 +152,7 @@ --- "Sparse vector in 7.x": - skip: - features: allowed_warnings + features: ["allowed_warnings"] version: "8.0.0 - " reason: "sparse_vector field type supported in 7.x" - do: @@ -163,8 +171,17 @@ - match: { acknowledged: true } + - do: + allowed_warnings: + - "[sparse_vector] field type in old 7.x indices is allowed to contain [sparse_vector] fields, but they cannot be indexed or searched." + search: + index: test + body: + query: + exists: + field: ml.tokens --- -"Sparse vector in 8.x": +"Sparse vector in 8.0.0 <= x < 8.11.0": - skip: version: " - 7.99.99, 8.11.0 - " reason: "sparse_vector field type not supported in 8.x until 8.11.0" @@ -179,3 +196,12 @@ type: text ml.tokens: type: sparse_vector + - do: + catch: /\[sparse_vector\] fields do not support \[exists\] queries/ + search: + rest_total_hits_as_int: true + index: test + body: + query: + exists: + field: ml.tokens diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/SparseVectorFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/SparseVectorFieldMapper.java index eded63e12e758..ee24ad0c4721b 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/SparseVectorFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/SparseVectorFieldMapper.java @@ -9,6 +9,7 @@ package org.elasticsearch.index.mapper.vectors; import org.apache.lucene.document.FeatureField; +import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.logging.DeprecationCategory; @@ -110,8 +111,11 @@ public Query termQuery(Object value, SearchExecutionContext context) { @Override public Query existsQuery(SearchExecutionContext context) { - // No support for exists queries prior to this version - if (context.getIndexSettings().getIndexVersionCreated().before(SPARSE_VECTOR_IN_FIELD_NAMES_INDEX_VERSION)) { + if (context.getIndexSettings().getIndexVersionCreated().before(PREVIOUS_SPARSE_VECTOR_INDEX_VERSION)) { + deprecationLogger.warn(DeprecationCategory.MAPPINGS, "sparse_vector", ERROR_MESSAGE_7X); + return new MatchNoDocsQuery(); + } else if (context.getIndexSettings().getIndexVersionCreated().before(SPARSE_VECTOR_IN_FIELD_NAMES_INDEX_VERSION)) { + // No support for exists queries prior to this version on 8.x throw new IllegalArgumentException("[sparse_vector] fields do not support [exists] queries"); } return super.existsQuery(context); From 3adae27bb33135555d9aca915fb2aae040fa8fb6 Mon Sep 17 00:00:00 2001 From: pmpailis Date: Tue, 3 Oct 2023 12:13:38 +0300 Subject: [PATCH 4/4] updating expected error message in test --- .../rest-api-spec/test/search.vectors/90_sparse_vector.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/90_sparse_vector.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/90_sparse_vector.yml index 8e15cf215cc4a..7b8ce0b961b93 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/90_sparse_vector.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/90_sparse_vector.yml @@ -197,7 +197,7 @@ ml.tokens: type: sparse_vector - do: - catch: /\[sparse_vector\] fields do not support \[exists\] queries/ + catch: /\[sparse_vector\] fields do not support \[exists\] queries|no such index.*/ search: rest_total_hits_as_int: true index: test