diff --git a/docs/reference/indices/field-usage-stats.asciidoc b/docs/reference/indices/field-usage-stats.asciidoc index d63faffb25fde..b48d17cb1c2a9 100644 --- a/docs/reference/indices/field-usage-stats.asciidoc +++ b/docs/reference/indices/field-usage-stats.asciidoc @@ -128,7 +128,8 @@ The API returns the following response: "doc_values" : 1, "points" : 0, "norms" : 1, - "term_vectors" : 0 + "term_vectors" : 0, + "knn_vectors" : 0 }, "fields": { "_id": { @@ -146,7 +147,8 @@ The API returns the following response: "doc_values" : 0, "points" : 0, "norms" : 0, - "term_vectors" : 0 + "term_vectors" : 0, + "knn_vectors" : 0 }, "_source": {...}, "context": {...}, diff --git a/server/src/main/java/org/elasticsearch/index/search/stats/FieldUsageStats.java b/server/src/main/java/org/elasticsearch/index/search/stats/FieldUsageStats.java index 4c21e557b4e26..4f068591be42b 100644 --- a/server/src/main/java/org/elasticsearch/index/search/stats/FieldUsageStats.java +++ b/server/src/main/java/org/elasticsearch/index/search/stats/FieldUsageStats.java @@ -8,6 +8,7 @@ package org.elasticsearch.index.search.stats; +import org.elasticsearch.Version; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -39,6 +40,7 @@ public class FieldUsageStats implements ToXContentObject, Writeable { public static final String TERM_VECTORS = "term_vectors"; // possibly refine this one public static final String POINTS = "points"; public static final String PROXIMITY = "proximity"; + public static final String KNN_VECTORS = "knn_vectors"; private final Map stats; @@ -122,11 +124,12 @@ public enum UsageContext { PAYLOADS, TERM_VECTORS, // possibly refine this one POINTS, + KNN_VECTORS, } public static class PerFieldUsageStats implements ToXContentFragment, Writeable { - static final PerFieldUsageStats EMPTY = new PerFieldUsageStats(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + static final PerFieldUsageStats EMPTY = new PerFieldUsageStats(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); private final long any; private final long proximity; @@ -141,6 +144,7 @@ public static class PerFieldUsageStats implements ToXContentFragment, Writeable private final long payloads; private final long termVectors; private final long points; + private final long knnVectors; public PerFieldUsageStats( long any, @@ -155,7 +159,8 @@ public PerFieldUsageStats( long norms, long payloads, long termVectors, - long points + long points, + long knnVectors ) { this.any = any; this.proximity = proximity; @@ -170,6 +175,7 @@ public PerFieldUsageStats( this.payloads = payloads; this.termVectors = termVectors; this.points = points; + this.knnVectors = knnVectors; } private PerFieldUsageStats add(PerFieldUsageStats other) { @@ -186,7 +192,8 @@ private PerFieldUsageStats add(PerFieldUsageStats other) { norms + other.norms, payloads + other.payloads, termVectors + other.termVectors, - points + other.points + points + other.points, + knnVectors + other.knnVectors ); } @@ -204,6 +211,11 @@ public PerFieldUsageStats(StreamInput in) throws IOException { payloads = in.readVLong(); termVectors = in.readVLong(); points = in.readVLong(); + if (in.getVersion().onOrAfter(Version.V_8_1_0)) { + knnVectors = in.readVLong(); + } else { + knnVectors = 0; + } } @Override @@ -221,6 +233,9 @@ public void writeTo(StreamOutput out) throws IOException { out.writeVLong(payloads); out.writeVLong(termVectors); out.writeVLong(points); + if (out.getVersion().onOrAfter(Version.V_8_1_0)) { + out.writeVLong(knnVectors); + } } @Override @@ -240,6 +255,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(POINTS, points); builder.field(NORMS, norms); builder.field(TERM_VECTORS, termVectors); + builder.field(KNN_VECTORS, knnVectors); return builder; } @@ -278,6 +294,9 @@ public Set keySet() { if (points > 0L) { set.add(UsageContext.POINTS); } + if (knnVectors > 0L) { + set.add(UsageContext.KNN_VECTORS); + } return set; } @@ -325,6 +344,10 @@ public long getPoints() { return points; } + public long getKnnVectors() { + return knnVectors; + } + public long getProximity() { return proximity; } diff --git a/server/src/main/java/org/elasticsearch/index/search/stats/ShardFieldUsageTracker.java b/server/src/main/java/org/elasticsearch/index/search/stats/ShardFieldUsageTracker.java index d166eebc50fdd..e4e2c3e80a926 100644 --- a/server/src/main/java/org/elasticsearch/index/search/stats/ShardFieldUsageTracker.java +++ b/server/src/main/java/org/elasticsearch/index/search/stats/ShardFieldUsageTracker.java @@ -61,7 +61,8 @@ public FieldUsageStats stats(String... fields) { ifs.norms.longValue(), ifs.payloads.longValue(), ifs.termVectors.longValue(), - ifs.points.longValue() + ifs.points.longValue(), + ifs.knnVectors.longValue() ); stats.put(entry.getKey(), pf); } @@ -83,6 +84,7 @@ static class InternalFieldStats { final LongAdder payloads = new LongAdder(); final LongAdder termVectors = new LongAdder(); final LongAdder points = new LongAdder(); + final LongAdder knnVectors = new LongAdder(); } static class PerField { @@ -98,6 +100,7 @@ static class PerField { volatile boolean payloads; volatile boolean termVectors; volatile boolean points; + volatile boolean knnVectors; } public class FieldUsageStatsTrackingSession implements FieldUsageNotifier, Releasable { @@ -159,6 +162,10 @@ public void close() { any = true; fieldStats.termVectors.increment(); } + if (pf.knnVectors) { + any = true; + fieldStats.knnVectors.increment(); + } if (any) { fieldStats.any.increment(); } @@ -227,5 +234,10 @@ public void onPointsUsed(String field) { public void onTermVectorsUsed(String field) { getOrAdd(field).termVectors = true; } + + @Override + public void onKnnVectorsUsed(String field) { + getOrAdd(field).knnVectors = true; + } } } diff --git a/server/src/main/java/org/elasticsearch/search/internal/FieldUsageTrackingDirectoryReader.java b/server/src/main/java/org/elasticsearch/search/internal/FieldUsageTrackingDirectoryReader.java index 87ab025a813c3..c4cf55f3113b3 100644 --- a/server/src/main/java/org/elasticsearch/search/internal/FieldUsageTrackingDirectoryReader.java +++ b/server/src/main/java/org/elasticsearch/search/internal/FieldUsageTrackingDirectoryReader.java @@ -25,7 +25,10 @@ import org.apache.lucene.index.StoredFieldVisitor; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.index.VectorValues; +import org.apache.lucene.search.TopDocs; import org.apache.lucene.search.suggest.document.CompletionTerms; +import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.automaton.CompiledAutomaton; import org.elasticsearch.common.lucene.index.SequentialStoredFieldsLeafReader; @@ -83,6 +86,8 @@ public interface FieldUsageNotifier { void onPointsUsed(String field); void onTermVectorsUsed(String field); + + void onKnnVectorsUsed(String field); } public static final class FieldUsageTrackingLeafReader extends SequentialStoredFieldsLeafReader { @@ -181,6 +186,24 @@ public NumericDocValues getNormValues(String field) throws IOException { return numericDocValues; } + @Override + public VectorValues getVectorValues(String field) throws IOException { + VectorValues vectorValues = super.getVectorValues(field); + if (vectorValues != null) { + notifier.onKnnVectorsUsed(field); + } + return vectorValues; + } + + @Override + public TopDocs searchNearestVectors(String field, float[] target, int k, Bits acceptDocs) throws IOException { + TopDocs topDocs = super.searchNearestVectors(field, target, k, acceptDocs); + if (topDocs != null) { + notifier.onKnnVectorsUsed(field); + } + return topDocs; + } + @Override public String toString() { final StringBuilder sb = new StringBuilder("FieldUsageTrackingLeafReader(reader="); diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/vectors/50_dense_vector_field_usage.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/vectors/50_dense_vector_field_usage.yml new file mode 100644 index 0000000000000..004c815556fe1 --- /dev/null +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/vectors/50_dense_vector_field_usage.yml @@ -0,0 +1,81 @@ +setup: + - skip: + features: headers + + - do: + indices.create: + index: futest + body: + settings: + routing.rebalance.enable: none + index.number_of_shards: 1 + index.number_of_replicas: 0 + mappings: + properties: + name: + type: keyword + vector: + type: dense_vector + dims: 5 + index: true + similarity: l2_norm + + - do: + index: + index: futest + body: + name: cow.jpg + vector: [ 230.0, 300.33, -34.8988, 15.555, -200.0 ] + + - do: + index: + index: futest + id: 2 + body: + name: moose.jpg + vector: [ -0.5, 100.0, -13, 14.8, -156.0 ] + + - do: + index: + index: futest + id: 3 + body: + name: rabbit.jpg + vector: [ 0.5, 111.3, -13.0, 14.8, -156.0 ] + + - do: + indices.refresh: { } + +--- +"Field usage": + - do: + knn_search: + index: futest + body: + fields: [ "name" ] + knn: + field: vector + query_vector: [-0.5, 90.0, -10, 14.8, -156.0] + k: 2 + num_candidates: 3 + + - match: {hits.hits.0._id: "2"} + - match: {hits.hits.0.fields.name.0: "moose.jpg"} + + - match: {hits.hits.1._id: "3"} + - match: {hits.hits.1.fields.name.0: "rabbit.jpg"} + + - do: + indices.field_usage_stats: { index: futest } + + - is_true: futest + - length: { futest.shards: 1 } + + - gt: { futest.shards.0.stats.all_fields.any: 0 } + - gt: { futest.shards.0.stats.all_fields.knn_vectors: 0 } + + - gt: { futest.shards.0.stats.fields.vector.any: 0 } + - gt: { futest.shards.0.stats.fields.vector.knn_vectors: 0 } + + - gt: { futest.shards.0.stats.fields._id.stored_fields: 0 } + - match: { futest.shards.0.stats.fields._id.knn_vectors: 0 }