From 4ab4133433bc13a41a011157485a447f38b1b3dc Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Thu, 17 Jan 2019 10:22:55 +0100 Subject: [PATCH 01/29] add fetch phase --- .../remote/RemoteRequestBuildersTests.java | 8 +++ .../index/query/InnerHitBuilder.java | 21 +++++- .../index/query/InnerHitContextBuilder.java | 1 + .../rest/action/search/RestSearchAction.java | 3 + .../search/DefaultSearchContext.java | 11 +++ .../org/elasticsearch/search/SearchHit.java | 45 ++++++++++++- .../elasticsearch/search/SearchModule.java | 2 + .../elasticsearch/search/SearchService.java | 5 ++ .../metrics/TopHitsAggregationBuilder.java | 37 ++++++++-- .../metrics/TopHitsAggregatorFactory.java | 7 +- .../search/builder/SearchSourceBuilder.java | 38 ++++++++++- .../SeqNoPrimaryTermFetchSubPhase.java | 67 +++++++++++++++++++ .../internal/FilteredSearchContext.java | 10 +++ .../search/internal/SearchContext.java | 8 ++- .../search/internal/SubSearchContext.java | 11 +++ .../action/search/ExpandSearchPhaseTests.java | 4 +- .../index/query/InnerHitBuilderTests.java | 2 + .../aggregations/metrics/TopHitsIT.java | 12 ++++ .../aggregations/metrics/TopHitsTests.java | 3 + .../search/RandomSearchRequestGenerator.java | 3 + .../elasticsearch/test/TestSearchContext.java | 10 +++ 21 files changed, 296 insertions(+), 12 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/search/fetch/subphase/SeqNoPrimaryTermFetchSubPhase.java diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/remote/RemoteRequestBuildersTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/remote/RemoteRequestBuildersTests.java index 2f801811327b8..4e5b99f82e8ba 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/remote/RemoteRequestBuildersTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/remote/RemoteRequestBuildersTests.java @@ -179,6 +179,11 @@ public void testInitialSearchParamsMisc() { fetchVersion = randomBoolean(); searchRequest.source().version(fetchVersion); } + Boolean fetchSeqNoAndTerm = null; + if (randomBoolean()) { + fetchSeqNoAndTerm = randomBoolean(); + searchRequest.source().seqNoAndPrimaryTerm(fetchSeqNoAndTerm); + } Map params = initialSearch(searchRequest, query, remoteVersion).getParameters(); @@ -189,6 +194,9 @@ public void testInitialSearchParamsMisc() { } assertThat(params, hasEntry("size", Integer.toString(size))); assertThat(params, fetchVersion == null || fetchVersion == true ? hasEntry("version", null) : not(hasEntry("version", null))); + assertThat(params, fetchSeqNoAndTerm == null || fetchSeqNoAndTerm ? hasEntry("seq_no", null) : not(hasEntry("seq_no", null))); + assertThat(params, fetchSeqNoAndTerm == null || fetchSeqNoAndTerm ? + hasEntry("primary_term", null) : not(hasEntry("primary_term", null))); } private void assertScroll(Version remoteVersion, Map params, TimeValue requested) { diff --git a/server/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java b/server/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java index 733875075b051..cc68ccd3f3e32 100644 --- a/server/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java @@ -68,6 +68,7 @@ public final class InnerHitBuilder implements Writeable, ToXContentObject { PARSER.declareInt(InnerHitBuilder::setSize, SearchSourceBuilder.SIZE_FIELD); PARSER.declareBoolean(InnerHitBuilder::setExplain, SearchSourceBuilder.EXPLAIN_FIELD); PARSER.declareBoolean(InnerHitBuilder::setVersion, SearchSourceBuilder.VERSION_FIELD); + PARSER.declareBoolean(InnerHitBuilder::setSeqNoAndPrimaryTerm, SearchSourceBuilder.SEQ_NO_PRIMARY_TERM_FIELD); PARSER.declareBoolean(InnerHitBuilder::setTrackScores, SearchSourceBuilder.TRACK_SCORES_FIELD); PARSER.declareStringArray(InnerHitBuilder::setStoredFieldNames, SearchSourceBuilder.STORED_FIELDS_FIELD); PARSER.declareObjectArray(InnerHitBuilder::setDocValueFields, @@ -117,7 +118,6 @@ public final class InnerHitBuilder implements Writeable, ToXContentObject { }, COLLAPSE_FIELD, ObjectParser.ValueType.OBJECT); } - private String name; private boolean ignoreUnmapped; @@ -125,6 +125,7 @@ public final class InnerHitBuilder implements Writeable, ToXContentObject { private int size = 3; private boolean explain; private boolean version; + private boolean seqNoAndPrimaryTerm; private boolean trackScores; private StoredFieldsContext storedFieldsContext; @@ -155,6 +156,11 @@ public InnerHitBuilder(StreamInput in) throws IOException { size = in.readVInt(); explain = in.readBoolean(); version = in.readBoolean(); + if (in.getVersion().onOrAfter(Version.V_7_0_0)){ + seqNoAndPrimaryTerm = in.readBoolean(); + } else { + seqNoAndPrimaryTerm = false; + } trackScores = in.readBoolean(); storedFieldsContext = in.readOptionalWriteable(StoredFieldsContext::new); if (in.getVersion().before(Version.V_6_4_0)) { @@ -199,6 +205,9 @@ public void writeTo(StreamOutput out) throws IOException { out.writeVInt(size); out.writeBoolean(explain); out.writeBoolean(version); + if (out.getVersion().onOrAfter(Version.V_7_0_0)) { + out.writeBoolean(seqNoAndPrimaryTerm); + } out.writeBoolean(trackScores); out.writeOptionalWriteable(storedFieldsContext); if (out.getVersion().before(Version.V_6_4_0)) { @@ -299,6 +308,15 @@ public InnerHitBuilder setVersion(boolean version) { return this; } + public boolean isSeqNoAndPrimaryTerm() { + return seqNoAndPrimaryTerm; + } + + public InnerHitBuilder setSeqNoAndPrimaryTerm(boolean seqNoAndPrimaryTerm) { + this.seqNoAndPrimaryTerm = seqNoAndPrimaryTerm; + return this; + } + public boolean isTrackScores() { return trackScores; } @@ -436,6 +454,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(SearchSourceBuilder.FROM_FIELD.getPreferredName(), from); builder.field(SearchSourceBuilder.SIZE_FIELD.getPreferredName(), size); builder.field(SearchSourceBuilder.VERSION_FIELD.getPreferredName(), version); + builder.field(SearchSourceBuilder.SEQ_NO_PRIMARY_TERM_FIELD.getPreferredName(), seqNoAndPrimaryTerm); builder.field(SearchSourceBuilder.EXPLAIN_FIELD.getPreferredName(), explain); builder.field(SearchSourceBuilder.TRACK_SCORES_FIELD.getPreferredName(), trackScores); if (fetchSourceContext != null) { diff --git a/server/src/main/java/org/elasticsearch/index/query/InnerHitContextBuilder.java b/server/src/main/java/org/elasticsearch/index/query/InnerHitContextBuilder.java index 1fe781f38ce27..9e6a766aed891 100644 --- a/server/src/main/java/org/elasticsearch/index/query/InnerHitContextBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/InnerHitContextBuilder.java @@ -78,6 +78,7 @@ protected void setupInnerHitsContext(QueryShardContext queryShardContext, innerHitsContext.size(innerHitBuilder.getSize()); innerHitsContext.explain(innerHitBuilder.isExplain()); innerHitsContext.version(innerHitBuilder.isVersion()); + innerHitsContext.seqNoAndPrimaryTerm(innerHitBuilder.isSeqNoAndPrimaryTerm()); innerHitsContext.trackScores(innerHitBuilder.isTrackScores()); if (innerHitBuilder.getStoredFieldsContext() != null) { innerHitsContext.storedFieldsContext(innerHitBuilder.getStoredFieldsContext()); diff --git a/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java b/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java index 3e3a1e02a174b..da773efed580d 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java @@ -201,6 +201,9 @@ private static void parseSearchSource(final SearchSourceBuilder searchSourceBuil if (request.hasParam("version")) { searchSourceBuilder.version(request.paramAsBoolean("version", null)); } + if (request.hasParam("seq_no_primary_term")) { + searchSourceBuilder.seqNoAndPrimaryTerm(request.paramAsBoolean("seq_no_primary_term", null)); + } if (request.hasParam("timeout")) { searchSourceBuilder.timeout(request.paramAsTime("timeout", null)); } diff --git a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java index 590c58b1f6615..4b82e23e42061 100644 --- a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java +++ b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java @@ -107,6 +107,7 @@ final class DefaultSearchContext extends SearchContext { private ScrollContext scrollContext; private boolean explain; private boolean version = false; // by default, we don't return versions + private boolean seqAndPrimaryTerm = false; private StoredFieldsContext storedFields; private ScriptFieldsContext scriptFields; private FetchSourceContext fetchSourceContext; @@ -719,6 +720,16 @@ public void version(boolean version) { this.version = version; } + @Override + public boolean seqNoAndPrimaryTerm() { + return seqAndPrimaryTerm; + } + + @Override + public void seqNoAndPrimaryTerm(boolean seqNoAndPrimaryTerm) { + this.seqAndPrimaryTerm = seqNoAndPrimaryTerm; + } + @Override public int[] docIdsToLoad() { return docIdsToLoad; diff --git a/server/src/main/java/org/elasticsearch/search/SearchHit.java b/server/src/main/java/org/elasticsearch/search/SearchHit.java index 7fd68852ce284..ffe55b2825b33 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchHit.java +++ b/server/src/main/java/org/elasticsearch/search/SearchHit.java @@ -21,6 +21,7 @@ import org.apache.lucene.search.Explanation; import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.Version; import org.elasticsearch.action.OriginalIndices; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.ParseField; @@ -46,6 +47,7 @@ import org.elasticsearch.index.mapper.IgnoredFieldMapper; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.SourceFieldMapper; +import org.elasticsearch.index.seqno.SequenceNumbers; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.search.fetch.subphase.highlight.HighlightField; import org.elasticsearch.search.lookup.SourceLookup; @@ -91,6 +93,8 @@ public final class SearchHit implements Streamable, ToXContentObject, Iterable plugins) { registerFetchSubPhase(new ScriptFieldsFetchSubPhase()); registerFetchSubPhase(new FetchSourceSubPhase()); registerFetchSubPhase(new VersionFetchSubPhase()); + registerFetchSubPhase(new SeqNoPrimaryTermFetchSubPhase()); registerFetchSubPhase(new MatchedQueriesFetchSubPhase()); registerFetchSubPhase(new HighlightPhase(highlighters)); registerFetchSubPhase(new ScoreFetchSubPhase()); diff --git a/server/src/main/java/org/elasticsearch/search/SearchService.java b/server/src/main/java/org/elasticsearch/search/SearchService.java index 4fee684276288..ddec3637ed491 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchService.java +++ b/server/src/main/java/org/elasticsearch/search/SearchService.java @@ -901,6 +901,11 @@ private void parseSource(DefaultSearchContext context, SearchSourceBuilder sourc if (source.version() != null) { context.version(source.version()); } + + if (source.seqNoAndPrimaryTerm() != null) { + context.seqNoAndPrimaryTerm(source.seqNoAndPrimaryTerm()); + } + if (source.stats() != null) { context.groupStats(source.stats()); } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregationBuilder.java index debbacdc6196c..ccab85411054e 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregationBuilder.java @@ -19,6 +19,7 @@ package org.elasticsearch.search.aggregations.metrics; +import org.elasticsearch.Version; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.Strings; @@ -66,6 +67,7 @@ public class TopHitsAggregationBuilder extends AbstractAggregationBuilder> sorts = null; private HighlightBuilder highlightBuilder; @@ -85,6 +87,7 @@ protected TopHitsAggregationBuilder(TopHitsAggregationBuilder clone, this.size = clone.size; this.explain = clone.explain; this.version = clone.version; + this.seqNoAndPrimaryTerm = clone.seqNoAndPrimaryTerm; this.trackScores = clone.trackScores; this.sorts = clone.sorts == null ? null : new ArrayList<>(clone.sorts); this.highlightBuilder = clone.highlightBuilder == null ? null : @@ -137,6 +140,9 @@ public TopHitsAggregationBuilder(StreamInput in) throws IOException { } trackScores = in.readBoolean(); version = in.readBoolean(); + if (in.getVersion().onOrAfter(Version.V_7_0_0)) { + seqNoAndPrimaryTerm = in.readBoolean(); + } } @Override @@ -173,6 +179,9 @@ protected void doWriteTo(StreamOutput out) throws IOException { } out.writeBoolean(trackScores); out.writeBoolean(version); + if (out.getVersion().onOrAfter(Version.V_7_0_0)) { + out.writeBoolean(seqNoAndPrimaryTerm); + } } /** @@ -526,6 +535,23 @@ public boolean version() { return version; } + /** + * Should each {@link org.elasticsearch.search.SearchHit} be returned with the + * sequence number and primary term of the last modification of the document. + */ + public TopHitsAggregationBuilder seqNoAndPrimaryTerm(Boolean seqNoAndPrimaryTerm) { + this.seqNoAndPrimaryTerm = seqNoAndPrimaryTerm; + return this; + } + + /** + * Indicates whether {@link org.elasticsearch.search.SearchHit}s should be returned with the + * sequence number and primary term of the last modification of the document. + */ + public Boolean seqNoAndPrimaryTerm() { + return seqNoAndPrimaryTerm; + } + /** * Applies when sorting, and controls if scores will be tracked as well. * Defaults to {@code false}. @@ -579,8 +605,9 @@ protected TopHitsAggregatorFactory doBuild(SearchContext context, AggregatorFact } else { optionalSort = SortBuilder.buildSort(sorts, context.getQueryShardContext()); } - return new TopHitsAggregatorFactory(name, from, size, explain, version, trackScores, optionalSort, highlightBuilder, - storedFieldsContext, docValueFields, fields, fetchSourceContext, context, parent, subfactoriesBuilder, metaData); + return new TopHitsAggregatorFactory(name, from, size, explain, version, seqNoAndPrimaryTerm, trackScores, optionalSort, + highlightBuilder, storedFieldsContext, docValueFields, fields, fetchSourceContext, context, parent, subfactoriesBuilder, + metaData); } @Override @@ -589,6 +616,7 @@ protected XContentBuilder internalXContent(XContentBuilder builder, Params param builder.field(SearchSourceBuilder.FROM_FIELD.getPreferredName(), from); builder.field(SearchSourceBuilder.SIZE_FIELD.getPreferredName(), size); builder.field(SearchSourceBuilder.VERSION_FIELD.getPreferredName(), version); + builder.field(SearchSourceBuilder.SEQ_NO_PRIMARY_TERM_FIELD.getPreferredName(), seqNoAndPrimaryTerm); builder.field(SearchSourceBuilder.EXPLAIN_FIELD.getPreferredName(), explain); if (fetchSourceContext != null) { builder.field(SearchSourceBuilder._SOURCE_FIELD.getPreferredName(), fetchSourceContext); @@ -745,7 +773,7 @@ public static TopHitsAggregationBuilder parse(String aggregationName, XContentPa @Override protected int doHashCode() { return Objects.hash(explain, fetchSourceContext, docValueFields, storedFieldsContext, from, highlightBuilder, - scriptFields, size, sorts, trackScores, version); + scriptFields, size, sorts, trackScores, version, seqNoAndPrimaryTerm); } @Override @@ -761,7 +789,8 @@ protected boolean doEquals(Object obj) { && Objects.equals(size, other.size) && Objects.equals(sorts, other.sorts) && Objects.equals(trackScores, other.trackScores) - && Objects.equals(version, other.version); + && Objects.equals(version, other.version) + && Objects.equals(seqNoAndPrimaryTerm, other.seqNoAndPrimaryTerm); } @Override diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregatorFactory.java index 6086942955122..7edaccb66d4bd 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregatorFactory.java @@ -44,6 +44,7 @@ class TopHitsAggregatorFactory extends AggregatorFactory sort; private final HighlightBuilder highlightBuilder; @@ -52,8 +53,8 @@ class TopHitsAggregatorFactory extends AggregatorFactory scriptFields; private final FetchSourceContext fetchSourceContext; - TopHitsAggregatorFactory(String name, int from, int size, boolean explain, boolean version, boolean trackScores, - Optional sort, HighlightBuilder highlightBuilder, StoredFieldsContext storedFieldsContext, + TopHitsAggregatorFactory(String name, int from, int size, boolean explain, boolean version, boolean seqNoAndPrimaryTerm, + boolean trackScores, Optional sort, HighlightBuilder highlightBuilder, StoredFieldsContext storedFieldsContext, List docValueFields, List scriptFields, FetchSourceContext fetchSourceContext, SearchContext context, AggregatorFactory parent, AggregatorFactories.Builder subFactories, Map metaData) throws IOException { @@ -62,6 +63,7 @@ class TopHitsAggregatorFactory extends AggregatorFactory> sorts; private boolean trackScores = false; @@ -247,6 +250,11 @@ public SearchSourceBuilder(StreamInput in) throws IOException { timeout = in.readOptionalTimeValue(); trackScores = in.readBoolean(); version = in.readOptionalBoolean(); + if (in.getVersion().onOrAfter(Version.V_7_0_0)) { + seqNoAndPrimaryTerm = in.readOptionalBoolean(); + } else { + seqNoAndPrimaryTerm = null; + } extBuilders = in.readNamedWriteableList(SearchExtBuilder.class); profile = in.readBoolean(); searchAfterBuilder = in.readOptionalWriteable(SearchAfterBuilder::new); @@ -310,6 +318,9 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalTimeValue(timeout); out.writeBoolean(trackScores); out.writeOptionalBoolean(version); + if (out.getVersion().onOrAfter(Version.V_7_0_0)) { + out.writeOptionalBoolean(seqNoAndPrimaryTerm); + } out.writeNamedWriteableList(extBuilders); out.writeBoolean(profile); out.writeOptionalWriteable(searchAfterBuilder); @@ -441,6 +452,23 @@ public Boolean version() { return version; } + /** + * Should each {@link org.elasticsearch.search.SearchHit} be returned with the + * sequence number and primary term of the last modification of the document. + */ + public SearchSourceBuilder seqNoAndPrimaryTerm(Boolean seqNoAndPrimaryTerm) { + this.seqNoAndPrimaryTerm = seqNoAndPrimaryTerm; + return this; + } + + /** + * Indicates whether {@link org.elasticsearch.search.SearchHit}s should be returned with the + * sequence number and primary term of the last modification of the document. + */ + public Boolean seqNoAndPrimaryTerm() { + return seqNoAndPrimaryTerm; + } + /** * An optional timeout to control how long search is allowed to take. */ @@ -999,6 +1027,7 @@ private SearchSourceBuilder shallowCopy(QueryBuilder queryBuilder, QueryBuilder rewrittenBuilder.trackScores = trackScores; rewrittenBuilder.trackTotalHitsUpTo = trackTotalHitsUpTo; rewrittenBuilder.version = version; + rewrittenBuilder.seqNoAndPrimaryTerm = seqNoAndPrimaryTerm; rewrittenBuilder.collapse = collapse; return rewrittenBuilder; } @@ -1038,6 +1067,8 @@ public void parseXContent(XContentParser parser, boolean checkTrailingTokens) th minScore = parser.floatValue(); } else if (VERSION_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { version = parser.booleanValue(); + } else if (SEQ_NO_PRIMARY_TERM_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { + seqNoAndPrimaryTerm = parser.booleanValue(); } else if (EXPLAIN_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { explain = parser.booleanValue(); } else if (TRACK_SCORES_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { @@ -1205,6 +1236,10 @@ public XContentBuilder innerToXContent(XContentBuilder builder, Params params) t builder.field(VERSION_FIELD.getPreferredName(), version); } + if (seqNoAndPrimaryTerm != null) { + builder.field(SEQ_NO_PRIMARY_TERM_FIELD.getPreferredName(), version); + } + if (explain != null) { builder.field(EXPLAIN_FIELD.getPreferredName(), explain); } @@ -1523,7 +1558,7 @@ public int hashCode() { return Objects.hash(aggregations, explain, fetchSourceContext, docValueFields, storedFieldsContext, from, highlightBuilder, indexBoosts, minScore, postQueryBuilder, queryBuilder, rescoreBuilders, scriptFields, size, sorts, searchAfterBuilder, sliceBuilder, stats, suggestBuilder, terminateAfter, timeout, trackScores, version, - profile, extBuilders, collapse, trackTotalHitsUpTo); + seqNoAndPrimaryTerm, profile, extBuilders, collapse, trackTotalHitsUpTo); } @Override @@ -1558,6 +1593,7 @@ public boolean equals(Object obj) { && Objects.equals(timeout, other.timeout) && Objects.equals(trackScores, other.trackScores) && Objects.equals(version, other.version) + && Objects.equals(seqNoAndPrimaryTerm, other.seqNoAndPrimaryTerm) && Objects.equals(profile, other.profile) && Objects.equals(extBuilders, other.extBuilders) && Objects.equals(collapse, other.collapse) diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/SeqNoPrimaryTermFetchSubPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/SeqNoPrimaryTermFetchSubPhase.java new file mode 100644 index 0000000000000..0ee37f84fe747 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/SeqNoPrimaryTermFetchSubPhase.java @@ -0,0 +1,67 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.search.fetch.subphase; + +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.NumericDocValues; +import org.apache.lucene.index.ReaderUtil; +import org.elasticsearch.index.mapper.SeqNoFieldMapper; +import org.elasticsearch.index.seqno.SequenceNumbers; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.fetch.FetchSubPhase; +import org.elasticsearch.search.internal.SearchContext; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Comparator; + +public final class SeqNoPrimaryTermFetchSubPhase implements FetchSubPhase { + @Override + public void hitsExecute(SearchContext context, SearchHit[] hits) throws IOException { + if (context.seqNoAndPrimaryTerm() == false || + (context.storedFieldsContext() != null && context.storedFieldsContext().fetchFields() == false)) { + return; + } + + hits = hits.clone(); // don't modify the incoming hits + Arrays.sort(hits, Comparator.comparingInt(SearchHit::docId)); + + int lastReaderId = -1; + NumericDocValues seqNoField = null; + NumericDocValues primaryTermField = null; + for (SearchHit hit : hits) { + int readerId = ReaderUtil.subIndex(hit.docId(), context.searcher().getIndexReader().leaves()); + LeafReaderContext subReaderContext = context.searcher().getIndexReader().leaves().get(readerId); + if (lastReaderId != readerId) { + seqNoField = subReaderContext.reader().getNumericDocValues(SeqNoFieldMapper.NAME); + primaryTermField = subReaderContext.reader().getNumericDocValues(SeqNoFieldMapper.PRIMARY_TERM_NAME); + lastReaderId = readerId; + } + int docId = hit.docId() - subReaderContext.docBase; + long seqNo = SequenceNumbers.UNASSIGNED_SEQ_NO; + long primaryTerm = SequenceNumbers.UNASSIGNED_PRIMARY_TERM; + if (seqNoField != null && seqNoField.advanceExact(docId)) { + seqNo = seqNoField.longValue(); + primaryTerm = primaryTermField.longValue(); + } + hit.setSeqNo(seqNo); + hit.setPrimaryTerm(primaryTerm); + } + } +} diff --git a/server/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java b/server/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java index 3a7fb9f823f3a..ecde28719cec6 100644 --- a/server/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java +++ b/server/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java @@ -422,6 +422,16 @@ public void version(boolean version) { in.version(version); } + @Override + public boolean seqNoAndPrimaryTerm() { + return in.seqNoAndPrimaryTerm(); + } + + @Override + public void seqNoAndPrimaryTerm(boolean seqNoAndPrimaryTerm) { + in.seqNoAndPrimaryTerm(seqNoAndPrimaryTerm); + } + @Override public int[] docIdsToLoad() { return in.docIdsToLoad(); diff --git a/server/src/main/java/org/elasticsearch/search/internal/SearchContext.java b/server/src/main/java/org/elasticsearch/search/internal/SearchContext.java index 768143dd8fb0b..bd6d9c501c8d1 100644 --- a/server/src/main/java/org/elasticsearch/search/internal/SearchContext.java +++ b/server/src/main/java/org/elasticsearch/search/internal/SearchContext.java @@ -38,7 +38,6 @@ import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.ObjectMapper; -import org.elasticsearch.search.collapse.CollapseContext; import org.elasticsearch.index.query.ParsedQuery; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.shard.IndexShard; @@ -46,6 +45,7 @@ import org.elasticsearch.search.SearchExtBuilder; import org.elasticsearch.search.SearchShardTarget; import org.elasticsearch.search.aggregations.SearchContextAggregations; +import org.elasticsearch.search.collapse.CollapseContext; import org.elasticsearch.search.dfs.DfsSearchResult; import org.elasticsearch.search.fetch.FetchPhase; import org.elasticsearch.search.fetch.FetchSearchResult; @@ -309,6 +309,12 @@ public InnerHitsContext innerHits() { public abstract void version(boolean version); + /** indicates whether the sequence number and primary term of the last modification to each hit should be returned */ + public abstract boolean seqNoAndPrimaryTerm(); + + /** controls whether the sequence number and primary term of the last modification to each hit should be returned */ + public abstract void seqNoAndPrimaryTerm(boolean seqNoAndPrimaryTerm); + public abstract int[] docIdsToLoad(); public abstract int docIdsToLoadFrom(); diff --git a/server/src/main/java/org/elasticsearch/search/internal/SubSearchContext.java b/server/src/main/java/org/elasticsearch/search/internal/SubSearchContext.java index 8c8137f5e4345..fb4d233f10ee8 100644 --- a/server/src/main/java/org/elasticsearch/search/internal/SubSearchContext.java +++ b/server/src/main/java/org/elasticsearch/search/internal/SubSearchContext.java @@ -65,6 +65,7 @@ public class SubSearchContext extends FilteredSearchContext { private boolean explain; private boolean trackScores; private boolean version; + private boolean seqNoAndPrimaryTerm; public SubSearchContext(SearchContext context) { super(context); @@ -294,6 +295,16 @@ public void version(boolean version) { this.version = version; } + @Override + public boolean seqNoAndPrimaryTerm() { + return seqNoAndPrimaryTerm; + } + + @Override + public void seqNoAndPrimaryTerm(boolean seqNoAndPrimaryTerm) { + this.seqNoAndPrimaryTerm = seqNoAndPrimaryTerm; + } + @Override public int[] docIdsToLoad() { return docIdsToLoad; diff --git a/server/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java b/server/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java index 9378a2cdd86bb..328950e4f3569 100644 --- a/server/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java @@ -241,6 +241,7 @@ public void run() throws IOException { public void testExpandRequestOptions() throws IOException { MockSearchPhaseContext mockSearchPhaseContext = new MockSearchPhaseContext(1); boolean version = randomBoolean(); + final boolean seqNoAndTerm = randomBoolean(); mockSearchPhaseContext.searchTransport = new SearchTransportService(null, null) { @Override @@ -249,13 +250,14 @@ void sendExecuteMultiSearch(MultiSearchRequest request, SearchTask task, ActionL assertTrue(request.requests().stream().allMatch((r) -> "foo".equals(r.preference()))); assertTrue(request.requests().stream().allMatch((r) -> "baz".equals(r.routing()))); assertTrue(request.requests().stream().allMatch((r) -> version == r.source().version())); + assertTrue(request.requests().stream().allMatch((r) -> seqNoAndTerm == r.source().seqNoAndPrimaryTerm())); assertTrue(request.requests().stream().allMatch((r) -> postFilter.equals(r.source().postFilter()))); } }; mockSearchPhaseContext.getRequest().source(new SearchSourceBuilder() .collapse( new CollapseBuilder("someField") - .setInnerHits(new InnerHitBuilder().setName("foobarbaz").setVersion(version)) + .setInnerHits(new InnerHitBuilder().setName("foobarbaz").setVersion(version).setSeqNoAndPrimaryTerm(seqNoAndTerm)) ) .postFilter(QueryBuilders.existsQuery("foo"))) .preference("foobar") diff --git a/server/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java index 5c76c77b5c888..c7716025268d0 100644 --- a/server/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java @@ -147,6 +147,7 @@ public static InnerHitBuilder randomInnerHits() { innerHits.setSize(randomIntBetween(0, 32)); innerHits.setExplain(randomBoolean()); innerHits.setVersion(randomBoolean()); + innerHits.setSeqNoAndPrimaryTerm(randomBoolean()); innerHits.setTrackScores(randomBoolean()); if (randomBoolean()) { innerHits.setStoredFieldNames(randomListStuff(16, () -> randomAlphaOfLengthBetween(1, 16))); @@ -189,6 +190,7 @@ static InnerHitBuilder mutate(InnerHitBuilder original) throws IOException { modifiers.add(() -> copy.setSize(randomValueOtherThan(copy.getSize(), () -> randomIntBetween(0, 128)))); modifiers.add(() -> copy.setExplain(!copy.isExplain())); modifiers.add(() -> copy.setVersion(!copy.isVersion())); + modifiers.add(() -> copy.setSeqNoAndPrimaryTerm(!copy.isSeqNoAndPrimaryTerm())); modifiers.add(() -> copy.setTrackScores(!copy.isTrackScores())); modifiers.add(() -> copy.setName(randomValueOtherThan(copy.getName(), () -> randomAlphaOfLengthBetween(1, 16)))); modifiers.add(() -> { diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/TopHitsIT.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/TopHitsIT.java index ce8d9c2da834f..4a5982b9dacfa 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/TopHitsIT.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/TopHitsIT.java @@ -31,6 +31,7 @@ import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.query.MatchAllQueryBuilder; import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.index.seqno.SequenceNumbers; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.script.MockScriptEngine; import org.elasticsearch.script.MockScriptPlugin; @@ -83,6 +84,7 @@ import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.lessThanOrEqualTo; import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.notNullValue; @@ -578,6 +580,7 @@ public void testFieldCollapsing() throws Exception { } public void testFetchFeatures() { + final boolean seqNoAndTerm = randomBoolean(); SearchResponse response = client().prepareSearch("idx") .setQuery(matchQuery("text", "text").queryName("test")) .addAggregation(terms("terms") @@ -593,6 +596,7 @@ public void testFetchFeatures() { new Script(ScriptType.INLINE, MockScriptEngine.NAME, "5", Collections.emptyMap())) .fetchSource("text", null) .version(true) + .seqNoAndPrimaryTerm(seqNoAndTerm) ) ) .get(); @@ -620,6 +624,14 @@ public void testFetchFeatures() { long version = hit.getVersion(); assertThat(version, equalTo(1L)); + if (seqNoAndTerm) { + assertThat(hit.getSeqNo(), greaterThanOrEqualTo(0L)); + assertThat(hit.getPrimaryTerm(), greaterThanOrEqualTo(1L)); + } else { + assertThat(hit.getSeqNo(), equalTo(SequenceNumbers.UNASSIGNED_SEQ_NO)); + assertThat(hit.getPrimaryTerm(), equalTo(SequenceNumbers.UNASSIGNED_PRIMARY_TERM)); + } + assertThat(hit.getMatchedQueries()[0], equalTo("test")); DocumentField field = hit.field("field1"); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/TopHitsTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/TopHitsTests.java index caf1d4ef4ee47..6ec1ea1cad301 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/TopHitsTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/TopHitsTests.java @@ -54,6 +54,9 @@ protected final TopHitsAggregationBuilder createTestAggregatorBuilder() { if (randomBoolean()) { factory.version(randomBoolean()); } + if (randomBoolean()) { + factory.seqNoAndPrimaryTerm(randomBoolean()); + } if (randomBoolean()) { factory.trackScores(randomBoolean()); } diff --git a/test/framework/src/main/java/org/elasticsearch/search/RandomSearchRequestGenerator.java b/test/framework/src/main/java/org/elasticsearch/search/RandomSearchRequestGenerator.java index 5d96cd37b054d..6ec2732aaf915 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/RandomSearchRequestGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/search/RandomSearchRequestGenerator.java @@ -145,6 +145,9 @@ public static SearchSourceBuilder randomSearchSourceBuilder( if (randomBoolean()) { builder.version(randomBoolean()); } + if (randomBoolean()) { + builder.seqNoAndPrimaryTerm(randomBoolean()); + } if (randomBoolean()) { builder.trackScores(randomBoolean()); } diff --git a/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java b/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java index 18edb5ec3790a..6a17f65790368 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java +++ b/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java @@ -504,6 +504,16 @@ public boolean version() { public void version(boolean version) { } + @Override + public boolean seqNoAndPrimaryTerm() { + return false; + } + + @Override + public void seqNoAndPrimaryTerm(boolean seqNoAndPrimaryTerm) { + + } + @Override public int[] docIdsToLoad() { return new int[0]; From 75df26557f63cd742068d1d4af9a61e8c703b35a Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Thu, 17 Jan 2019 14:02:40 +0100 Subject: [PATCH 02/29] fix InnerHitBuilder.equals and TopHitsAggregationBuilder parsing --- .../java/org/elasticsearch/index/query/InnerHitBuilder.java | 1 + .../search/aggregations/metrics/TopHitsAggregationBuilder.java | 2 ++ 2 files changed, 3 insertions(+) diff --git a/server/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java b/server/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java index cc68ccd3f3e32..90275800d3769 100644 --- a/server/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java @@ -513,6 +513,7 @@ public boolean equals(Object o) { Objects.equals(size, that.size) && Objects.equals(explain, that.explain) && Objects.equals(version, that.version) && + Objects.equals(seqNoAndPrimaryTerm, that.seqNoAndPrimaryTerm) && Objects.equals(trackScores, that.trackScores) && Objects.equals(storedFieldsContext, that.storedFieldsContext) && Objects.equals(docValueFields, that.docValueFields) && diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregationBuilder.java index ccab85411054e..ba51099d6bc00 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregationBuilder.java @@ -674,6 +674,8 @@ public static TopHitsAggregationBuilder parse(String aggregationName, XContentPa factory.size(parser.intValue()); } else if (SearchSourceBuilder.VERSION_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { factory.version(parser.booleanValue()); + } else if (SearchSourceBuilder.SEQ_NO_PRIMARY_TERM_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { + factory.seqNoAndPrimaryTerm(parser.booleanValue()); } else if (SearchSourceBuilder.EXPLAIN_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { factory.explain(parser.booleanValue()); } else if (SearchSourceBuilder.TRACK_SCORES_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { From da27bf72630647dd686d0fc2603ca642b6590d53 Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Thu, 17 Jan 2019 15:15:15 +0100 Subject: [PATCH 03/29] fix to xcontent --- .../org/elasticsearch/search/builder/SearchSourceBuilder.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java b/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java index 4972e065f13aa..d7867af9ea139 100644 --- a/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java @@ -1237,7 +1237,7 @@ public XContentBuilder innerToXContent(XContentBuilder builder, Params params) t } if (seqNoAndPrimaryTerm != null) { - builder.field(SEQ_NO_PRIMARY_TERM_FIELD.getPreferredName(), version); + builder.field(SEQ_NO_PRIMARY_TERM_FIELD.getPreferredName(), seqNoAndPrimaryTerm); } if (explain != null) { From 51f1117c55b181d02ad09d75a134a0a41a0e197d Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Thu, 17 Jan 2019 15:31:24 +0100 Subject: [PATCH 04/29] fix seq no loading --- .../search/fetch/subphase/SeqNoPrimaryTermFetchSubPhase.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/SeqNoPrimaryTermFetchSubPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/SeqNoPrimaryTermFetchSubPhase.java index 0ee37f84fe747..d99ee265c894b 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/SeqNoPrimaryTermFetchSubPhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/SeqNoPrimaryTermFetchSubPhase.java @@ -57,6 +57,8 @@ public void hitsExecute(SearchContext context, SearchHit[] hits) throws IOExcept long seqNo = SequenceNumbers.UNASSIGNED_SEQ_NO; long primaryTerm = SequenceNumbers.UNASSIGNED_PRIMARY_TERM; if (seqNoField != null && seqNoField.advanceExact(docId)) { + boolean found = primaryTermField.advanceExact(docId); + assert found: "found seq no for " + docId + " but not a primary term"; seqNo = seqNoField.longValue(); primaryTerm = primaryTermField.longValue(); } From a4208dda7a57cca99dd14340303b53e8be76d622 Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Thu, 17 Jan 2019 16:56:49 +0100 Subject: [PATCH 05/29] doh --- server/src/main/java/org/elasticsearch/search/SearchHit.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/search/SearchHit.java b/server/src/main/java/org/elasticsearch/search/SearchHit.java index ffe55b2825b33..ecfb31f5fc4f3 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchHit.java +++ b/server/src/main/java/org/elasticsearch/search/SearchHit.java @@ -177,7 +177,7 @@ public void setSeqNo(long seqNo) { } public void setPrimaryTerm(long primaryTerm) { - this.seqNo = seqNo; + this.primaryTerm = primaryTerm; } /** returns the sequence number of the last modification to the document */ From 9611838e105658b8004f02d787cac36f3d1e3531 Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Thu, 17 Jan 2019 19:58:42 +0100 Subject: [PATCH 06/29] and rest tests and fix --- .../join/query/HasChildQueryBuilderTests.java | 1 + .../resources/rest-api-spec/api/search.json | 4 + .../200_top_hits_metric.yml | 36 +++++++++ .../test/search/110_field_collapsing.yml | 60 +++++++++++++++ .../test/search/300_sequence_numbers.yml | 74 +++++++++++++++++++ 5 files changed, 175 insertions(+) create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/search/300_sequence_numbers.yml diff --git a/modules/parent-join/src/test/java/org/elasticsearch/join/query/HasChildQueryBuilderTests.java b/modules/parent-join/src/test/java/org/elasticsearch/join/query/HasChildQueryBuilderTests.java index 6e4e79d16e5a5..eea01d61386de 100644 --- a/modules/parent-join/src/test/java/org/elasticsearch/join/query/HasChildQueryBuilderTests.java +++ b/modules/parent-join/src/test/java/org/elasticsearch/join/query/HasChildQueryBuilderTests.java @@ -252,6 +252,7 @@ public void testFromJson() throws IOException { " \"from\" : 0,\n" + " \"size\" : 100,\n" + " \"version\" : false,\n" + + " \"seq_no_primary_term\" : false,\n" + " \"explain\" : false,\n" + " \"track_scores\" : false,\n" + " \"sort\" : [ {\n" + diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/search.json b/rest-api-spec/src/main/resources/rest-api-spec/api/search.json index 5834ca623a99b..9ac02b1214a2f 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/search.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/search.json @@ -164,6 +164,10 @@ "type" : "boolean", "description" : "Specify whether to return document version as part of a hit" }, + "seq_no_primary_term": { + "type" : "boolean", + "description" : "Specify whether to return sequence number and primary term of the last modification of each hit" + }, "request_cache": { "type" : "boolean", "description" : "Specify if request cache should be used for this request or not, defaults to index level setting" diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/200_top_hits_metric.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/200_top_hits_metric.yml index 267a9e85d1d5d..25d966d87b38e 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/200_top_hits_metric.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/200_top_hits_metric.yml @@ -81,3 +81,39 @@ - match: { aggregations.to-users.users.hits.hits.2._index: my-index } - match: { aggregations.to-users.users.hits.hits.2._nested.field: users } - match: { aggregations.to-users.users.hits.hits.2._nested.offset: 1 } + + +--- +"top_hits aggregation with sequence numbers": + - skip: + version: " - 6.99.99" + reason: support was added in 7.0 + + - do: + search: + rest_total_hits_as_int: true + body: + aggs: + to-users: + nested: + path: users + aggs: + users: + top_hits: + sort: "users.last.keyword" + seq_no_primary_term: true + + - match: { hits.total: 2 } + - length: { aggregations.to-users.users.hits.hits: 3 } + - match: { aggregations.to-users.users.hits.hits.0._id: "2" } + - match: { aggregations.to-users.users.hits.hits.0._index: my-index } + - gte: { aggregations.to-users.users.hits.hits.0._seq_no: 0 } + - gte: { aggregations.to-users.users.hits.hits.0._primary_term: 1 } + - match: { aggregations.to-users.users.hits.hits.1._id: "1" } + - match: { aggregations.to-users.users.hits.hits.1._index: my-index } + - gte: { aggregations.to-users.users.hits.hits.1._seq_no: 0 } + - gte: { aggregations.to-users.users.hits.hits.1._primary_term: 1 } + - match: { aggregations.to-users.users.hits.hits.2._id: "1" } + - match: { aggregations.to-users.users.hits.hits.2._index: my-index } + - gte: { aggregations.to-users.users.hits.hits.2._seq_no: 0 } + - gte: { aggregations.to-users.users.hits.hits.2._primary_term: 1 } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/110_field_collapsing.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/110_field_collapsing.yml index 7ba1d9b62fbc7..9bc65f9dcecdd 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/110_field_collapsing.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/110_field_collapsing.yml @@ -405,3 +405,63 @@ setup: - match: { hits.hits.1.inner_hits.sub_hits.hits.total: 3} - match: { hits.hits.2.fields.group_alias: [25] } - match: { hits.hits.2.inner_hits.sub_hits.hits.total: 2} + +--- +"field collapsing, inner_hits and seq_no": + + - skip: + version: " - 6.99.0" + reason: "sequence numbers introduced in 7.0.0" + + - do: + search: + rest_total_hits_as_int: true + index: test + body: + collapse: { field: numeric_group, inner_hits: { name: sub_hits, version: true, size: 2, sort: [{ sort: asc }] } } + sort: [{ sort: desc }] + seq_no_primary_term: true + + - match: { hits.total: 6 } + - length: { hits.hits: 3 } + - match: { hits.hits.0._index: test } + - match: { hits.hits.0.fields.numeric_group: [3] } + - match: { hits.hits.0.sort: [36] } + - match: { hits.hits.0._id: "6" } + - gte: { hits.hits.0._seq_no: 0 } + - gte: { hits.hits.0._primary_term: 1 } + - match: { hits.hits.0.inner_hits.sub_hits.hits.total: 1 } + - length: { hits.hits.0.inner_hits.sub_hits.hits.hits: 1 } + - match: { hits.hits.0.inner_hits.sub_hits.hits.hits.0._id: "6" } + - gte: { hits.hits.0.inner_hits.sub_hits.hits.hits.0._seq_no: 0 } + - gte: { hits.hits.0.inner_hits.sub_hits.hits.hits.0._primary_term: 1 } + - match: { hits.hits.1._index: test } + - match: { hits.hits.1.fields.numeric_group: [1] } + - match: { hits.hits.1.sort: [24] } + - match: { hits.hits.1._id: "3" } + - match: { hits.hits.1._version: 33 } + - gte: { hits.hits.1._seq_no: 0 } + - gte: { hits.hits.1._primary_term: 1 } + - match: { hits.hits.1.inner_hits.sub_hits.hits.total: 3 } + - length: { hits.hits.1.inner_hits.sub_hits.hits.hits: 2 } + - match: { hits.hits.1.inner_hits.sub_hits.hits.hits.0._id: "2" } + - gte: { hits.hits.1.inner_hits.sub_hits.hits.hits.0._seq_no: 0 } + - gte: { hits.hits.1.inner_hits.sub_hits.hits.hits.0._primary_term: 1 } + - match: { hits.hits.1.inner_hits.sub_hits.hits.hits.1._id: "1" } + - gte: { hits.hits.1.inner_hits.sub_hits.hits.hits.1._seq_no: 0 } + - gte: { hits.hits.1.inner_hits.sub_hits.hits.hits.1._primary_term: 1 } + - match: { hits.hits.2._index: test } + - match: { hits.hits.2._type: test } + - match: { hits.hits.2.fields.numeric_group: [25] } + - match: { hits.hits.2.sort: [10] } + - match: { hits.hits.2._id: "4" } + - gte: { hits.hits.2._seq_no: 0 } + - gte: { hits.hits.2._primary_term: 1 } + - match: { hits.hits.2.inner_hits.sub_hits.hits.total: 2 } + - length: { hits.hits.2.inner_hits.sub_hits.hits.hits: 2 } + - match: { hits.hits.2.inner_hits.sub_hits.hits.hits.0._id: "5" } + - gte: { hits.hits.2.inner_hits.sub_hits.hits.hits.0._seq_no: 0 } + - gte: { hits.hits.2.inner_hits.sub_hits.hits.hits.0._primary_term: 1 } + - match: { hits.hits.2.inner_hits.sub_hits.hits.hits.1._id: "4" } + - gte: { hits.hits.2.inner_hits.sub_hits.hits.hits.1._seq_no: 0 } + - gte: { hits.hits.2.inner_hits.sub_hits.hits.hits.1._primary_term: 1 } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/300_sequence_numbers.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/300_sequence_numbers.yml new file mode 100644 index 0000000000000..2c60215cbd47c --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/300_sequence_numbers.yml @@ -0,0 +1,74 @@ +setup: + - do: + indices.create: + index: test_1 + + - do: + index: + index: test_1 + type: test + id: 1 + body: { foo: foo } +## we index again in order to make the seq# 1 (so we can check for the field existence with is_false) + - do: + index: + index: test_1 + type: test + id: 1 + body: { foo: bar } + + - do: + indices.refresh: + index: [test_1] + +--- +"sequence numbers are returned if requested from body": + - skip: + version: " - 6.99.99" + reason: sequence numbers were added in 7.0.0 + + - do: + search: + index: _all + body: + query: + match: + foo: bar + seq_no_primary_term: true + + - match: {hits.total.value: 1} + - match: {hits.hits.0._seq_no: 0} + - gte: {hits.hits.0._primary_term: 1} + +--- +"sequence numbers are returned if requested from url": + - skip: + version: " - 6.99.99" + reason: sequence numbers were added in 7.0.0 + + - do: + search: + index: _all + body: + query: + match: + foo: bar + seq_no_primary_term: true + + - match: {hits.total.value: 1} + - match: {hits.hits.0._seq_no: 1} + - gte: {hits.hits.0._primary_term: 1} + +--- +"sequence numbers are not returned if not requested": + - do: + search: + index: _all + body: + query: + match: + foo: bar + + - match: {hits.total.value: 1} + - is_false: hits.hits.0._seq_no + - is_false: hits.hits.0._primary_term From d161f08df2701999a0ac5b4f8dfce9934d3e91fb Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Thu, 17 Jan 2019 20:23:42 +0100 Subject: [PATCH 07/29] fix indenting --- .../rest-api-spec/test/search/300_sequence_numbers.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/300_sequence_numbers.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/300_sequence_numbers.yml index 2c60215cbd47c..145e77c86923d 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/300_sequence_numbers.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/300_sequence_numbers.yml @@ -8,7 +8,8 @@ setup: index: test_1 type: test id: 1 - body: { foo: foo } + body: { foo: foo } + ## we index again in order to make the seq# 1 (so we can check for the field existence with is_false) - do: index: From 87f4ef835e3e0da3f37c7dcf46668f999cc8092d Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Thu, 17 Jan 2019 21:21:09 +0100 Subject: [PATCH 08/29] fix rest tests --- .../200_top_hits_metric.yml | 51 +++++++++++++++++++ .../test/search/110_field_collapsing.yml | 5 +- .../test/search/300_sequence_numbers.yml | 2 +- 3 files changed, 55 insertions(+), 3 deletions(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/200_top_hits_metric.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/200_top_hits_metric.yml index 25d966d87b38e..9a0b9657eb3d5 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/200_top_hits_metric.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/200_top_hits_metric.yml @@ -89,6 +89,57 @@ version: " - 6.99.99" reason: support was added in 7.0 + - do: + indices.create: + index: my-index + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + doc: + properties: + users: + type: nested + + - do: + index: + index: my-index + type: doc + id: 1 + refresh: true + body: | + { + "group" : "fans", + "users" : [ + { + "first" : "John", + "last" : "Smith" + }, + { + "first" : "Alice", + "last" : "White" + } + ] + } + + - do: + index: + index: my-index + type: doc + id: 2 + refresh: true + body: | + { + "group" : "fans", + "users" : [ + { + "first" : "Mark", + "last" : "Doe" + } + ] + } + - do: search: rest_total_hits_as_int: true diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/110_field_collapsing.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/110_field_collapsing.yml index 9bc65f9dcecdd..6406eaecadcfa 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/110_field_collapsing.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/110_field_collapsing.yml @@ -418,9 +418,10 @@ setup: rest_total_hits_as_int: true index: test body: - collapse: { field: numeric_group, inner_hits: { name: sub_hits, version: true, size: 2, sort: [{ sort: asc }] } } + collapse: { field: numeric_group, inner_hits: { + name: sub_hits, version: true, seq_no_primary_term: true, size: 2, sort: [{ sort: asc }] + } } sort: [{ sort: desc }] - seq_no_primary_term: true - match: { hits.total: 6 } - length: { hits.hits: 3 } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/300_sequence_numbers.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/300_sequence_numbers.yml index 145e77c86923d..5f30114df89f8 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/300_sequence_numbers.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/300_sequence_numbers.yml @@ -38,7 +38,7 @@ setup: seq_no_primary_term: true - match: {hits.total.value: 1} - - match: {hits.hits.0._seq_no: 0} + - match: {hits.hits.0._seq_no: 1} - gte: {hits.hits.0._primary_term: 1} --- From 5d20b1856367251306a9cc33dca34e3ebd6e3848 Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Thu, 17 Jan 2019 21:49:59 +0100 Subject: [PATCH 09/29] nested fields --- .../elasticsearch/join/query/InnerHitsIT.java | 22 +++++++- .../rest-api-spec/test/11_parent_child.yml | 55 +++++++++++++------ .../200_top_hits_metric.yml | 12 ++-- .../index/query/NestedQueryBuilder.java | 8 +++ .../SeqNoPrimaryTermFetchSubPhase.java | 5 +- .../index/query/InnerHitBuilderTests.java | 5 ++ .../index/query/NestedQueryBuilderTests.java | 16 +++--- 7 files changed, 90 insertions(+), 33 deletions(-) diff --git a/modules/parent-join/src/test/java/org/elasticsearch/join/query/InnerHitsIT.java b/modules/parent-join/src/test/java/org/elasticsearch/join/query/InnerHitsIT.java index 7a8d2cd9dbc21..89929985ea594 100644 --- a/modules/parent-join/src/test/java/org/elasticsearch/join/query/InnerHitsIT.java +++ b/modules/parent-join/src/test/java/org/elasticsearch/join/query/InnerHitsIT.java @@ -56,6 +56,8 @@ import static org.elasticsearch.index.query.QueryBuilders.matchQuery; import static org.elasticsearch.index.query.QueryBuilders.nestedQuery; import static org.elasticsearch.index.query.QueryBuilders.termQuery; +import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_PRIMARY_TERM; +import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_SEQ_NO; import static org.elasticsearch.join.query.JoinQueryBuilders.hasChildQuery; import static org.elasticsearch.join.query.JoinQueryBuilders.hasParentQuery; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; @@ -66,6 +68,7 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.hasId; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; @@ -133,9 +136,10 @@ public void testSimpleParentChild() throws Exception { assertThat(innerHits.getAt(1).getId(), equalTo("c2")); assertThat(innerHits.getAt(1).getType(), equalTo("doc")); + final boolean seqNoAndTerm = randomBoolean(); response = client().prepareSearch("articles") .setQuery(hasChildQuery("comment", matchQuery("message", "elephant"), ScoreMode.None) - .innerHit(new InnerHitBuilder())) + .innerHit(new InnerHitBuilder().setSeqNoAndPrimaryTerm(seqNoAndTerm))) .get(); assertNoFailures(response); assertHitCount(response, 1); @@ -152,6 +156,22 @@ public void testSimpleParentChild() throws Exception { assertThat(innerHits.getAt(2).getId(), equalTo("c6")); assertThat(innerHits.getAt(2).getType(), equalTo("doc")); + if (seqNoAndTerm) { + assertThat(innerHits.getAt(0).getPrimaryTerm(), equalTo(1L)); + assertThat(innerHits.getAt(1).getPrimaryTerm(), equalTo(1L)); + assertThat(innerHits.getAt(2).getPrimaryTerm(), equalTo(1L)); + assertThat(innerHits.getAt(0).getSeqNo(), greaterThanOrEqualTo(0L)); + assertThat(innerHits.getAt(1).getSeqNo(), greaterThanOrEqualTo(0L)); + assertThat(innerHits.getAt(2).getSeqNo(), greaterThanOrEqualTo(0L)); + } else { + assertThat(innerHits.getAt(0).getPrimaryTerm(), equalTo(UNASSIGNED_PRIMARY_TERM)); + assertThat(innerHits.getAt(1).getPrimaryTerm(), equalTo(UNASSIGNED_PRIMARY_TERM)); + assertThat(innerHits.getAt(2).getPrimaryTerm(), equalTo(UNASSIGNED_PRIMARY_TERM)); + assertThat(innerHits.getAt(0).getSeqNo(), equalTo(UNASSIGNED_SEQ_NO)); + assertThat(innerHits.getAt(1).getSeqNo(), equalTo(UNASSIGNED_SEQ_NO)); + assertThat(innerHits.getAt(2).getSeqNo(), equalTo(UNASSIGNED_SEQ_NO)); + } + response = client().prepareSearch("articles") .setQuery( hasChildQuery("comment", matchQuery("message", "fox"), ScoreMode.None).innerHit( diff --git a/modules/parent-join/src/test/resources/rest-api-spec/test/11_parent_child.yml b/modules/parent-join/src/test/resources/rest-api-spec/test/11_parent_child.yml index 7273ac6a95d5e..db893580ee5bf 100644 --- a/modules/parent-join/src/test/resources/rest-api-spec/test/11_parent_child.yml +++ b/modules/parent-join/src/test/resources/rest-api-spec/test/11_parent_child.yml @@ -11,26 +11,26 @@ setup: relations: parent: child ---- -"Parent/child inner hits": - - do: - index: - index: test - type: doc - id: 1 - body: {"foo": "bar", "join_field": {"name" : "parent"} } + - do: + index: + index: test + type: doc + id: 1 + body: {"foo": "bar", "join_field": {"name" : "parent"} } - - do: - index: - index: test - type: doc - id: 2 - routing: 1 - body: {"bar": "baz", "join_field": { "name" : "child", "parent": "1"} } + - do: + index: + index: test + type: doc + id: 2 + routing: 1 + body: {"bar": "baz", "join_field": { "name" : "child", "parent": "1"} } - - do: - indices.refresh: {} + - do: + indices.refresh: {} +--- +"Parent/child inner hits": - do: search: rest_total_hits_as_int: true @@ -41,3 +41,24 @@ setup: - match: { hits.hits.0.inner_hits.child.hits.hits.0._index: "test"} - match: { hits.hits.0.inner_hits.child.hits.hits.0._id: "2" } - is_false: hits.hits.0.inner_hits.child.hits.hits.0._nested + - is_false: hits.hits.0.inner_hits.child.hits.hits.0._seq_no + - is_false: hits.hits.0.inner_hits.child.hits.hits.0._primary_term + +--- +"Parent/child inner hits with seq no": + - skip: + version: " - 6.99.99" + reason: support was added in 7.0 + + - do: + search: + rest_total_hits_as_int: true + body: { "query" : { "has_child" : + { "type" : "child", "query" : { "match_all" : {} }, "inner_hits" : { "seq_no_primary_term": true} } + } } + - match: { hits.total: 1 } + - match: { hits.hits.0._index: "test" } + - match: { hits.hits.0._id: "1" } + - match: { hits.hits.0.inner_hits.child.hits.hits.0._index: "test"} + - match: { hits.hits.0.inner_hits.child.hits.hits.0._id: "2" } + - is_false: hits.hits.0.inner_hits.child.hits.hits.0._nested diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/200_top_hits_metric.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/200_top_hits_metric.yml index 9a0b9657eb3d5..c8e588a463b71 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/200_top_hits_metric.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/200_top_hits_metric.yml @@ -1,8 +1,4 @@ ---- -"top_hits aggregation with nested documents": - - skip: - version: " - 6.1.99" - reason: "<= 6.1 nodes don't always include index or id in nested top hits" +setup: - do: indices.create: index: my-index @@ -54,6 +50,12 @@ ] } +--- +"top_hits aggregation with nested documents": + - skip: + version: " - 6.1.99" + reason: "<= 6.1 nodes don't always include index or id in nested top hits" + - do: search: rest_total_hits_as_int: true diff --git a/server/src/main/java/org/elasticsearch/index/query/NestedQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/NestedQueryBuilder.java index d2b432e7c7ca1..3c3856e208f04 100644 --- a/server/src/main/java/org/elasticsearch/index/query/NestedQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/NestedQueryBuilder.java @@ -368,6 +368,14 @@ static final class NestedInnerHitSubContext extends InnerHitsContext.InnerHitSub this.childObjectMapper = childObjectMapper; } + @Override + public void seqNoAndPrimaryTerm(boolean seqNoAndPrimaryTerm) { + assert seqNoAndPrimaryTerm() == false; + if (seqNoAndPrimaryTerm) { + throw new UnsupportedOperationException("nested documents are not assigned sequence numbers"); + } + } + @Override public TopDocsAndMaxScore[] topDocs(SearchHit[] hits) throws IOException { Weight innerHitQueryWeight = createInnerHitQueryWeight(); diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/SeqNoPrimaryTermFetchSubPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/SeqNoPrimaryTermFetchSubPhase.java index d99ee265c894b..527c88740af2c 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/SeqNoPrimaryTermFetchSubPhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/SeqNoPrimaryTermFetchSubPhase.java @@ -56,8 +56,9 @@ public void hitsExecute(SearchContext context, SearchHit[] hits) throws IOExcept int docId = hit.docId() - subReaderContext.docBase; long seqNo = SequenceNumbers.UNASSIGNED_SEQ_NO; long primaryTerm = SequenceNumbers.UNASSIGNED_PRIMARY_TERM; - if (seqNoField != null && seqNoField.advanceExact(docId)) { - boolean found = primaryTermField.advanceExact(docId); + // we have to check the primary term field as it is only assigned for non-nested documents + if (primaryTermField != null && primaryTermField.advanceExact(docId)) { + boolean found = seqNoField.advanceExact(docId); assert found: "found seq no for " + docId + " but not a primary term"; seqNo = seqNoField.longValue(); primaryTerm = primaryTermField.longValue(); diff --git a/server/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java index c7716025268d0..257ee807419b6 100644 --- a/server/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java @@ -140,6 +140,11 @@ public void testEqualsAndHashcode() { } } + public static InnerHitBuilder randomNestedInnerHits() { + InnerHitBuilder innerHitBuilder = randomInnerHits(); + innerHitBuilder.setSeqNoAndPrimaryTerm(false); // not supported by nested queries + return innerHitBuilder; + } public static InnerHitBuilder randomInnerHits() { InnerHitBuilder innerHits = new InnerHitBuilder(); innerHits.setName(randomAlphaOfLengthBetween(1, 16)); diff --git a/server/src/test/java/org/elasticsearch/index/query/NestedQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/NestedQueryBuilderTests.java index 76479791283b4..ac9ae8d0fa7fb 100644 --- a/server/src/test/java/org/elasticsearch/index/query/NestedQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/NestedQueryBuilderTests.java @@ -45,7 +45,7 @@ import java.util.Map; import static org.elasticsearch.index.IndexSettingsTests.newIndexMeta; -import static org.elasticsearch.index.query.InnerHitBuilderTests.randomInnerHits; +import static org.elasticsearch.index.query.InnerHitBuilderTests.randomNestedInnerHits; import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.instanceOf; @@ -267,7 +267,7 @@ public void testThatUnrecognizedFromStringThrowsException() { } public void testInlineLeafInnerHitsNestedQuery() throws Exception { - InnerHitBuilder leafInnerHits = randomInnerHits(); + InnerHitBuilder leafInnerHits = randomNestedInnerHits(); NestedQueryBuilder nestedQueryBuilder = new NestedQueryBuilder("path", new MatchAllQueryBuilder(), ScoreMode.None); nestedQueryBuilder.innerHit(leafInnerHits); Map innerHitBuilders = new HashMap<>(); @@ -276,7 +276,7 @@ public void testInlineLeafInnerHitsNestedQuery() throws Exception { } public void testInlineLeafInnerHitsNestedQueryViaBoolQuery() { - InnerHitBuilder leafInnerHits = randomInnerHits(); + InnerHitBuilder leafInnerHits = randomNestedInnerHits(); NestedQueryBuilder nestedQueryBuilder = new NestedQueryBuilder("path", new MatchAllQueryBuilder(), ScoreMode.None) .innerHit(leafInnerHits); BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder().should(nestedQueryBuilder); @@ -286,7 +286,7 @@ public void testInlineLeafInnerHitsNestedQueryViaBoolQuery() { } public void testInlineLeafInnerHitsNestedQueryViaConstantScoreQuery() { - InnerHitBuilder leafInnerHits = randomInnerHits(); + InnerHitBuilder leafInnerHits = randomNestedInnerHits(); NestedQueryBuilder nestedQueryBuilder = new NestedQueryBuilder("path", new MatchAllQueryBuilder(), ScoreMode.None) .innerHit(leafInnerHits); ConstantScoreQueryBuilder constantScoreQueryBuilder = new ConstantScoreQueryBuilder(nestedQueryBuilder); @@ -296,10 +296,10 @@ public void testInlineLeafInnerHitsNestedQueryViaConstantScoreQuery() { } public void testInlineLeafInnerHitsNestedQueryViaBoostingQuery() { - InnerHitBuilder leafInnerHits1 = randomInnerHits(); + InnerHitBuilder leafInnerHits1 = randomNestedInnerHits(); NestedQueryBuilder nestedQueryBuilder1 = new NestedQueryBuilder("path", new MatchAllQueryBuilder(), ScoreMode.None) .innerHit(leafInnerHits1); - InnerHitBuilder leafInnerHits2 = randomInnerHits(); + InnerHitBuilder leafInnerHits2 = randomNestedInnerHits(); NestedQueryBuilder nestedQueryBuilder2 = new NestedQueryBuilder("path", new MatchAllQueryBuilder(), ScoreMode.None) .innerHit(leafInnerHits2); BoostingQueryBuilder constantScoreQueryBuilder = new BoostingQueryBuilder(nestedQueryBuilder1, nestedQueryBuilder2); @@ -310,7 +310,7 @@ public void testInlineLeafInnerHitsNestedQueryViaBoostingQuery() { } public void testInlineLeafInnerHitsNestedQueryViaFunctionScoreQuery() { - InnerHitBuilder leafInnerHits = randomInnerHits(); + InnerHitBuilder leafInnerHits = randomNestedInnerHits(); NestedQueryBuilder nestedQueryBuilder = new NestedQueryBuilder("path", new MatchAllQueryBuilder(), ScoreMode.None) .innerHit(leafInnerHits); FunctionScoreQueryBuilder functionScoreQueryBuilder = new FunctionScoreQueryBuilder(nestedQueryBuilder); @@ -330,7 +330,7 @@ public void testBuildIgnoreUnmappedNestQuery() throws Exception { when(mapperService.getIndexSettings()).thenReturn(settings); when(searchContext.mapperService()).thenReturn(mapperService); - InnerHitBuilder leafInnerHits = randomInnerHits(); + InnerHitBuilder leafInnerHits = randomNestedInnerHits(); NestedQueryBuilder query1 = new NestedQueryBuilder("path", new MatchAllQueryBuilder(), ScoreMode.None); query1.innerHit(leafInnerHits); final Map innerHitBuilders = new HashMap<>(); From 38c17107e0424ee805e018912d5f4edccc497c64 Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Thu, 17 Jan 2019 22:09:50 +0100 Subject: [PATCH 10/29] more rest fixes --- .../200_top_hits_metric.yml | 51 ------------------- 1 file changed, 51 deletions(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/200_top_hits_metric.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/200_top_hits_metric.yml index c8e588a463b71..20c9b003e7631 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/200_top_hits_metric.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/200_top_hits_metric.yml @@ -91,57 +91,6 @@ setup: version: " - 6.99.99" reason: support was added in 7.0 - - do: - indices.create: - index: my-index - body: - settings: - number_of_shards: 1 - number_of_replicas: 0 - mappings: - doc: - properties: - users: - type: nested - - - do: - index: - index: my-index - type: doc - id: 1 - refresh: true - body: | - { - "group" : "fans", - "users" : [ - { - "first" : "John", - "last" : "Smith" - }, - { - "first" : "Alice", - "last" : "White" - } - ] - } - - - do: - index: - index: my-index - type: doc - id: 2 - refresh: true - body: | - { - "group" : "fans", - "users" : [ - { - "first" : "Mark", - "last" : "Doe" - } - ] - } - - do: search: rest_total_hits_as_int: true From f1716afba57d93859b8c6f50c0212e499f33ee34 Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Thu, 17 Jan 2019 22:33:34 +0100 Subject: [PATCH 11/29] fixed collapsing --- .../rest-api-spec/test/search/110_field_collapsing.yml | 9 +-------- .../elasticsearch/action/search/ExpandSearchPhase.java | 1 + 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/110_field_collapsing.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/110_field_collapsing.yml index 6406eaecadcfa..a85abb4e6e28b 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/110_field_collapsing.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/110_field_collapsing.yml @@ -419,7 +419,7 @@ setup: index: test body: collapse: { field: numeric_group, inner_hits: { - name: sub_hits, version: true, seq_no_primary_term: true, size: 2, sort: [{ sort: asc }] + name: sub_hits, seq_no_primary_term: true, size: 2, sort: [{ sort: asc }] } } sort: [{ sort: desc }] @@ -429,8 +429,6 @@ setup: - match: { hits.hits.0.fields.numeric_group: [3] } - match: { hits.hits.0.sort: [36] } - match: { hits.hits.0._id: "6" } - - gte: { hits.hits.0._seq_no: 0 } - - gte: { hits.hits.0._primary_term: 1 } - match: { hits.hits.0.inner_hits.sub_hits.hits.total: 1 } - length: { hits.hits.0.inner_hits.sub_hits.hits.hits: 1 } - match: { hits.hits.0.inner_hits.sub_hits.hits.hits.0._id: "6" } @@ -440,9 +438,6 @@ setup: - match: { hits.hits.1.fields.numeric_group: [1] } - match: { hits.hits.1.sort: [24] } - match: { hits.hits.1._id: "3" } - - match: { hits.hits.1._version: 33 } - - gte: { hits.hits.1._seq_no: 0 } - - gte: { hits.hits.1._primary_term: 1 } - match: { hits.hits.1.inner_hits.sub_hits.hits.total: 3 } - length: { hits.hits.1.inner_hits.sub_hits.hits.hits: 2 } - match: { hits.hits.1.inner_hits.sub_hits.hits.hits.0._id: "2" } @@ -456,8 +451,6 @@ setup: - match: { hits.hits.2.fields.numeric_group: [25] } - match: { hits.hits.2.sort: [10] } - match: { hits.hits.2._id: "4" } - - gte: { hits.hits.2._seq_no: 0 } - - gte: { hits.hits.2._primary_term: 1 } - match: { hits.hits.2.inner_hits.sub_hits.hits.total: 2 } - length: { hits.hits.2.inner_hits.sub_hits.hits.hits: 2 } - match: { hits.hits.2.inner_hits.sub_hits.hits.hits.0._id: "5" } diff --git a/server/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java b/server/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java index da481b7a4a8ee..10a85b723166c 100644 --- a/server/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java +++ b/server/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java @@ -153,6 +153,7 @@ private SearchSourceBuilder buildExpandSearchSourceBuilder(InnerHitBuilder optio groupSource.explain(options.isExplain()); groupSource.trackScores(options.isTrackScores()); groupSource.version(options.isVersion()); + groupSource.seqNoAndPrimaryTerm(options.isSeqNoAndPrimaryTerm()); if (innerCollapseBuilder != null) { groupSource.collapse(innerCollapseBuilder); } From 6d7df1e696535c3382b0d4000d0de380caf5cfe6 Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Fri, 18 Jan 2019 08:15:41 +0100 Subject: [PATCH 12/29] top hits move away from nested --- .../200_top_hits_metric.yml | 28 ++++++++----------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/200_top_hits_metric.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/200_top_hits_metric.yml index 20c9b003e7631..775475e01a597 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/200_top_hits_metric.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/200_top_hits_metric.yml @@ -96,9 +96,9 @@ setup: rest_total_hits_as_int: true body: aggs: - to-users: - nested: - path: users + groups: + terms: + field: group.keyword aggs: users: top_hits: @@ -106,16 +106,12 @@ setup: seq_no_primary_term: true - match: { hits.total: 2 } - - length: { aggregations.to-users.users.hits.hits: 3 } - - match: { aggregations.to-users.users.hits.hits.0._id: "2" } - - match: { aggregations.to-users.users.hits.hits.0._index: my-index } - - gte: { aggregations.to-users.users.hits.hits.0._seq_no: 0 } - - gte: { aggregations.to-users.users.hits.hits.0._primary_term: 1 } - - match: { aggregations.to-users.users.hits.hits.1._id: "1" } - - match: { aggregations.to-users.users.hits.hits.1._index: my-index } - - gte: { aggregations.to-users.users.hits.hits.1._seq_no: 0 } - - gte: { aggregations.to-users.users.hits.hits.1._primary_term: 1 } - - match: { aggregations.to-users.users.hits.hits.2._id: "1" } - - match: { aggregations.to-users.users.hits.hits.2._index: my-index } - - gte: { aggregations.to-users.users.hits.hits.2._seq_no: 0 } - - gte: { aggregations.to-users.users.hits.hits.2._primary_term: 1 } + - length: { aggregations.groups.buckets.0.users.hits.hits: 2 } + - match: { aggregations.groups.buckets.0.users.hits.hits.0._id: "1" } + - match: { aggregations.groups.buckets.0.users.hits.hits.0._index: my-index } + - gte: { aggregations.groups.buckets.0.users.hits.hits.0._seq_no: 0 } + - gte: { aggregations.groups.buckets.0.users.hits.hits.0._primary_term: 1 } + - match: { aggregations.groups.buckets.0.users.hits.hits.1._id: "2" } + - match: { aggregations.groups.buckets.0.users.hits.hits.1._index: my-index } + - gte: { aggregations.groups.buckets.0.users.hits.hits.1._seq_no: 0 } + - gte: { aggregations.groups.buckets.0.users.hits.hits.1._primary_term: 1 } From e56da1ebb172fea58eea81c56d27572700d97b5f Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Fri, 18 Jan 2019 11:52:17 +0100 Subject: [PATCH 13/29] remove seq_no_primary_term from testInitialSearchParamsMisc it's irrelevant --- .../reindex/remote/RemoteRequestBuildersTests.java | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/remote/RemoteRequestBuildersTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/remote/RemoteRequestBuildersTests.java index 4e5b99f82e8ba..0f985fd37016a 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/remote/RemoteRequestBuildersTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/remote/RemoteRequestBuildersTests.java @@ -179,12 +179,7 @@ public void testInitialSearchParamsMisc() { fetchVersion = randomBoolean(); searchRequest.source().version(fetchVersion); } - Boolean fetchSeqNoAndTerm = null; - if (randomBoolean()) { - fetchSeqNoAndTerm = randomBoolean(); - searchRequest.source().seqNoAndPrimaryTerm(fetchSeqNoAndTerm); - } - + Map params = initialSearch(searchRequest, query, remoteVersion).getParameters(); if (scroll == null) { @@ -194,9 +189,6 @@ public void testInitialSearchParamsMisc() { } assertThat(params, hasEntry("size", Integer.toString(size))); assertThat(params, fetchVersion == null || fetchVersion == true ? hasEntry("version", null) : not(hasEntry("version", null))); - assertThat(params, fetchSeqNoAndTerm == null || fetchSeqNoAndTerm ? hasEntry("seq_no", null) : not(hasEntry("seq_no", null))); - assertThat(params, fetchSeqNoAndTerm == null || fetchSeqNoAndTerm ? - hasEntry("primary_term", null) : not(hasEntry("primary_term", null))); } private void assertScroll(Version remoteVersion, Map params, TimeValue requested) { From 6ba7569a6d3db248778ef5bf8656b2aee6ca1301 Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Fri, 18 Jan 2019 23:16:03 +0100 Subject: [PATCH 14/29] remote cluster fix --- .../resources/rest-api-spec/test/multi_cluster/10_basic.yml | 3 +++ server/src/main/java/org/elasticsearch/search/SearchHit.java | 2 ++ 2 files changed, 5 insertions(+) diff --git a/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/10_basic.yml b/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/10_basic.yml index 70afa9bf917e8..1cb601fc0e848 100644 --- a/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/10_basic.yml +++ b/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/10_basic.yml @@ -30,6 +30,7 @@ rest_total_hits_as_int: true index: test_index,my_remote_cluster:test_index body: + seq_no_and_primary_term: true aggs: cluster: terms: @@ -37,6 +38,8 @@ - match: { _shards.total: 5 } - match: { hits.total: 11 } + - gte: { hits.hits.0._seq_no: 0 } + - gte: { hits.hits.0._primary_term: 1 } - length: { aggregations.cluster.buckets: 2 } - match: { aggregations.cluster.buckets.0.key: "remote_cluster" } - match: { aggregations.cluster.buckets.0.doc_count: 6 } diff --git a/server/src/main/java/org/elasticsearch/search/SearchHit.java b/server/src/main/java/org/elasticsearch/search/SearchHit.java index ecfb31f5fc4f3..1a6e44fd1ab18 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchHit.java +++ b/server/src/main/java/org/elasticsearch/search/SearchHit.java @@ -618,6 +618,8 @@ public static SearchHit createFromMap(Map values) { } searchHit.score(get(Fields._SCORE, values, DEFAULT_SCORE)); searchHit.version(get(Fields._VERSION, values, -1L)); + searchHit.setSeqNo(get(Fields._SEQ_NO, values, SequenceNumbers.UNASSIGNED_SEQ_NO)); + searchHit.setPrimaryTerm(get(Fields._PRIMARY_TERM, values, SequenceNumbers.UNASSIGNED_PRIMARY_TERM)); searchHit.sortValues(get(Fields.SORT, values, SearchSortValues.EMPTY)); searchHit.highlightFields(get(Fields.HIGHLIGHT, values, null)); searchHit.sourceRef(get(SourceFieldMapper.NAME, values, null)); From f6d3c086f0b9305b0ee15c594d28e8b5f436387a Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Fri, 18 Jan 2019 23:54:50 +0100 Subject: [PATCH 15/29] remove seq_no_and_term --- .../join/query/HasChildQueryBuilderTests.java | 1 - .../elasticsearch/join/query/InnerHitsIT.java | 26 ++++--------- .../rest-api-spec/test/11_parent_child.yml | 4 +- .../resources/rest-api-spec/api/search.json | 4 -- .../200_top_hits_metric.yml | 1 - .../test/search/110_field_collapsing.yml | 4 +- .../test/search/300_sequence_numbers.yml | 16 -------- .../action/search/ExpandSearchPhase.java | 1 - .../index/query/InnerHitBuilder.java | 21 ---------- .../index/query/InnerHitContextBuilder.java | 1 - .../index/query/NestedQueryBuilder.java | 8 ---- .../rest/action/search/RestSearchAction.java | 3 -- .../search/DefaultSearchContext.java | 11 ------ .../elasticsearch/search/SearchService.java | 4 -- .../metrics/TopHitsAggregationBuilder.java | 36 ++---------------- .../metrics/TopHitsAggregatorFactory.java | 10 ++--- .../search/builder/SearchSourceBuilder.java | 38 +------------------ .../SeqNoPrimaryTermFetchSubPhase.java | 4 -- .../internal/FilteredSearchContext.java | 10 ----- .../search/internal/SearchContext.java | 6 --- .../search/internal/SubSearchContext.java | 11 ------ .../action/search/ExpandSearchPhaseTests.java | 4 +- .../index/query/InnerHitBuilderTests.java | 7 ---- .../index/query/NestedQueryBuilderTests.java | 16 ++++---- .../aggregations/metrics/TopHitsIT.java | 12 +----- .../aggregations/metrics/TopHitsTests.java | 3 -- .../search/RandomSearchRequestGenerator.java | 3 -- .../elasticsearch/test/TestSearchContext.java | 10 ----- 28 files changed, 28 insertions(+), 247 deletions(-) diff --git a/modules/parent-join/src/test/java/org/elasticsearch/join/query/HasChildQueryBuilderTests.java b/modules/parent-join/src/test/java/org/elasticsearch/join/query/HasChildQueryBuilderTests.java index eea01d61386de..6e4e79d16e5a5 100644 --- a/modules/parent-join/src/test/java/org/elasticsearch/join/query/HasChildQueryBuilderTests.java +++ b/modules/parent-join/src/test/java/org/elasticsearch/join/query/HasChildQueryBuilderTests.java @@ -252,7 +252,6 @@ public void testFromJson() throws IOException { " \"from\" : 0,\n" + " \"size\" : 100,\n" + " \"version\" : false,\n" + - " \"seq_no_primary_term\" : false,\n" + " \"explain\" : false,\n" + " \"track_scores\" : false,\n" + " \"sort\" : [ {\n" + diff --git a/modules/parent-join/src/test/java/org/elasticsearch/join/query/InnerHitsIT.java b/modules/parent-join/src/test/java/org/elasticsearch/join/query/InnerHitsIT.java index 89929985ea594..b6718f6c8dbb3 100644 --- a/modules/parent-join/src/test/java/org/elasticsearch/join/query/InnerHitsIT.java +++ b/modules/parent-join/src/test/java/org/elasticsearch/join/query/InnerHitsIT.java @@ -56,8 +56,6 @@ import static org.elasticsearch.index.query.QueryBuilders.matchQuery; import static org.elasticsearch.index.query.QueryBuilders.nestedQuery; import static org.elasticsearch.index.query.QueryBuilders.termQuery; -import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_PRIMARY_TERM; -import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_SEQ_NO; import static org.elasticsearch.join.query.JoinQueryBuilders.hasChildQuery; import static org.elasticsearch.join.query.JoinQueryBuilders.hasParentQuery; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; @@ -136,10 +134,9 @@ public void testSimpleParentChild() throws Exception { assertThat(innerHits.getAt(1).getId(), equalTo("c2")); assertThat(innerHits.getAt(1).getType(), equalTo("doc")); - final boolean seqNoAndTerm = randomBoolean(); response = client().prepareSearch("articles") .setQuery(hasChildQuery("comment", matchQuery("message", "elephant"), ScoreMode.None) - .innerHit(new InnerHitBuilder().setSeqNoAndPrimaryTerm(seqNoAndTerm))) + .innerHit(new InnerHitBuilder())) .get(); assertNoFailures(response); assertHitCount(response, 1); @@ -156,21 +153,12 @@ public void testSimpleParentChild() throws Exception { assertThat(innerHits.getAt(2).getId(), equalTo("c6")); assertThat(innerHits.getAt(2).getType(), equalTo("doc")); - if (seqNoAndTerm) { - assertThat(innerHits.getAt(0).getPrimaryTerm(), equalTo(1L)); - assertThat(innerHits.getAt(1).getPrimaryTerm(), equalTo(1L)); - assertThat(innerHits.getAt(2).getPrimaryTerm(), equalTo(1L)); - assertThat(innerHits.getAt(0).getSeqNo(), greaterThanOrEqualTo(0L)); - assertThat(innerHits.getAt(1).getSeqNo(), greaterThanOrEqualTo(0L)); - assertThat(innerHits.getAt(2).getSeqNo(), greaterThanOrEqualTo(0L)); - } else { - assertThat(innerHits.getAt(0).getPrimaryTerm(), equalTo(UNASSIGNED_PRIMARY_TERM)); - assertThat(innerHits.getAt(1).getPrimaryTerm(), equalTo(UNASSIGNED_PRIMARY_TERM)); - assertThat(innerHits.getAt(2).getPrimaryTerm(), equalTo(UNASSIGNED_PRIMARY_TERM)); - assertThat(innerHits.getAt(0).getSeqNo(), equalTo(UNASSIGNED_SEQ_NO)); - assertThat(innerHits.getAt(1).getSeqNo(), equalTo(UNASSIGNED_SEQ_NO)); - assertThat(innerHits.getAt(2).getSeqNo(), equalTo(UNASSIGNED_SEQ_NO)); - } + assertThat(innerHits.getAt(0).getPrimaryTerm(), equalTo(1L)); + assertThat(innerHits.getAt(1).getPrimaryTerm(), equalTo(1L)); + assertThat(innerHits.getAt(2).getPrimaryTerm(), equalTo(1L)); + assertThat(innerHits.getAt(0).getSeqNo(), greaterThanOrEqualTo(0L)); + assertThat(innerHits.getAt(1).getSeqNo(), greaterThanOrEqualTo(0L)); + assertThat(innerHits.getAt(2).getSeqNo(), greaterThanOrEqualTo(0L)); response = client().prepareSearch("articles") .setQuery( diff --git a/modules/parent-join/src/test/resources/rest-api-spec/test/11_parent_child.yml b/modules/parent-join/src/test/resources/rest-api-spec/test/11_parent_child.yml index db893580ee5bf..efdca604b2ed6 100644 --- a/modules/parent-join/src/test/resources/rest-api-spec/test/11_parent_child.yml +++ b/modules/parent-join/src/test/resources/rest-api-spec/test/11_parent_child.yml @@ -53,9 +53,7 @@ setup: - do: search: rest_total_hits_as_int: true - body: { "query" : { "has_child" : - { "type" : "child", "query" : { "match_all" : {} }, "inner_hits" : { "seq_no_primary_term": true} } - } } + body: { "query" : { "has_child" : { "type" : "child", "query" : { "match_all" : {} }, "inner_hits" : {} } } } - match: { hits.total: 1 } - match: { hits.hits.0._index: "test" } - match: { hits.hits.0._id: "1" } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/search.json b/rest-api-spec/src/main/resources/rest-api-spec/api/search.json index 9ac02b1214a2f..5834ca623a99b 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/search.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/search.json @@ -164,10 +164,6 @@ "type" : "boolean", "description" : "Specify whether to return document version as part of a hit" }, - "seq_no_primary_term": { - "type" : "boolean", - "description" : "Specify whether to return sequence number and primary term of the last modification of each hit" - }, "request_cache": { "type" : "boolean", "description" : "Specify if request cache should be used for this request or not, defaults to index level setting" diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/200_top_hits_metric.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/200_top_hits_metric.yml index 775475e01a597..e8ef0f172debf 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/200_top_hits_metric.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/200_top_hits_metric.yml @@ -103,7 +103,6 @@ setup: users: top_hits: sort: "users.last.keyword" - seq_no_primary_term: true - match: { hits.total: 2 } - length: { aggregations.groups.buckets.0.users.hits.hits: 2 } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/110_field_collapsing.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/110_field_collapsing.yml index a85abb4e6e28b..5e56d02586e06 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/110_field_collapsing.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/110_field_collapsing.yml @@ -418,9 +418,7 @@ setup: rest_total_hits_as_int: true index: test body: - collapse: { field: numeric_group, inner_hits: { - name: sub_hits, seq_no_primary_term: true, size: 2, sort: [{ sort: asc }] - } } + collapse: { field: numeric_group, inner_hits: { name: sub_hits, size: 2, sort: [{ sort: asc }]} } sort: [{ sort: desc }] - match: { hits.total: 6 } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/300_sequence_numbers.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/300_sequence_numbers.yml index 5f30114df89f8..ec04ab92fa6c9 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/300_sequence_numbers.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/300_sequence_numbers.yml @@ -35,7 +35,6 @@ setup: query: match: foo: bar - seq_no_primary_term: true - match: {hits.total.value: 1} - match: {hits.hits.0._seq_no: 1} @@ -54,22 +53,7 @@ setup: query: match: foo: bar - seq_no_primary_term: true - match: {hits.total.value: 1} - match: {hits.hits.0._seq_no: 1} - gte: {hits.hits.0._primary_term: 1} - ---- -"sequence numbers are not returned if not requested": - - do: - search: - index: _all - body: - query: - match: - foo: bar - - - match: {hits.total.value: 1} - - is_false: hits.hits.0._seq_no - - is_false: hits.hits.0._primary_term diff --git a/server/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java b/server/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java index 10a85b723166c..da481b7a4a8ee 100644 --- a/server/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java +++ b/server/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java @@ -153,7 +153,6 @@ private SearchSourceBuilder buildExpandSearchSourceBuilder(InnerHitBuilder optio groupSource.explain(options.isExplain()); groupSource.trackScores(options.isTrackScores()); groupSource.version(options.isVersion()); - groupSource.seqNoAndPrimaryTerm(options.isSeqNoAndPrimaryTerm()); if (innerCollapseBuilder != null) { groupSource.collapse(innerCollapseBuilder); } diff --git a/server/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java b/server/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java index 90275800d3769..9eb2a7fb9affb 100644 --- a/server/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java @@ -68,7 +68,6 @@ public final class InnerHitBuilder implements Writeable, ToXContentObject { PARSER.declareInt(InnerHitBuilder::setSize, SearchSourceBuilder.SIZE_FIELD); PARSER.declareBoolean(InnerHitBuilder::setExplain, SearchSourceBuilder.EXPLAIN_FIELD); PARSER.declareBoolean(InnerHitBuilder::setVersion, SearchSourceBuilder.VERSION_FIELD); - PARSER.declareBoolean(InnerHitBuilder::setSeqNoAndPrimaryTerm, SearchSourceBuilder.SEQ_NO_PRIMARY_TERM_FIELD); PARSER.declareBoolean(InnerHitBuilder::setTrackScores, SearchSourceBuilder.TRACK_SCORES_FIELD); PARSER.declareStringArray(InnerHitBuilder::setStoredFieldNames, SearchSourceBuilder.STORED_FIELDS_FIELD); PARSER.declareObjectArray(InnerHitBuilder::setDocValueFields, @@ -125,7 +124,6 @@ public final class InnerHitBuilder implements Writeable, ToXContentObject { private int size = 3; private boolean explain; private boolean version; - private boolean seqNoAndPrimaryTerm; private boolean trackScores; private StoredFieldsContext storedFieldsContext; @@ -156,11 +154,6 @@ public InnerHitBuilder(StreamInput in) throws IOException { size = in.readVInt(); explain = in.readBoolean(); version = in.readBoolean(); - if (in.getVersion().onOrAfter(Version.V_7_0_0)){ - seqNoAndPrimaryTerm = in.readBoolean(); - } else { - seqNoAndPrimaryTerm = false; - } trackScores = in.readBoolean(); storedFieldsContext = in.readOptionalWriteable(StoredFieldsContext::new); if (in.getVersion().before(Version.V_6_4_0)) { @@ -205,9 +198,6 @@ public void writeTo(StreamOutput out) throws IOException { out.writeVInt(size); out.writeBoolean(explain); out.writeBoolean(version); - if (out.getVersion().onOrAfter(Version.V_7_0_0)) { - out.writeBoolean(seqNoAndPrimaryTerm); - } out.writeBoolean(trackScores); out.writeOptionalWriteable(storedFieldsContext); if (out.getVersion().before(Version.V_6_4_0)) { @@ -308,15 +298,6 @@ public InnerHitBuilder setVersion(boolean version) { return this; } - public boolean isSeqNoAndPrimaryTerm() { - return seqNoAndPrimaryTerm; - } - - public InnerHitBuilder setSeqNoAndPrimaryTerm(boolean seqNoAndPrimaryTerm) { - this.seqNoAndPrimaryTerm = seqNoAndPrimaryTerm; - return this; - } - public boolean isTrackScores() { return trackScores; } @@ -454,7 +435,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(SearchSourceBuilder.FROM_FIELD.getPreferredName(), from); builder.field(SearchSourceBuilder.SIZE_FIELD.getPreferredName(), size); builder.field(SearchSourceBuilder.VERSION_FIELD.getPreferredName(), version); - builder.field(SearchSourceBuilder.SEQ_NO_PRIMARY_TERM_FIELD.getPreferredName(), seqNoAndPrimaryTerm); builder.field(SearchSourceBuilder.EXPLAIN_FIELD.getPreferredName(), explain); builder.field(SearchSourceBuilder.TRACK_SCORES_FIELD.getPreferredName(), trackScores); if (fetchSourceContext != null) { @@ -513,7 +493,6 @@ public boolean equals(Object o) { Objects.equals(size, that.size) && Objects.equals(explain, that.explain) && Objects.equals(version, that.version) && - Objects.equals(seqNoAndPrimaryTerm, that.seqNoAndPrimaryTerm) && Objects.equals(trackScores, that.trackScores) && Objects.equals(storedFieldsContext, that.storedFieldsContext) && Objects.equals(docValueFields, that.docValueFields) && diff --git a/server/src/main/java/org/elasticsearch/index/query/InnerHitContextBuilder.java b/server/src/main/java/org/elasticsearch/index/query/InnerHitContextBuilder.java index 9e6a766aed891..1fe781f38ce27 100644 --- a/server/src/main/java/org/elasticsearch/index/query/InnerHitContextBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/InnerHitContextBuilder.java @@ -78,7 +78,6 @@ protected void setupInnerHitsContext(QueryShardContext queryShardContext, innerHitsContext.size(innerHitBuilder.getSize()); innerHitsContext.explain(innerHitBuilder.isExplain()); innerHitsContext.version(innerHitBuilder.isVersion()); - innerHitsContext.seqNoAndPrimaryTerm(innerHitBuilder.isSeqNoAndPrimaryTerm()); innerHitsContext.trackScores(innerHitBuilder.isTrackScores()); if (innerHitBuilder.getStoredFieldsContext() != null) { innerHitsContext.storedFieldsContext(innerHitBuilder.getStoredFieldsContext()); diff --git a/server/src/main/java/org/elasticsearch/index/query/NestedQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/NestedQueryBuilder.java index 3c3856e208f04..d2b432e7c7ca1 100644 --- a/server/src/main/java/org/elasticsearch/index/query/NestedQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/NestedQueryBuilder.java @@ -368,14 +368,6 @@ static final class NestedInnerHitSubContext extends InnerHitsContext.InnerHitSub this.childObjectMapper = childObjectMapper; } - @Override - public void seqNoAndPrimaryTerm(boolean seqNoAndPrimaryTerm) { - assert seqNoAndPrimaryTerm() == false; - if (seqNoAndPrimaryTerm) { - throw new UnsupportedOperationException("nested documents are not assigned sequence numbers"); - } - } - @Override public TopDocsAndMaxScore[] topDocs(SearchHit[] hits) throws IOException { Weight innerHitQueryWeight = createInnerHitQueryWeight(); diff --git a/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java b/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java index da773efed580d..3e3a1e02a174b 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java @@ -201,9 +201,6 @@ private static void parseSearchSource(final SearchSourceBuilder searchSourceBuil if (request.hasParam("version")) { searchSourceBuilder.version(request.paramAsBoolean("version", null)); } - if (request.hasParam("seq_no_primary_term")) { - searchSourceBuilder.seqNoAndPrimaryTerm(request.paramAsBoolean("seq_no_primary_term", null)); - } if (request.hasParam("timeout")) { searchSourceBuilder.timeout(request.paramAsTime("timeout", null)); } diff --git a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java index 4b82e23e42061..590c58b1f6615 100644 --- a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java +++ b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java @@ -107,7 +107,6 @@ final class DefaultSearchContext extends SearchContext { private ScrollContext scrollContext; private boolean explain; private boolean version = false; // by default, we don't return versions - private boolean seqAndPrimaryTerm = false; private StoredFieldsContext storedFields; private ScriptFieldsContext scriptFields; private FetchSourceContext fetchSourceContext; @@ -720,16 +719,6 @@ public void version(boolean version) { this.version = version; } - @Override - public boolean seqNoAndPrimaryTerm() { - return seqAndPrimaryTerm; - } - - @Override - public void seqNoAndPrimaryTerm(boolean seqNoAndPrimaryTerm) { - this.seqAndPrimaryTerm = seqNoAndPrimaryTerm; - } - @Override public int[] docIdsToLoad() { return docIdsToLoad; diff --git a/server/src/main/java/org/elasticsearch/search/SearchService.java b/server/src/main/java/org/elasticsearch/search/SearchService.java index ddec3637ed491..92c5bcbb48734 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchService.java +++ b/server/src/main/java/org/elasticsearch/search/SearchService.java @@ -902,10 +902,6 @@ private void parseSource(DefaultSearchContext context, SearchSourceBuilder sourc context.version(source.version()); } - if (source.seqNoAndPrimaryTerm() != null) { - context.seqNoAndPrimaryTerm(source.seqNoAndPrimaryTerm()); - } - if (source.stats() != null) { context.groupStats(source.stats()); } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregationBuilder.java index ba51099d6bc00..771e5739e6641 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregationBuilder.java @@ -19,7 +19,6 @@ package org.elasticsearch.search.aggregations.metrics; -import org.elasticsearch.Version; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.Strings; @@ -67,7 +66,6 @@ public class TopHitsAggregationBuilder extends AbstractAggregationBuilder> sorts = null; private HighlightBuilder highlightBuilder; @@ -87,7 +85,6 @@ protected TopHitsAggregationBuilder(TopHitsAggregationBuilder clone, this.size = clone.size; this.explain = clone.explain; this.version = clone.version; - this.seqNoAndPrimaryTerm = clone.seqNoAndPrimaryTerm; this.trackScores = clone.trackScores; this.sorts = clone.sorts == null ? null : new ArrayList<>(clone.sorts); this.highlightBuilder = clone.highlightBuilder == null ? null : @@ -140,9 +137,6 @@ public TopHitsAggregationBuilder(StreamInput in) throws IOException { } trackScores = in.readBoolean(); version = in.readBoolean(); - if (in.getVersion().onOrAfter(Version.V_7_0_0)) { - seqNoAndPrimaryTerm = in.readBoolean(); - } } @Override @@ -179,9 +173,6 @@ protected void doWriteTo(StreamOutput out) throws IOException { } out.writeBoolean(trackScores); out.writeBoolean(version); - if (out.getVersion().onOrAfter(Version.V_7_0_0)) { - out.writeBoolean(seqNoAndPrimaryTerm); - } } /** @@ -535,23 +526,6 @@ public boolean version() { return version; } - /** - * Should each {@link org.elasticsearch.search.SearchHit} be returned with the - * sequence number and primary term of the last modification of the document. - */ - public TopHitsAggregationBuilder seqNoAndPrimaryTerm(Boolean seqNoAndPrimaryTerm) { - this.seqNoAndPrimaryTerm = seqNoAndPrimaryTerm; - return this; - } - - /** - * Indicates whether {@link org.elasticsearch.search.SearchHit}s should be returned with the - * sequence number and primary term of the last modification of the document. - */ - public Boolean seqNoAndPrimaryTerm() { - return seqNoAndPrimaryTerm; - } - /** * Applies when sorting, and controls if scores will be tracked as well. * Defaults to {@code false}. @@ -605,7 +579,7 @@ protected TopHitsAggregatorFactory doBuild(SearchContext context, AggregatorFact } else { optionalSort = SortBuilder.buildSort(sorts, context.getQueryShardContext()); } - return new TopHitsAggregatorFactory(name, from, size, explain, version, seqNoAndPrimaryTerm, trackScores, optionalSort, + return new TopHitsAggregatorFactory(name, from, size, explain, version, trackScores, optionalSort, highlightBuilder, storedFieldsContext, docValueFields, fields, fetchSourceContext, context, parent, subfactoriesBuilder, metaData); } @@ -616,7 +590,6 @@ protected XContentBuilder internalXContent(XContentBuilder builder, Params param builder.field(SearchSourceBuilder.FROM_FIELD.getPreferredName(), from); builder.field(SearchSourceBuilder.SIZE_FIELD.getPreferredName(), size); builder.field(SearchSourceBuilder.VERSION_FIELD.getPreferredName(), version); - builder.field(SearchSourceBuilder.SEQ_NO_PRIMARY_TERM_FIELD.getPreferredName(), seqNoAndPrimaryTerm); builder.field(SearchSourceBuilder.EXPLAIN_FIELD.getPreferredName(), explain); if (fetchSourceContext != null) { builder.field(SearchSourceBuilder._SOURCE_FIELD.getPreferredName(), fetchSourceContext); @@ -674,8 +647,6 @@ public static TopHitsAggregationBuilder parse(String aggregationName, XContentPa factory.size(parser.intValue()); } else if (SearchSourceBuilder.VERSION_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { factory.version(parser.booleanValue()); - } else if (SearchSourceBuilder.SEQ_NO_PRIMARY_TERM_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { - factory.seqNoAndPrimaryTerm(parser.booleanValue()); } else if (SearchSourceBuilder.EXPLAIN_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { factory.explain(parser.booleanValue()); } else if (SearchSourceBuilder.TRACK_SCORES_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { @@ -775,7 +746,7 @@ public static TopHitsAggregationBuilder parse(String aggregationName, XContentPa @Override protected int doHashCode() { return Objects.hash(explain, fetchSourceContext, docValueFields, storedFieldsContext, from, highlightBuilder, - scriptFields, size, sorts, trackScores, version, seqNoAndPrimaryTerm); + scriptFields, size, sorts, trackScores, version); } @Override @@ -791,8 +762,7 @@ protected boolean doEquals(Object obj) { && Objects.equals(size, other.size) && Objects.equals(sorts, other.sorts) && Objects.equals(trackScores, other.trackScores) - && Objects.equals(version, other.version) - && Objects.equals(seqNoAndPrimaryTerm, other.seqNoAndPrimaryTerm); + && Objects.equals(version, other.version); } @Override diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregatorFactory.java index 7edaccb66d4bd..85d6d127c8892 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregatorFactory.java @@ -44,7 +44,6 @@ class TopHitsAggregatorFactory extends AggregatorFactory sort; private final HighlightBuilder highlightBuilder; @@ -53,9 +52,10 @@ class TopHitsAggregatorFactory extends AggregatorFactory scriptFields; private final FetchSourceContext fetchSourceContext; - TopHitsAggregatorFactory(String name, int from, int size, boolean explain, boolean version, boolean seqNoAndPrimaryTerm, - boolean trackScores, Optional sort, HighlightBuilder highlightBuilder, StoredFieldsContext storedFieldsContext, - List docValueFields, List scriptFields, FetchSourceContext fetchSourceContext, + TopHitsAggregatorFactory(String name, int from, int size, boolean explain, boolean version, boolean trackScores, + Optional sort, HighlightBuilder highlightBuilder, StoredFieldsContext storedFieldsContext, + List docValueFields, List scriptFields, + FetchSourceContext fetchSourceContext, SearchContext context, AggregatorFactory parent, AggregatorFactories.Builder subFactories, Map metaData) throws IOException { super(name, context, parent, subFactories, metaData); @@ -63,7 +63,6 @@ class TopHitsAggregatorFactory extends AggregatorFactory> sorts; private boolean trackScores = false; @@ -250,11 +247,6 @@ public SearchSourceBuilder(StreamInput in) throws IOException { timeout = in.readOptionalTimeValue(); trackScores = in.readBoolean(); version = in.readOptionalBoolean(); - if (in.getVersion().onOrAfter(Version.V_7_0_0)) { - seqNoAndPrimaryTerm = in.readOptionalBoolean(); - } else { - seqNoAndPrimaryTerm = null; - } extBuilders = in.readNamedWriteableList(SearchExtBuilder.class); profile = in.readBoolean(); searchAfterBuilder = in.readOptionalWriteable(SearchAfterBuilder::new); @@ -318,9 +310,6 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalTimeValue(timeout); out.writeBoolean(trackScores); out.writeOptionalBoolean(version); - if (out.getVersion().onOrAfter(Version.V_7_0_0)) { - out.writeOptionalBoolean(seqNoAndPrimaryTerm); - } out.writeNamedWriteableList(extBuilders); out.writeBoolean(profile); out.writeOptionalWriteable(searchAfterBuilder); @@ -452,23 +441,6 @@ public Boolean version() { return version; } - /** - * Should each {@link org.elasticsearch.search.SearchHit} be returned with the - * sequence number and primary term of the last modification of the document. - */ - public SearchSourceBuilder seqNoAndPrimaryTerm(Boolean seqNoAndPrimaryTerm) { - this.seqNoAndPrimaryTerm = seqNoAndPrimaryTerm; - return this; - } - - /** - * Indicates whether {@link org.elasticsearch.search.SearchHit}s should be returned with the - * sequence number and primary term of the last modification of the document. - */ - public Boolean seqNoAndPrimaryTerm() { - return seqNoAndPrimaryTerm; - } - /** * An optional timeout to control how long search is allowed to take. */ @@ -1027,7 +999,6 @@ private SearchSourceBuilder shallowCopy(QueryBuilder queryBuilder, QueryBuilder rewrittenBuilder.trackScores = trackScores; rewrittenBuilder.trackTotalHitsUpTo = trackTotalHitsUpTo; rewrittenBuilder.version = version; - rewrittenBuilder.seqNoAndPrimaryTerm = seqNoAndPrimaryTerm; rewrittenBuilder.collapse = collapse; return rewrittenBuilder; } @@ -1067,8 +1038,6 @@ public void parseXContent(XContentParser parser, boolean checkTrailingTokens) th minScore = parser.floatValue(); } else if (VERSION_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { version = parser.booleanValue(); - } else if (SEQ_NO_PRIMARY_TERM_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { - seqNoAndPrimaryTerm = parser.booleanValue(); } else if (EXPLAIN_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { explain = parser.booleanValue(); } else if (TRACK_SCORES_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { @@ -1236,10 +1205,6 @@ public XContentBuilder innerToXContent(XContentBuilder builder, Params params) t builder.field(VERSION_FIELD.getPreferredName(), version); } - if (seqNoAndPrimaryTerm != null) { - builder.field(SEQ_NO_PRIMARY_TERM_FIELD.getPreferredName(), seqNoAndPrimaryTerm); - } - if (explain != null) { builder.field(EXPLAIN_FIELD.getPreferredName(), explain); } @@ -1558,7 +1523,7 @@ public int hashCode() { return Objects.hash(aggregations, explain, fetchSourceContext, docValueFields, storedFieldsContext, from, highlightBuilder, indexBoosts, minScore, postQueryBuilder, queryBuilder, rescoreBuilders, scriptFields, size, sorts, searchAfterBuilder, sliceBuilder, stats, suggestBuilder, terminateAfter, timeout, trackScores, version, - seqNoAndPrimaryTerm, profile, extBuilders, collapse, trackTotalHitsUpTo); + profile, extBuilders, collapse, trackTotalHitsUpTo); } @Override @@ -1593,7 +1558,6 @@ public boolean equals(Object obj) { && Objects.equals(timeout, other.timeout) && Objects.equals(trackScores, other.trackScores) && Objects.equals(version, other.version) - && Objects.equals(seqNoAndPrimaryTerm, other.seqNoAndPrimaryTerm) && Objects.equals(profile, other.profile) && Objects.equals(extBuilders, other.extBuilders) && Objects.equals(collapse, other.collapse) diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/SeqNoPrimaryTermFetchSubPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/SeqNoPrimaryTermFetchSubPhase.java index 527c88740af2c..ea81610ea5bc5 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/SeqNoPrimaryTermFetchSubPhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/SeqNoPrimaryTermFetchSubPhase.java @@ -34,10 +34,6 @@ public final class SeqNoPrimaryTermFetchSubPhase implements FetchSubPhase { @Override public void hitsExecute(SearchContext context, SearchHit[] hits) throws IOException { - if (context.seqNoAndPrimaryTerm() == false || - (context.storedFieldsContext() != null && context.storedFieldsContext().fetchFields() == false)) { - return; - } hits = hits.clone(); // don't modify the incoming hits Arrays.sort(hits, Comparator.comparingInt(SearchHit::docId)); diff --git a/server/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java b/server/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java index ecde28719cec6..3a7fb9f823f3a 100644 --- a/server/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java +++ b/server/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java @@ -422,16 +422,6 @@ public void version(boolean version) { in.version(version); } - @Override - public boolean seqNoAndPrimaryTerm() { - return in.seqNoAndPrimaryTerm(); - } - - @Override - public void seqNoAndPrimaryTerm(boolean seqNoAndPrimaryTerm) { - in.seqNoAndPrimaryTerm(seqNoAndPrimaryTerm); - } - @Override public int[] docIdsToLoad() { return in.docIdsToLoad(); diff --git a/server/src/main/java/org/elasticsearch/search/internal/SearchContext.java b/server/src/main/java/org/elasticsearch/search/internal/SearchContext.java index bd6d9c501c8d1..5e8fec3195a65 100644 --- a/server/src/main/java/org/elasticsearch/search/internal/SearchContext.java +++ b/server/src/main/java/org/elasticsearch/search/internal/SearchContext.java @@ -309,12 +309,6 @@ public InnerHitsContext innerHits() { public abstract void version(boolean version); - /** indicates whether the sequence number and primary term of the last modification to each hit should be returned */ - public abstract boolean seqNoAndPrimaryTerm(); - - /** controls whether the sequence number and primary term of the last modification to each hit should be returned */ - public abstract void seqNoAndPrimaryTerm(boolean seqNoAndPrimaryTerm); - public abstract int[] docIdsToLoad(); public abstract int docIdsToLoadFrom(); diff --git a/server/src/main/java/org/elasticsearch/search/internal/SubSearchContext.java b/server/src/main/java/org/elasticsearch/search/internal/SubSearchContext.java index fb4d233f10ee8..8c8137f5e4345 100644 --- a/server/src/main/java/org/elasticsearch/search/internal/SubSearchContext.java +++ b/server/src/main/java/org/elasticsearch/search/internal/SubSearchContext.java @@ -65,7 +65,6 @@ public class SubSearchContext extends FilteredSearchContext { private boolean explain; private boolean trackScores; private boolean version; - private boolean seqNoAndPrimaryTerm; public SubSearchContext(SearchContext context) { super(context); @@ -295,16 +294,6 @@ public void version(boolean version) { this.version = version; } - @Override - public boolean seqNoAndPrimaryTerm() { - return seqNoAndPrimaryTerm; - } - - @Override - public void seqNoAndPrimaryTerm(boolean seqNoAndPrimaryTerm) { - this.seqNoAndPrimaryTerm = seqNoAndPrimaryTerm; - } - @Override public int[] docIdsToLoad() { return docIdsToLoad; diff --git a/server/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java b/server/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java index 328950e4f3569..9378a2cdd86bb 100644 --- a/server/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java @@ -241,7 +241,6 @@ public void run() throws IOException { public void testExpandRequestOptions() throws IOException { MockSearchPhaseContext mockSearchPhaseContext = new MockSearchPhaseContext(1); boolean version = randomBoolean(); - final boolean seqNoAndTerm = randomBoolean(); mockSearchPhaseContext.searchTransport = new SearchTransportService(null, null) { @Override @@ -250,14 +249,13 @@ void sendExecuteMultiSearch(MultiSearchRequest request, SearchTask task, ActionL assertTrue(request.requests().stream().allMatch((r) -> "foo".equals(r.preference()))); assertTrue(request.requests().stream().allMatch((r) -> "baz".equals(r.routing()))); assertTrue(request.requests().stream().allMatch((r) -> version == r.source().version())); - assertTrue(request.requests().stream().allMatch((r) -> seqNoAndTerm == r.source().seqNoAndPrimaryTerm())); assertTrue(request.requests().stream().allMatch((r) -> postFilter.equals(r.source().postFilter()))); } }; mockSearchPhaseContext.getRequest().source(new SearchSourceBuilder() .collapse( new CollapseBuilder("someField") - .setInnerHits(new InnerHitBuilder().setName("foobarbaz").setVersion(version).setSeqNoAndPrimaryTerm(seqNoAndTerm)) + .setInnerHits(new InnerHitBuilder().setName("foobarbaz").setVersion(version)) ) .postFilter(QueryBuilders.existsQuery("foo"))) .preference("foobar") diff --git a/server/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java index 257ee807419b6..5c76c77b5c888 100644 --- a/server/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java @@ -140,11 +140,6 @@ public void testEqualsAndHashcode() { } } - public static InnerHitBuilder randomNestedInnerHits() { - InnerHitBuilder innerHitBuilder = randomInnerHits(); - innerHitBuilder.setSeqNoAndPrimaryTerm(false); // not supported by nested queries - return innerHitBuilder; - } public static InnerHitBuilder randomInnerHits() { InnerHitBuilder innerHits = new InnerHitBuilder(); innerHits.setName(randomAlphaOfLengthBetween(1, 16)); @@ -152,7 +147,6 @@ public static InnerHitBuilder randomInnerHits() { innerHits.setSize(randomIntBetween(0, 32)); innerHits.setExplain(randomBoolean()); innerHits.setVersion(randomBoolean()); - innerHits.setSeqNoAndPrimaryTerm(randomBoolean()); innerHits.setTrackScores(randomBoolean()); if (randomBoolean()) { innerHits.setStoredFieldNames(randomListStuff(16, () -> randomAlphaOfLengthBetween(1, 16))); @@ -195,7 +189,6 @@ static InnerHitBuilder mutate(InnerHitBuilder original) throws IOException { modifiers.add(() -> copy.setSize(randomValueOtherThan(copy.getSize(), () -> randomIntBetween(0, 128)))); modifiers.add(() -> copy.setExplain(!copy.isExplain())); modifiers.add(() -> copy.setVersion(!copy.isVersion())); - modifiers.add(() -> copy.setSeqNoAndPrimaryTerm(!copy.isSeqNoAndPrimaryTerm())); modifiers.add(() -> copy.setTrackScores(!copy.isTrackScores())); modifiers.add(() -> copy.setName(randomValueOtherThan(copy.getName(), () -> randomAlphaOfLengthBetween(1, 16)))); modifiers.add(() -> { diff --git a/server/src/test/java/org/elasticsearch/index/query/NestedQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/NestedQueryBuilderTests.java index ac9ae8d0fa7fb..76479791283b4 100644 --- a/server/src/test/java/org/elasticsearch/index/query/NestedQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/NestedQueryBuilderTests.java @@ -45,7 +45,7 @@ import java.util.Map; import static org.elasticsearch.index.IndexSettingsTests.newIndexMeta; -import static org.elasticsearch.index.query.InnerHitBuilderTests.randomNestedInnerHits; +import static org.elasticsearch.index.query.InnerHitBuilderTests.randomInnerHits; import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.instanceOf; @@ -267,7 +267,7 @@ public void testThatUnrecognizedFromStringThrowsException() { } public void testInlineLeafInnerHitsNestedQuery() throws Exception { - InnerHitBuilder leafInnerHits = randomNestedInnerHits(); + InnerHitBuilder leafInnerHits = randomInnerHits(); NestedQueryBuilder nestedQueryBuilder = new NestedQueryBuilder("path", new MatchAllQueryBuilder(), ScoreMode.None); nestedQueryBuilder.innerHit(leafInnerHits); Map innerHitBuilders = new HashMap<>(); @@ -276,7 +276,7 @@ public void testInlineLeafInnerHitsNestedQuery() throws Exception { } public void testInlineLeafInnerHitsNestedQueryViaBoolQuery() { - InnerHitBuilder leafInnerHits = randomNestedInnerHits(); + InnerHitBuilder leafInnerHits = randomInnerHits(); NestedQueryBuilder nestedQueryBuilder = new NestedQueryBuilder("path", new MatchAllQueryBuilder(), ScoreMode.None) .innerHit(leafInnerHits); BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder().should(nestedQueryBuilder); @@ -286,7 +286,7 @@ public void testInlineLeafInnerHitsNestedQueryViaBoolQuery() { } public void testInlineLeafInnerHitsNestedQueryViaConstantScoreQuery() { - InnerHitBuilder leafInnerHits = randomNestedInnerHits(); + InnerHitBuilder leafInnerHits = randomInnerHits(); NestedQueryBuilder nestedQueryBuilder = new NestedQueryBuilder("path", new MatchAllQueryBuilder(), ScoreMode.None) .innerHit(leafInnerHits); ConstantScoreQueryBuilder constantScoreQueryBuilder = new ConstantScoreQueryBuilder(nestedQueryBuilder); @@ -296,10 +296,10 @@ public void testInlineLeafInnerHitsNestedQueryViaConstantScoreQuery() { } public void testInlineLeafInnerHitsNestedQueryViaBoostingQuery() { - InnerHitBuilder leafInnerHits1 = randomNestedInnerHits(); + InnerHitBuilder leafInnerHits1 = randomInnerHits(); NestedQueryBuilder nestedQueryBuilder1 = new NestedQueryBuilder("path", new MatchAllQueryBuilder(), ScoreMode.None) .innerHit(leafInnerHits1); - InnerHitBuilder leafInnerHits2 = randomNestedInnerHits(); + InnerHitBuilder leafInnerHits2 = randomInnerHits(); NestedQueryBuilder nestedQueryBuilder2 = new NestedQueryBuilder("path", new MatchAllQueryBuilder(), ScoreMode.None) .innerHit(leafInnerHits2); BoostingQueryBuilder constantScoreQueryBuilder = new BoostingQueryBuilder(nestedQueryBuilder1, nestedQueryBuilder2); @@ -310,7 +310,7 @@ public void testInlineLeafInnerHitsNestedQueryViaBoostingQuery() { } public void testInlineLeafInnerHitsNestedQueryViaFunctionScoreQuery() { - InnerHitBuilder leafInnerHits = randomNestedInnerHits(); + InnerHitBuilder leafInnerHits = randomInnerHits(); NestedQueryBuilder nestedQueryBuilder = new NestedQueryBuilder("path", new MatchAllQueryBuilder(), ScoreMode.None) .innerHit(leafInnerHits); FunctionScoreQueryBuilder functionScoreQueryBuilder = new FunctionScoreQueryBuilder(nestedQueryBuilder); @@ -330,7 +330,7 @@ public void testBuildIgnoreUnmappedNestQuery() throws Exception { when(mapperService.getIndexSettings()).thenReturn(settings); when(searchContext.mapperService()).thenReturn(mapperService); - InnerHitBuilder leafInnerHits = randomNestedInnerHits(); + InnerHitBuilder leafInnerHits = randomInnerHits(); NestedQueryBuilder query1 = new NestedQueryBuilder("path", new MatchAllQueryBuilder(), ScoreMode.None); query1.innerHit(leafInnerHits); final Map innerHitBuilders = new HashMap<>(); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/TopHitsIT.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/TopHitsIT.java index 4a5982b9dacfa..bb275650712d7 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/TopHitsIT.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/TopHitsIT.java @@ -31,7 +31,6 @@ import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.query.MatchAllQueryBuilder; import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.index.seqno.SequenceNumbers; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.script.MockScriptEngine; import org.elasticsearch.script.MockScriptPlugin; @@ -580,7 +579,6 @@ public void testFieldCollapsing() throws Exception { } public void testFetchFeatures() { - final boolean seqNoAndTerm = randomBoolean(); SearchResponse response = client().prepareSearch("idx") .setQuery(matchQuery("text", "text").queryName("test")) .addAggregation(terms("terms") @@ -596,7 +594,6 @@ public void testFetchFeatures() { new Script(ScriptType.INLINE, MockScriptEngine.NAME, "5", Collections.emptyMap())) .fetchSource("text", null) .version(true) - .seqNoAndPrimaryTerm(seqNoAndTerm) ) ) .get(); @@ -624,13 +621,8 @@ public void testFetchFeatures() { long version = hit.getVersion(); assertThat(version, equalTo(1L)); - if (seqNoAndTerm) { - assertThat(hit.getSeqNo(), greaterThanOrEqualTo(0L)); - assertThat(hit.getPrimaryTerm(), greaterThanOrEqualTo(1L)); - } else { - assertThat(hit.getSeqNo(), equalTo(SequenceNumbers.UNASSIGNED_SEQ_NO)); - assertThat(hit.getPrimaryTerm(), equalTo(SequenceNumbers.UNASSIGNED_PRIMARY_TERM)); - } + assertThat(hit.getSeqNo(), greaterThanOrEqualTo(0L)); + assertThat(hit.getPrimaryTerm(), greaterThanOrEqualTo(1L)); assertThat(hit.getMatchedQueries()[0], equalTo("test")); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/TopHitsTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/TopHitsTests.java index 6ec1ea1cad301..caf1d4ef4ee47 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/TopHitsTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/TopHitsTests.java @@ -54,9 +54,6 @@ protected final TopHitsAggregationBuilder createTestAggregatorBuilder() { if (randomBoolean()) { factory.version(randomBoolean()); } - if (randomBoolean()) { - factory.seqNoAndPrimaryTerm(randomBoolean()); - } if (randomBoolean()) { factory.trackScores(randomBoolean()); } diff --git a/test/framework/src/main/java/org/elasticsearch/search/RandomSearchRequestGenerator.java b/test/framework/src/main/java/org/elasticsearch/search/RandomSearchRequestGenerator.java index 6ec2732aaf915..5d96cd37b054d 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/RandomSearchRequestGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/search/RandomSearchRequestGenerator.java @@ -145,9 +145,6 @@ public static SearchSourceBuilder randomSearchSourceBuilder( if (randomBoolean()) { builder.version(randomBoolean()); } - if (randomBoolean()) { - builder.seqNoAndPrimaryTerm(randomBoolean()); - } if (randomBoolean()) { builder.trackScores(randomBoolean()); } diff --git a/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java b/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java index 6a17f65790368..18edb5ec3790a 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java +++ b/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java @@ -504,16 +504,6 @@ public boolean version() { public void version(boolean version) { } - @Override - public boolean seqNoAndPrimaryTerm() { - return false; - } - - @Override - public void seqNoAndPrimaryTerm(boolean seqNoAndPrimaryTerm) { - - } - @Override public int[] docIdsToLoad() { return new int[0]; From 3d745cac719129c452e744ae7eef0bbba0ca08e9 Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Sat, 19 Jan 2019 15:23:35 +0100 Subject: [PATCH 16/29] add seq no + primary term to examples --- .../metrics/tophits-aggregation.asciidoc | 11 +++++++++++ docs/reference/aggregations/misc.asciidoc | 2 ++ .../charfilters/pattern-replace-charfilter.asciidoc | 2 ++ .../analysis/tokenizers/edgengram-tokenizer.asciidoc | 2 ++ docs/reference/getting-started.asciidoc | 8 ++++++++ docs/reference/how-to/recipes/stemming.asciidoc | 8 ++++++++ docs/reference/index-modules/similarity.asciidoc | 4 ++++ docs/reference/ingest/processors/geoip.asciidoc | 2 ++ docs/reference/mapping/params/normalizer.asciidoc | 4 ++++ docs/reference/mapping/types/parent-join.asciidoc | 8 ++++++++ docs/reference/mapping/types/percolator.asciidoc | 6 ++++++ docs/reference/mapping/types/range.asciidoc | 4 ++++ docs/reference/modules/cross-cluster-search.asciidoc | 12 ++++++++++++ docs/reference/query-dsl/percolate-query.asciidoc | 12 ++++++++++++ docs/reference/query-dsl/terms-set-query.asciidoc | 2 ++ docs/reference/search/rank-eval.asciidoc | 2 ++ docs/reference/search/request-body.asciidoc | 2 ++ docs/reference/search/request/highlighting.asciidoc | 4 ++++ docs/reference/search/request/inner-hits.asciidoc | 10 ++++++++++ .../search/suggesters/completion-suggest.asciidoc | 4 ++++ docs/reference/search/uri-request.asciidoc | 2 ++ 21 files changed, 111 insertions(+) diff --git a/docs/reference/aggregations/metrics/tophits-aggregation.asciidoc b/docs/reference/aggregations/metrics/tophits-aggregation.asciidoc index 485e900d9628c..57863eb02667f 100644 --- a/docs/reference/aggregations/metrics/tophits-aggregation.asciidoc +++ b/docs/reference/aggregations/metrics/tophits-aggregation.asciidoc @@ -91,6 +91,8 @@ Possible response: "_index": "sales", "_type": "_doc", "_id": "AVnNBmauCQpcRyxw6ChK", + "_seq_no": 123, + "_primary_term": 1, "_source": { "date": "2015/03/01 00:00:00", "price": 200 @@ -119,6 +121,8 @@ Possible response: "_index": "sales", "_type": "_doc", "_id": "AVnNBmauCQpcRyxw6ChL", + "_seq_no": 324, + "_primary_term": 1, "_source": { "date": "2015/03/01 00:00:00", "price": 175 @@ -147,6 +151,8 @@ Possible response: "_index": "sales", "_type": "_doc", "_id": "AVnNBmatCQpcRyxw6ChH", + "_seq_no": 545, + "_primary_term": 1, "_source": { "date": "2015/01/01 00:00:00", "price": 150 @@ -169,6 +175,9 @@ Possible response: // TESTRESPONSE[s/AVnNBmauCQpcRyxw6ChK/$body.aggregations.top_tags.buckets.0.top_sales_hits.hits.hits.0._id/] // TESTRESPONSE[s/AVnNBmauCQpcRyxw6ChL/$body.aggregations.top_tags.buckets.1.top_sales_hits.hits.hits.0._id/] // TESTRESPONSE[s/AVnNBmatCQpcRyxw6ChH/$body.aggregations.top_tags.buckets.2.top_sales_hits.hits.hits.0._id/] +// TESTRESPONSE[s/"_seq_no": 123/"_seq_no": $body.aggregations.top_tags.buckets.0.top_sales_hits.hits.hits.0._seq_no/] +// TESTRESPONSE[s/"_seq_no": 324/"_seq_no": $body.aggregations.top_tags.buckets.1.top_sales_hits.hits.hits.0._seq_no/] +// TESTRESPONSE[s/"_seq_no": 545/"_seq_no": $body.aggregations.top_tags.buckets.2.top_sales_hits.hits.hits.0._seq_no/] ==== Field collapse example @@ -391,6 +400,8 @@ the second slow of the `nested_child_field` field: "_index": "a", "_type": "b", "_id": "1", + "_seq_no": 0, + "_primary_term": 1, "_score": 1, "_nested" : { "field" : "nested_child_field", diff --git a/docs/reference/aggregations/misc.asciidoc b/docs/reference/aggregations/misc.asciidoc index 288643dbf9313..4db1d5efbc886 100644 --- a/docs/reference/aggregations/misc.asciidoc +++ b/docs/reference/aggregations/misc.asciidoc @@ -143,6 +143,8 @@ In the response, the aggregations names will be changed to respectively `date_hi "_index": "twitter", "_type": "_doc", "_id": "0", + "_seq_no": 0, + "_primary_term": 1, "_score": 1.0, "_source": { "date": "2009-11-15T14:12:12", diff --git a/docs/reference/analysis/charfilters/pattern-replace-charfilter.asciidoc b/docs/reference/analysis/charfilters/pattern-replace-charfilter.asciidoc index adfde27138af8..a6adb818eb9c6 100644 --- a/docs/reference/analysis/charfilters/pattern-replace-charfilter.asciidoc +++ b/docs/reference/analysis/charfilters/pattern-replace-charfilter.asciidoc @@ -251,6 +251,8 @@ The output from the above is: "_index": "my_index", "_type": "_doc", "_id": "1", + "_seq_no": 0, + "_primary_term": 1, "_score": 0.2876821, "_source": { "text": "The fooBarBaz method" diff --git a/docs/reference/analysis/tokenizers/edgengram-tokenizer.asciidoc b/docs/reference/analysis/tokenizers/edgengram-tokenizer.asciidoc index 14f94ff21d3dc..352ec781a3805 100644 --- a/docs/reference/analysis/tokenizers/edgengram-tokenizer.asciidoc +++ b/docs/reference/analysis/tokenizers/edgengram-tokenizer.asciidoc @@ -310,6 +310,8 @@ GET my_index/_search "_index": "my_index", "_type": "_doc", "_id": "1", + "_seq_no": 0, + "_primary_term": 1, "_score": 0.5753642, "_source": { "title": "Quick Foxes" diff --git a/docs/reference/getting-started.asciidoc b/docs/reference/getting-started.asciidoc index b79dd5c36c244..c65e8fb5c0690 100755 --- a/docs/reference/getting-started.asciidoc +++ b/docs/reference/getting-started.asciidoc @@ -761,6 +761,8 @@ And the response (partially shown): "_type" : "_doc", "_id" : "0", "sort": [0], + "_primary_term": 1, + "_seq_no": 124, "_score" : null, "_source" : {"account_number":0,"balance":16623,"firstname":"Bradshaw","lastname":"Mckenzie","age":29,"gender":"F","address":"244 Columbus Place","employer":"Euron","email":"bradshawmckenzie@euron.com","city":"Hobucken","state":"CO"} }, { @@ -768,6 +770,8 @@ And the response (partially shown): "_type" : "_doc", "_id" : "1", "sort": [1], + "_primary_term": 1, + "_seq_no": 0, "_score" : null, "_source" : {"account_number":1,"balance":39225,"firstname":"Amber","lastname":"Duke","age":32,"gender":"M","address":"880 Holmes Lane","employer":"Pyrami","email":"amberduke@pyrami.com","city":"Brogan","state":"IL"} }, ... @@ -838,6 +842,8 @@ to clutter the docs with it: "_type" : "_doc", "_id" : "0", "sort": [0], + "_primary_term": 1, + "_seq_no": 124, "_score": null, "_source" : {"account_number":0,"balance":16623,"firstname":"Bradshaw","lastname":"Mckenzie","age":29,"gender":"F","address":"244 Columbus Place","employer":"Euron","email":"bradshawmckenzie@euron.com","city":"Hobucken","state":"CO"} }, { @@ -845,6 +851,8 @@ to clutter the docs with it: "_type" : "_doc", "_id" : "1", "sort": [1], + "_primary_term": 1, + "_seq_no": 0, "_score": null, "_source" : {"account_number":1,"balance":39225,"firstname":"Amber","lastname":"Duke","age":32,"gender":"M","address":"880 Holmes Lane","employer":"Pyrami","email":"amberduke@pyrami.com","city":"Brogan","state":"IL"} }, ... diff --git a/docs/reference/how-to/recipes/stemming.asciidoc b/docs/reference/how-to/recipes/stemming.asciidoc index c0ec3f2a04c09..699ea541f1f24 100644 --- a/docs/reference/how-to/recipes/stemming.asciidoc +++ b/docs/reference/how-to/recipes/stemming.asciidoc @@ -94,6 +94,8 @@ GET index/_search "_index": "index", "_type": "_doc", "_id": "1", + "_seq_no": 0, + "_primary_term": 1, "_score": 0.18232156, "_source": { "body": "Ski resort" @@ -103,6 +105,8 @@ GET index/_search "_index": "index", "_type": "_doc", "_id": "2", + "_seq_no": 1, + "_primary_term": 1, "_score": 0.18232156, "_source": { "body": "A pair of skis" @@ -155,6 +159,8 @@ GET index/_search "_index": "index", "_type": "_doc", "_id": "1", + "_seq_no": 0, + "_primary_term": 1, "_score": 0.8025915, "_source": { "body": "Ski resort" @@ -215,6 +221,8 @@ GET index/_search "_index": "index", "_type": "_doc", "_id": "1", + "_seq_no": 0, + "_primary_term": 1, "_score": 0.8025915, "_source": { "body": "Ski resort" diff --git a/docs/reference/index-modules/similarity.asciidoc b/docs/reference/index-modules/similarity.asciidoc index 4d2404c3b52b0..ba27744c8ac6c 100644 --- a/docs/reference/index-modules/similarity.asciidoc +++ b/docs/reference/index-modules/similarity.asciidoc @@ -275,6 +275,8 @@ Which yields: "_index": "index", "_type": "_doc", "_id": "1", + "_seq_no": 0, + "_primary_term": 1, "_score": 1.9508477, "_source": { "field": "foo bar foo" @@ -452,6 +454,8 @@ GET /index/_search?explain=true "_index": "index", "_type": "_doc", "_id": "1", + "_seq_no": 0, + "_primary_term": 1, "_score": 1.9508477, "_source": { "field": "foo bar foo" diff --git a/docs/reference/ingest/processors/geoip.asciidoc b/docs/reference/ingest/processors/geoip.asciidoc index f9f902395c4b2..76c3e30535fec 100644 --- a/docs/reference/ingest/processors/geoip.asciidoc +++ b/docs/reference/ingest/processors/geoip.asciidoc @@ -278,6 +278,8 @@ GET /my_ip_locations/_search "_index" : "my_ip_locations", "_type" : "_doc", "_id" : "1", + "_seq_no": 0, + "_primary_term": 1, "_score" : 1.0, "_source" : { "geoip" : { diff --git a/docs/reference/mapping/params/normalizer.asciidoc b/docs/reference/mapping/params/normalizer.asciidoc index 79ba39e194726..6a98e23ccc3d4 100644 --- a/docs/reference/mapping/params/normalizer.asciidoc +++ b/docs/reference/mapping/params/normalizer.asciidoc @@ -99,6 +99,8 @@ both index and query time. "_index": "index", "_type": "_doc", "_id": "1", + "_seq_no": 0, + "_primary_term": 1, "_score": 0.47000363, "_source": { "foo": "BÀR" @@ -108,6 +110,8 @@ both index and query time. "_index": "index", "_type": "_doc", "_id": "2", + "_seq_no": 1, + "_primary_term": 1, "_score": 0.47000363, "_source": { "foo": "bar" diff --git a/docs/reference/mapping/types/parent-join.asciidoc b/docs/reference/mapping/types/parent-join.asciidoc index dacef7c4bc7cb..561d747283821 100644 --- a/docs/reference/mapping/types/parent-join.asciidoc +++ b/docs/reference/mapping/types/parent-join.asciidoc @@ -185,6 +185,8 @@ Will return: "_index": "my_index", "_type": "_doc", "_id": "1", + "_seq_no": 2, + "_primary_term": 1, "_score": null, "_source": { "text": "This is a question", @@ -198,6 +200,8 @@ Will return: "_index": "my_index", "_type": "_doc", "_id": "2", + "_seq_no": 3, + "_primary_term": 1, "_score": null, "_source": { "text": "This is another question", @@ -211,6 +215,8 @@ Will return: "_index": "my_index", "_type": "_doc", "_id": "3", + "_seq_no": 4, + "_primary_term": 1, "_score": null, "_routing": "1", "_source": { @@ -228,6 +234,8 @@ Will return: "_index": "my_index", "_type": "_doc", "_id": "4", + "_seq_no": 5, + "_primary_term": 1, "_score": null, "_routing": "1", "_source": { diff --git a/docs/reference/mapping/types/percolator.asciidoc b/docs/reference/mapping/types/percolator.asciidoc index 7324826eb44ef..74a2ad2a2433b 100644 --- a/docs/reference/mapping/types/percolator.asciidoc +++ b/docs/reference/mapping/types/percolator.asciidoc @@ -209,6 +209,8 @@ now returns matches from the new index: "_index": "new_index", <1> "_type": "_doc", "_id": "1", + "_seq_no": 0, + "_primary_term": 1, "_score": 0.13076457, "_source": { "query": { @@ -407,6 +409,8 @@ This results in a response like this: "_index": "test_index", "_type": "_doc", "_id": "1", + "_seq_no": 0, + "_primary_term": 1, "_score": 0.13076457, "_source": { "query": { @@ -569,6 +573,8 @@ GET /my_queries1/_search "_index": "my_queries1", "_type": "_doc", "_id": "1", + "_seq_no": 0, + "_primary_term": 1, "_score": 0.18864399, "_source": { "query": { diff --git a/docs/reference/mapping/types/range.asciidoc b/docs/reference/mapping/types/range.asciidoc index 630458b4866e5..9712ef01ec4ef 100644 --- a/docs/reference/mapping/types/range.asciidoc +++ b/docs/reference/mapping/types/range.asciidoc @@ -97,6 +97,8 @@ The result produced by the above query. "_index" : "range_index", "_type" : "_doc", "_id" : "1", + "_seq_no": 0, + "_primary_term": 1, "_score" : 1.0, "_source" : { "expected_attendees" : { @@ -160,6 +162,8 @@ This query produces a similar result: "_index" : "range_index", "_type" : "_doc", "_id" : "1", + "_seq_no": 0, + "_primary_term": 1, "_score" : 1.0, "_source" : { "expected_attendees" : { diff --git a/docs/reference/modules/cross-cluster-search.asciidoc b/docs/reference/modules/cross-cluster-search.asciidoc index 61b0bb50aedc7..4eeb23dd56844 100644 --- a/docs/reference/modules/cross-cluster-search.asciidoc +++ b/docs/reference/modules/cross-cluster-search.asciidoc @@ -87,6 +87,8 @@ GET /cluster_one:twitter/_search "_index": "cluster_one:twitter", "_type": "_doc", "_id": "0", + "_seq_no": 0, + "_primary_term": 1, "_score": 1, "_source": { "user": "kimchy", @@ -152,6 +154,8 @@ will be prefixed with their remote cluster name: "_index": "cluster_one:twitter", "_type": "_doc", "_id": "0", + "_seq_no": 0, + "_primary_term": 1, "_score": 1, "_source": { "user": "kimchy", @@ -164,6 +168,8 @@ will be prefixed with their remote cluster name: "_index": "twitter", "_type": "_doc", "_id": "0", + "_seq_no": 0, + "_primary_term": 1, "_score": 2, "_source": { "user": "kimchy", @@ -244,6 +250,8 @@ GET /cluster_one:twitter,cluster_two:twitter,twitter/_search <1> "_index": "cluster_one:twitter", "_type": "_doc", "_id": "0", + "_seq_no": 0, + "_primary_term": 1, "_score": 1, "_source": { "user": "kimchy", @@ -256,6 +264,8 @@ GET /cluster_one:twitter,cluster_two:twitter,twitter/_search <1> "_index": "twitter", "_type": "_doc", "_id": "0", + "_seq_no": 10, + "_primary_term": 1, "_score": 2, "_source": { "user": "kimchy", @@ -272,4 +282,6 @@ GET /cluster_one:twitter,cluster_two:twitter,twitter/_search <1> // TESTRESPONSE[s/"max_score": 1/"max_score": "$body.hits.max_score"/] // TESTRESPONSE[s/"_score": 1/"_score": "$body.hits.hits.0._score"/] // TESTRESPONSE[s/"_score": 2/"_score": "$body.hits.hits.1._score"/] +// TESTRESPONSE[s/"_seq_no": 0/"_seq_no": "$body.hits.hits.0._seq_no"/] +// TESTRESPONSE[s/"_seq_no": 10/"_seq_no": "$body.hits.hits.1._seq_no"/] <1> The `clusters` section indicates that one cluster was unavailable and got skipped diff --git a/docs/reference/query-dsl/percolate-query.asciidoc b/docs/reference/query-dsl/percolate-query.asciidoc index 9ae6dd802137c..39d141827c686 100644 --- a/docs/reference/query-dsl/percolate-query.asciidoc +++ b/docs/reference/query-dsl/percolate-query.asciidoc @@ -99,6 +99,8 @@ The above request will yield the following response: "_index": "my-index", "_type": "_doc", "_id": "1", + "_seq_no": 0, + "_primary_term": 1, "_score": 0.26152915, "_source": { "query": { @@ -250,6 +252,8 @@ GET /my-index/_search "_index": "my-index", "_type": "_doc", "_id": "1", + "_seq_no": 0, + "_primary_term": 1, "_score": 0.7093853, "_source": { "query": { @@ -433,6 +437,8 @@ This will yield the following response. "_index": "my-index", "_type": "_doc", "_id": "3", + "_seq_no": 2, + "_primary_term": 1, "_score": 0.26152915, "_source": { "query": { @@ -454,6 +460,8 @@ This will yield the following response. "_index": "my-index", "_type": "_doc", "_id": "4", + "_seq_no": 3, + "_primary_term": 1, "_score": 0.26152915, "_source": { "query": { @@ -541,6 +549,8 @@ The slightly different response: "_index": "my-index", "_type": "_doc", "_id": "1", + "_seq_no": 0, + "_primary_term": 1, "_score": 0.7093853, "_source": { "query": { @@ -640,6 +650,8 @@ The above search request returns a response similar to this: "_index": "my-index", "_type": "_doc", "_id": "1", + "_seq_no": 0, + "_primary_term": 1, "_score": 0.26152915, "_source": { "query": { diff --git a/docs/reference/query-dsl/terms-set-query.asciidoc b/docs/reference/query-dsl/terms-set-query.asciidoc index 5c0e00ce360fa..8c7fb929200eb 100644 --- a/docs/reference/query-dsl/terms-set-query.asciidoc +++ b/docs/reference/query-dsl/terms-set-query.asciidoc @@ -82,6 +82,8 @@ Response: "_index": "my-index", "_type": "_doc", "_id": "2", + "_seq_no": 1, + "_primary_term": 1, "_score": 0.87546873, "_source": { "codes": ["def", "ghi"], diff --git a/docs/reference/search/rank-eval.asciidoc b/docs/reference/search/rank-eval.asciidoc index c549b5e7a689b..2c8123ddbb9dc 100644 --- a/docs/reference/search/rank-eval.asciidoc +++ b/docs/reference/search/rank-eval.asciidoc @@ -336,6 +336,8 @@ that shows potential errors of individual queries. The response has the followin "_index": "my_index", "_type": "page", "_id": "1528558", + "_seq_no": 0, + "_primary_term": 1, "_score": 7.0556192 }, "rating": 1 diff --git a/docs/reference/search/request-body.asciidoc b/docs/reference/search/request-body.asciidoc index 9970c4cc6223f..3a6b1f551f13c 100644 --- a/docs/reference/search/request-body.asciidoc +++ b/docs/reference/search/request-body.asciidoc @@ -41,6 +41,8 @@ And here is a sample response: "_index" : "twitter", "_type" : "_doc", "_id" : "0", + "_seq_no": 0, + "_primary_term": 1, "_score": 1.3862944, "_source" : { "user" : "kimchy", diff --git a/docs/reference/search/request/highlighting.asciidoc b/docs/reference/search/request/highlighting.asciidoc index 4dd037cf3c001..706c841f28d11 100644 --- a/docs/reference/search/request/highlighting.asciidoc +++ b/docs/reference/search/request/highlighting.asciidoc @@ -870,6 +870,8 @@ Response: "_index": "twitter", "_type": "_doc", "_id": "1", + "_seq_no": 1, + "_primary_term": 1, "_score": 1.601195, "_source": { "user": "test", @@ -929,6 +931,8 @@ Response: "_index": "twitter", "_type": "_doc", "_id": "1", + "_seq_no": 1, + "_primary_term": 1, "_score": 1.601195, "_source": { "user": "test", diff --git a/docs/reference/search/request/inner-hits.asciidoc b/docs/reference/search/request/inner-hits.asciidoc index b67377edfe901..40e7cdeba29a5 100644 --- a/docs/reference/search/request/inner-hits.asciidoc +++ b/docs/reference/search/request/inner-hits.asciidoc @@ -146,6 +146,8 @@ An example of a response snippet that could be generated from the above search r "_index": "test", "_type": "_doc", "_id": "1", + "_seq_no": 0, + "_primary_term": 1, "_score": 1.0, "_source": ..., "inner_hits": { @@ -280,6 +282,8 @@ Response not included in text but tested for completeness sake. "_index": "test", "_type": "_doc", "_id": "1", + "_seq_no": 0, + "_primary_term": 1, "_score": 1.0444684, "_source": ..., "inner_hits": { @@ -401,6 +405,8 @@ Which would look like: "_index": "test", "_type": "_doc", "_id": "1", + "_seq_no": 0, + "_primary_term": 1, "_score": 0.6931472, "_source": ..., "inner_hits": { @@ -518,6 +524,8 @@ An example of a response snippet that could be generated from the above search r "_index": "test", "_type": "_doc", "_id": "1", + "_seq_no": 0, + "_primary_term": 1, "_score": 1.0, "_source": { "number": 1, @@ -536,6 +544,8 @@ An example of a response snippet that could be generated from the above search r "_index": "test", "_type": "_doc", "_id": "2", + "_seq_no": 1, + "_primary_term": 1, "_score": 1.0, "_routing": "1", "_source": { diff --git a/docs/reference/search/suggesters/completion-suggest.asciidoc b/docs/reference/search/suggesters/completion-suggest.asciidoc index 191f01f17394d..826fd8f73dd59 100644 --- a/docs/reference/search/suggesters/completion-suggest.asciidoc +++ b/docs/reference/search/suggesters/completion-suggest.asciidoc @@ -195,6 +195,8 @@ returns this response: "_index": "music", "_type": "_doc", "_id": "1", + "_seq_no": 2, + "_primary_term": 1, "_score": 1.0, "_source": { "suggest": ["Nevermind", "Nirvana"] @@ -274,6 +276,8 @@ Which should look like: "_index": "music", "_type": "_doc", "_id": "1", + "_seq_no": 2, + "_primary_term": 1, "_score": 1.0, "_source": { "suggest": ["Nevermind", "Nirvana"] diff --git a/docs/reference/search/uri-request.asciidoc b/docs/reference/search/uri-request.asciidoc index 87e1da907fb7d..32ef816974e50 100644 --- a/docs/reference/search/uri-request.asciidoc +++ b/docs/reference/search/uri-request.asciidoc @@ -37,6 +37,8 @@ And here is a sample response: "_index" : "twitter", "_type" : "_doc", "_id" : "0", + "_seq_no": 0, + "_primary_term": 1, "_score": 1.3862944, "_source" : { "user" : "kimchy", From 772d8fc85a47ce73f38e9685aed0ab486b26cdd1 Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Sun, 20 Jan 2019 17:54:04 +0100 Subject: [PATCH 17/29] fix parent_child --- .../src/test/resources/rest-api-spec/test/11_parent_child.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/parent-join/src/test/resources/rest-api-spec/test/11_parent_child.yml b/modules/parent-join/src/test/resources/rest-api-spec/test/11_parent_child.yml index efdca604b2ed6..7d14cfdab8822 100644 --- a/modules/parent-join/src/test/resources/rest-api-spec/test/11_parent_child.yml +++ b/modules/parent-join/src/test/resources/rest-api-spec/test/11_parent_child.yml @@ -41,8 +41,6 @@ setup: - match: { hits.hits.0.inner_hits.child.hits.hits.0._index: "test"} - match: { hits.hits.0.inner_hits.child.hits.hits.0._id: "2" } - is_false: hits.hits.0.inner_hits.child.hits.hits.0._nested - - is_false: hits.hits.0.inner_hits.child.hits.hits.0._seq_no - - is_false: hits.hits.0.inner_hits.child.hits.hits.0._primary_term --- "Parent/child inner hits with seq no": @@ -60,3 +58,5 @@ setup: - match: { hits.hits.0.inner_hits.child.hits.hits.0._index: "test"} - match: { hits.hits.0.inner_hits.child.hits.hits.0._id: "2" } - is_false: hits.hits.0.inner_hits.child.hits.hits.0._nested + - gte: { hits.hits.0.inner_hits.child.hits.hits.0._seq_no: 0 } + - gte: { hits.hits.0.inner_hits.child.hits.hits.0._primary_term: 1 } From 7b38fe55903b669a54f5b2857658c7a4e6601b31 Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Sun, 20 Jan 2019 17:56:53 +0100 Subject: [PATCH 18/29] fix testMonitoringBulk --- .../xpack/monitoring/integration/MonitoringIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/integration/MonitoringIT.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/integration/MonitoringIT.java index 555f2659113fd..ac9c9cec93958 100644 --- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/integration/MonitoringIT.java +++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/integration/MonitoringIT.java @@ -260,7 +260,7 @@ private void assertMonitoringDoc(final Map document, final MonitoredSystem expectedSystem, final String expectedType, final TimeValue interval) { - assertEquals(document.toString(),4, document.size()); + assertEquals(document.toString(),6, document.size()); final String index = (String) document.get("_index"); assertThat(index, containsString(".monitoring-" + expectedSystem.getSystem() + "-" + TEMPLATE_VERSION + "-")); From 3400b4926826d8dca53d6c5e7b4f16074a30eb3a Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Sun, 20 Jan 2019 19:07:11 +0100 Subject: [PATCH 19/29] remove left over seq_no_and_primary_term --- .../test/resources/rest-api-spec/test/multi_cluster/10_basic.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/10_basic.yml b/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/10_basic.yml index 1cb601fc0e848..a4b6f503458c9 100644 --- a/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/10_basic.yml +++ b/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/10_basic.yml @@ -30,7 +30,6 @@ rest_total_hits_as_int: true index: test_index,my_remote_cluster:test_index body: - seq_no_and_primary_term: true aggs: cluster: terms: From cecf185656e30ab9bf14c2dd7f31e6836ab7ba63 Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Mon, 21 Jan 2019 10:34:37 +0100 Subject: [PATCH 20/29] feedback --- server/src/main/java/org/elasticsearch/search/SearchHit.java | 2 ++ .../test/java/org/elasticsearch/search/SearchHitTests.java | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/server/src/main/java/org/elasticsearch/search/SearchHit.java b/server/src/main/java/org/elasticsearch/search/SearchHit.java index 1a6e44fd1ab18..c88c4098cca72 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchHit.java +++ b/server/src/main/java/org/elasticsearch/search/SearchHit.java @@ -567,6 +567,8 @@ public static void declareInnerHitsParseFields(ObjectParser, parser.declareField((map, value) -> map.put(Fields._SCORE, value), SearchHit::parseScore, new ParseField(Fields._SCORE), ValueType.FLOAT_OR_NULL); parser.declareLong((map, value) -> map.put(Fields._VERSION, value), new ParseField(Fields._VERSION)); + parser.declareLong((map, value) -> map.put(Fields._SEQ_NO, value), new ParseField(Fields._SEQ_NO)); + parser.declareLong((map, value) -> map.put(Fields._PRIMARY_TERM, value), new ParseField(Fields._PRIMARY_TERM)); parser.declareField((map, value) -> map.put(Fields._SHARD, value), (p, c) -> ShardId.fromString(p.text()), new ParseField(Fields._SHARD), ValueType.STRING); parser.declareObject((map, value) -> map.put(SourceFieldMapper.NAME, value), (p, c) -> parseSourceBytes(p), diff --git a/server/src/test/java/org/elasticsearch/search/SearchHitTests.java b/server/src/test/java/org/elasticsearch/search/SearchHitTests.java index fee55f1e22f23..4831729201183 100644 --- a/server/src/test/java/org/elasticsearch/search/SearchHitTests.java +++ b/server/src/test/java/org/elasticsearch/search/SearchHitTests.java @@ -90,6 +90,11 @@ public static SearchHit createTestItem(XContentType xContentType, boolean withOp if (randomBoolean()) { hit.version(randomLong()); } + + if (randomBoolean()) { + hit.version(randomNonNegativeLong()); + hit.version(randomLongBetween(1, Long.MAX_VALUE)); + } if (randomBoolean()) { hit.sortValues(SearchSortValuesTests.createTestItem(xContentType, transportSerialization)); } From 44190757bd38bee4bc77bcbf6bdb6f5aab851650 Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Mon, 21 Jan 2019 13:25:24 +0100 Subject: [PATCH 21/29] Revert "remove left over seq_no_and_primary_term" This reverts commit 3400b4926826d8dca53d6c5e7b4f16074a30eb3a. --- .../test/resources/rest-api-spec/test/multi_cluster/10_basic.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/10_basic.yml b/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/10_basic.yml index a4b6f503458c9..1cb601fc0e848 100644 --- a/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/10_basic.yml +++ b/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/10_basic.yml @@ -30,6 +30,7 @@ rest_total_hits_as_int: true index: test_index,my_remote_cluster:test_index body: + seq_no_and_primary_term: true aggs: cluster: terms: From a076ca9a72280f8f6b681b8adcb274c0e614dd1a Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Mon, 21 Jan 2019 13:25:36 +0100 Subject: [PATCH 22/29] Revert "fix testMonitoringBulk" This reverts commit 7b38fe55903b669a54f5b2857658c7a4e6601b31. --- .../xpack/monitoring/integration/MonitoringIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/integration/MonitoringIT.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/integration/MonitoringIT.java index ac9c9cec93958..555f2659113fd 100644 --- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/integration/MonitoringIT.java +++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/integration/MonitoringIT.java @@ -260,7 +260,7 @@ private void assertMonitoringDoc(final Map document, final MonitoredSystem expectedSystem, final String expectedType, final TimeValue interval) { - assertEquals(document.toString(),6, document.size()); + assertEquals(document.toString(),4, document.size()); final String index = (String) document.get("_index"); assertThat(index, containsString(".monitoring-" + expectedSystem.getSystem() + "-" + TEMPLATE_VERSION + "-")); From 9a792b87de6bb1186120fc8aa656faa611b10932 Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Mon, 21 Jan 2019 13:25:59 +0100 Subject: [PATCH 23/29] Revert "add seq no + primary term to examples" This reverts commit 3d745cac719129c452e744ae7eef0bbba0ca08e9. --- .../metrics/tophits-aggregation.asciidoc | 11 ----------- docs/reference/aggregations/misc.asciidoc | 2 -- .../charfilters/pattern-replace-charfilter.asciidoc | 2 -- .../analysis/tokenizers/edgengram-tokenizer.asciidoc | 2 -- docs/reference/getting-started.asciidoc | 8 -------- docs/reference/how-to/recipes/stemming.asciidoc | 8 -------- docs/reference/index-modules/similarity.asciidoc | 4 ---- docs/reference/ingest/processors/geoip.asciidoc | 2 -- docs/reference/mapping/params/normalizer.asciidoc | 4 ---- docs/reference/mapping/types/parent-join.asciidoc | 8 -------- docs/reference/mapping/types/percolator.asciidoc | 6 ------ docs/reference/mapping/types/range.asciidoc | 4 ---- docs/reference/modules/cross-cluster-search.asciidoc | 12 ------------ docs/reference/query-dsl/percolate-query.asciidoc | 12 ------------ docs/reference/query-dsl/terms-set-query.asciidoc | 2 -- docs/reference/search/rank-eval.asciidoc | 2 -- docs/reference/search/request-body.asciidoc | 2 -- docs/reference/search/request/highlighting.asciidoc | 4 ---- docs/reference/search/request/inner-hits.asciidoc | 10 ---------- .../search/suggesters/completion-suggest.asciidoc | 4 ---- docs/reference/search/uri-request.asciidoc | 2 -- 21 files changed, 111 deletions(-) diff --git a/docs/reference/aggregations/metrics/tophits-aggregation.asciidoc b/docs/reference/aggregations/metrics/tophits-aggregation.asciidoc index 885c0089a6077..c3b6a9bad4cfc 100644 --- a/docs/reference/aggregations/metrics/tophits-aggregation.asciidoc +++ b/docs/reference/aggregations/metrics/tophits-aggregation.asciidoc @@ -91,8 +91,6 @@ Possible response: "_index": "sales", "_type": "_doc", "_id": "AVnNBmauCQpcRyxw6ChK", - "_seq_no": 123, - "_primary_term": 1, "_source": { "date": "2015/03/01 00:00:00", "price": 200 @@ -121,8 +119,6 @@ Possible response: "_index": "sales", "_type": "_doc", "_id": "AVnNBmauCQpcRyxw6ChL", - "_seq_no": 324, - "_primary_term": 1, "_source": { "date": "2015/03/01 00:00:00", "price": 175 @@ -151,8 +147,6 @@ Possible response: "_index": "sales", "_type": "_doc", "_id": "AVnNBmatCQpcRyxw6ChH", - "_seq_no": 545, - "_primary_term": 1, "_source": { "date": "2015/01/01 00:00:00", "price": 150 @@ -175,9 +169,6 @@ Possible response: // TESTRESPONSE[s/AVnNBmauCQpcRyxw6ChK/$body.aggregations.top_tags.buckets.0.top_sales_hits.hits.hits.0._id/] // TESTRESPONSE[s/AVnNBmauCQpcRyxw6ChL/$body.aggregations.top_tags.buckets.1.top_sales_hits.hits.hits.0._id/] // TESTRESPONSE[s/AVnNBmatCQpcRyxw6ChH/$body.aggregations.top_tags.buckets.2.top_sales_hits.hits.hits.0._id/] -// TESTRESPONSE[s/"_seq_no": 123/"_seq_no": $body.aggregations.top_tags.buckets.0.top_sales_hits.hits.hits.0._seq_no/] -// TESTRESPONSE[s/"_seq_no": 324/"_seq_no": $body.aggregations.top_tags.buckets.1.top_sales_hits.hits.hits.0._seq_no/] -// TESTRESPONSE[s/"_seq_no": 545/"_seq_no": $body.aggregations.top_tags.buckets.2.top_sales_hits.hits.hits.0._seq_no/] ==== Field collapse example @@ -398,8 +389,6 @@ the second slow of the `nested_child_field` field: "_index": "a", "_type": "b", "_id": "1", - "_seq_no": 0, - "_primary_term": 1, "_score": 1, "_nested" : { "field" : "nested_child_field", diff --git a/docs/reference/aggregations/misc.asciidoc b/docs/reference/aggregations/misc.asciidoc index 4db1d5efbc886..288643dbf9313 100644 --- a/docs/reference/aggregations/misc.asciidoc +++ b/docs/reference/aggregations/misc.asciidoc @@ -143,8 +143,6 @@ In the response, the aggregations names will be changed to respectively `date_hi "_index": "twitter", "_type": "_doc", "_id": "0", - "_seq_no": 0, - "_primary_term": 1, "_score": 1.0, "_source": { "date": "2009-11-15T14:12:12", diff --git a/docs/reference/analysis/charfilters/pattern-replace-charfilter.asciidoc b/docs/reference/analysis/charfilters/pattern-replace-charfilter.asciidoc index a6adb818eb9c6..adfde27138af8 100644 --- a/docs/reference/analysis/charfilters/pattern-replace-charfilter.asciidoc +++ b/docs/reference/analysis/charfilters/pattern-replace-charfilter.asciidoc @@ -251,8 +251,6 @@ The output from the above is: "_index": "my_index", "_type": "_doc", "_id": "1", - "_seq_no": 0, - "_primary_term": 1, "_score": 0.2876821, "_source": { "text": "The fooBarBaz method" diff --git a/docs/reference/analysis/tokenizers/edgengram-tokenizer.asciidoc b/docs/reference/analysis/tokenizers/edgengram-tokenizer.asciidoc index 352ec781a3805..14f94ff21d3dc 100644 --- a/docs/reference/analysis/tokenizers/edgengram-tokenizer.asciidoc +++ b/docs/reference/analysis/tokenizers/edgengram-tokenizer.asciidoc @@ -310,8 +310,6 @@ GET my_index/_search "_index": "my_index", "_type": "_doc", "_id": "1", - "_seq_no": 0, - "_primary_term": 1, "_score": 0.5753642, "_source": { "title": "Quick Foxes" diff --git a/docs/reference/getting-started.asciidoc b/docs/reference/getting-started.asciidoc index c65e8fb5c0690..b79dd5c36c244 100755 --- a/docs/reference/getting-started.asciidoc +++ b/docs/reference/getting-started.asciidoc @@ -761,8 +761,6 @@ And the response (partially shown): "_type" : "_doc", "_id" : "0", "sort": [0], - "_primary_term": 1, - "_seq_no": 124, "_score" : null, "_source" : {"account_number":0,"balance":16623,"firstname":"Bradshaw","lastname":"Mckenzie","age":29,"gender":"F","address":"244 Columbus Place","employer":"Euron","email":"bradshawmckenzie@euron.com","city":"Hobucken","state":"CO"} }, { @@ -770,8 +768,6 @@ And the response (partially shown): "_type" : "_doc", "_id" : "1", "sort": [1], - "_primary_term": 1, - "_seq_no": 0, "_score" : null, "_source" : {"account_number":1,"balance":39225,"firstname":"Amber","lastname":"Duke","age":32,"gender":"M","address":"880 Holmes Lane","employer":"Pyrami","email":"amberduke@pyrami.com","city":"Brogan","state":"IL"} }, ... @@ -842,8 +838,6 @@ to clutter the docs with it: "_type" : "_doc", "_id" : "0", "sort": [0], - "_primary_term": 1, - "_seq_no": 124, "_score": null, "_source" : {"account_number":0,"balance":16623,"firstname":"Bradshaw","lastname":"Mckenzie","age":29,"gender":"F","address":"244 Columbus Place","employer":"Euron","email":"bradshawmckenzie@euron.com","city":"Hobucken","state":"CO"} }, { @@ -851,8 +845,6 @@ to clutter the docs with it: "_type" : "_doc", "_id" : "1", "sort": [1], - "_primary_term": 1, - "_seq_no": 0, "_score": null, "_source" : {"account_number":1,"balance":39225,"firstname":"Amber","lastname":"Duke","age":32,"gender":"M","address":"880 Holmes Lane","employer":"Pyrami","email":"amberduke@pyrami.com","city":"Brogan","state":"IL"} }, ... diff --git a/docs/reference/how-to/recipes/stemming.asciidoc b/docs/reference/how-to/recipes/stemming.asciidoc index 699ea541f1f24..c0ec3f2a04c09 100644 --- a/docs/reference/how-to/recipes/stemming.asciidoc +++ b/docs/reference/how-to/recipes/stemming.asciidoc @@ -94,8 +94,6 @@ GET index/_search "_index": "index", "_type": "_doc", "_id": "1", - "_seq_no": 0, - "_primary_term": 1, "_score": 0.18232156, "_source": { "body": "Ski resort" @@ -105,8 +103,6 @@ GET index/_search "_index": "index", "_type": "_doc", "_id": "2", - "_seq_no": 1, - "_primary_term": 1, "_score": 0.18232156, "_source": { "body": "A pair of skis" @@ -159,8 +155,6 @@ GET index/_search "_index": "index", "_type": "_doc", "_id": "1", - "_seq_no": 0, - "_primary_term": 1, "_score": 0.8025915, "_source": { "body": "Ski resort" @@ -221,8 +215,6 @@ GET index/_search "_index": "index", "_type": "_doc", "_id": "1", - "_seq_no": 0, - "_primary_term": 1, "_score": 0.8025915, "_source": { "body": "Ski resort" diff --git a/docs/reference/index-modules/similarity.asciidoc b/docs/reference/index-modules/similarity.asciidoc index ba27744c8ac6c..4d2404c3b52b0 100644 --- a/docs/reference/index-modules/similarity.asciidoc +++ b/docs/reference/index-modules/similarity.asciidoc @@ -275,8 +275,6 @@ Which yields: "_index": "index", "_type": "_doc", "_id": "1", - "_seq_no": 0, - "_primary_term": 1, "_score": 1.9508477, "_source": { "field": "foo bar foo" @@ -454,8 +452,6 @@ GET /index/_search?explain=true "_index": "index", "_type": "_doc", "_id": "1", - "_seq_no": 0, - "_primary_term": 1, "_score": 1.9508477, "_source": { "field": "foo bar foo" diff --git a/docs/reference/ingest/processors/geoip.asciidoc b/docs/reference/ingest/processors/geoip.asciidoc index 76c3e30535fec..f9f902395c4b2 100644 --- a/docs/reference/ingest/processors/geoip.asciidoc +++ b/docs/reference/ingest/processors/geoip.asciidoc @@ -278,8 +278,6 @@ GET /my_ip_locations/_search "_index" : "my_ip_locations", "_type" : "_doc", "_id" : "1", - "_seq_no": 0, - "_primary_term": 1, "_score" : 1.0, "_source" : { "geoip" : { diff --git a/docs/reference/mapping/params/normalizer.asciidoc b/docs/reference/mapping/params/normalizer.asciidoc index 6a98e23ccc3d4..79ba39e194726 100644 --- a/docs/reference/mapping/params/normalizer.asciidoc +++ b/docs/reference/mapping/params/normalizer.asciidoc @@ -99,8 +99,6 @@ both index and query time. "_index": "index", "_type": "_doc", "_id": "1", - "_seq_no": 0, - "_primary_term": 1, "_score": 0.47000363, "_source": { "foo": "BÀR" @@ -110,8 +108,6 @@ both index and query time. "_index": "index", "_type": "_doc", "_id": "2", - "_seq_no": 1, - "_primary_term": 1, "_score": 0.47000363, "_source": { "foo": "bar" diff --git a/docs/reference/mapping/types/parent-join.asciidoc b/docs/reference/mapping/types/parent-join.asciidoc index 561d747283821..dacef7c4bc7cb 100644 --- a/docs/reference/mapping/types/parent-join.asciidoc +++ b/docs/reference/mapping/types/parent-join.asciidoc @@ -185,8 +185,6 @@ Will return: "_index": "my_index", "_type": "_doc", "_id": "1", - "_seq_no": 2, - "_primary_term": 1, "_score": null, "_source": { "text": "This is a question", @@ -200,8 +198,6 @@ Will return: "_index": "my_index", "_type": "_doc", "_id": "2", - "_seq_no": 3, - "_primary_term": 1, "_score": null, "_source": { "text": "This is another question", @@ -215,8 +211,6 @@ Will return: "_index": "my_index", "_type": "_doc", "_id": "3", - "_seq_no": 4, - "_primary_term": 1, "_score": null, "_routing": "1", "_source": { @@ -234,8 +228,6 @@ Will return: "_index": "my_index", "_type": "_doc", "_id": "4", - "_seq_no": 5, - "_primary_term": 1, "_score": null, "_routing": "1", "_source": { diff --git a/docs/reference/mapping/types/percolator.asciidoc b/docs/reference/mapping/types/percolator.asciidoc index 74a2ad2a2433b..7324826eb44ef 100644 --- a/docs/reference/mapping/types/percolator.asciidoc +++ b/docs/reference/mapping/types/percolator.asciidoc @@ -209,8 +209,6 @@ now returns matches from the new index: "_index": "new_index", <1> "_type": "_doc", "_id": "1", - "_seq_no": 0, - "_primary_term": 1, "_score": 0.13076457, "_source": { "query": { @@ -409,8 +407,6 @@ This results in a response like this: "_index": "test_index", "_type": "_doc", "_id": "1", - "_seq_no": 0, - "_primary_term": 1, "_score": 0.13076457, "_source": { "query": { @@ -573,8 +569,6 @@ GET /my_queries1/_search "_index": "my_queries1", "_type": "_doc", "_id": "1", - "_seq_no": 0, - "_primary_term": 1, "_score": 0.18864399, "_source": { "query": { diff --git a/docs/reference/mapping/types/range.asciidoc b/docs/reference/mapping/types/range.asciidoc index 9712ef01ec4ef..630458b4866e5 100644 --- a/docs/reference/mapping/types/range.asciidoc +++ b/docs/reference/mapping/types/range.asciidoc @@ -97,8 +97,6 @@ The result produced by the above query. "_index" : "range_index", "_type" : "_doc", "_id" : "1", - "_seq_no": 0, - "_primary_term": 1, "_score" : 1.0, "_source" : { "expected_attendees" : { @@ -162,8 +160,6 @@ This query produces a similar result: "_index" : "range_index", "_type" : "_doc", "_id" : "1", - "_seq_no": 0, - "_primary_term": 1, "_score" : 1.0, "_source" : { "expected_attendees" : { diff --git a/docs/reference/modules/cross-cluster-search.asciidoc b/docs/reference/modules/cross-cluster-search.asciidoc index 4eeb23dd56844..61b0bb50aedc7 100644 --- a/docs/reference/modules/cross-cluster-search.asciidoc +++ b/docs/reference/modules/cross-cluster-search.asciidoc @@ -87,8 +87,6 @@ GET /cluster_one:twitter/_search "_index": "cluster_one:twitter", "_type": "_doc", "_id": "0", - "_seq_no": 0, - "_primary_term": 1, "_score": 1, "_source": { "user": "kimchy", @@ -154,8 +152,6 @@ will be prefixed with their remote cluster name: "_index": "cluster_one:twitter", "_type": "_doc", "_id": "0", - "_seq_no": 0, - "_primary_term": 1, "_score": 1, "_source": { "user": "kimchy", @@ -168,8 +164,6 @@ will be prefixed with their remote cluster name: "_index": "twitter", "_type": "_doc", "_id": "0", - "_seq_no": 0, - "_primary_term": 1, "_score": 2, "_source": { "user": "kimchy", @@ -250,8 +244,6 @@ GET /cluster_one:twitter,cluster_two:twitter,twitter/_search <1> "_index": "cluster_one:twitter", "_type": "_doc", "_id": "0", - "_seq_no": 0, - "_primary_term": 1, "_score": 1, "_source": { "user": "kimchy", @@ -264,8 +256,6 @@ GET /cluster_one:twitter,cluster_two:twitter,twitter/_search <1> "_index": "twitter", "_type": "_doc", "_id": "0", - "_seq_no": 10, - "_primary_term": 1, "_score": 2, "_source": { "user": "kimchy", @@ -282,6 +272,4 @@ GET /cluster_one:twitter,cluster_two:twitter,twitter/_search <1> // TESTRESPONSE[s/"max_score": 1/"max_score": "$body.hits.max_score"/] // TESTRESPONSE[s/"_score": 1/"_score": "$body.hits.hits.0._score"/] // TESTRESPONSE[s/"_score": 2/"_score": "$body.hits.hits.1._score"/] -// TESTRESPONSE[s/"_seq_no": 0/"_seq_no": "$body.hits.hits.0._seq_no"/] -// TESTRESPONSE[s/"_seq_no": 10/"_seq_no": "$body.hits.hits.1._seq_no"/] <1> The `clusters` section indicates that one cluster was unavailable and got skipped diff --git a/docs/reference/query-dsl/percolate-query.asciidoc b/docs/reference/query-dsl/percolate-query.asciidoc index 39d141827c686..9ae6dd802137c 100644 --- a/docs/reference/query-dsl/percolate-query.asciidoc +++ b/docs/reference/query-dsl/percolate-query.asciidoc @@ -99,8 +99,6 @@ The above request will yield the following response: "_index": "my-index", "_type": "_doc", "_id": "1", - "_seq_no": 0, - "_primary_term": 1, "_score": 0.26152915, "_source": { "query": { @@ -252,8 +250,6 @@ GET /my-index/_search "_index": "my-index", "_type": "_doc", "_id": "1", - "_seq_no": 0, - "_primary_term": 1, "_score": 0.7093853, "_source": { "query": { @@ -437,8 +433,6 @@ This will yield the following response. "_index": "my-index", "_type": "_doc", "_id": "3", - "_seq_no": 2, - "_primary_term": 1, "_score": 0.26152915, "_source": { "query": { @@ -460,8 +454,6 @@ This will yield the following response. "_index": "my-index", "_type": "_doc", "_id": "4", - "_seq_no": 3, - "_primary_term": 1, "_score": 0.26152915, "_source": { "query": { @@ -549,8 +541,6 @@ The slightly different response: "_index": "my-index", "_type": "_doc", "_id": "1", - "_seq_no": 0, - "_primary_term": 1, "_score": 0.7093853, "_source": { "query": { @@ -650,8 +640,6 @@ The above search request returns a response similar to this: "_index": "my-index", "_type": "_doc", "_id": "1", - "_seq_no": 0, - "_primary_term": 1, "_score": 0.26152915, "_source": { "query": { diff --git a/docs/reference/query-dsl/terms-set-query.asciidoc b/docs/reference/query-dsl/terms-set-query.asciidoc index 8c7fb929200eb..5c0e00ce360fa 100644 --- a/docs/reference/query-dsl/terms-set-query.asciidoc +++ b/docs/reference/query-dsl/terms-set-query.asciidoc @@ -82,8 +82,6 @@ Response: "_index": "my-index", "_type": "_doc", "_id": "2", - "_seq_no": 1, - "_primary_term": 1, "_score": 0.87546873, "_source": { "codes": ["def", "ghi"], diff --git a/docs/reference/search/rank-eval.asciidoc b/docs/reference/search/rank-eval.asciidoc index 2c8123ddbb9dc..c549b5e7a689b 100644 --- a/docs/reference/search/rank-eval.asciidoc +++ b/docs/reference/search/rank-eval.asciidoc @@ -336,8 +336,6 @@ that shows potential errors of individual queries. The response has the followin "_index": "my_index", "_type": "page", "_id": "1528558", - "_seq_no": 0, - "_primary_term": 1, "_score": 7.0556192 }, "rating": 1 diff --git a/docs/reference/search/request-body.asciidoc b/docs/reference/search/request-body.asciidoc index 3a6b1f551f13c..9970c4cc6223f 100644 --- a/docs/reference/search/request-body.asciidoc +++ b/docs/reference/search/request-body.asciidoc @@ -41,8 +41,6 @@ And here is a sample response: "_index" : "twitter", "_type" : "_doc", "_id" : "0", - "_seq_no": 0, - "_primary_term": 1, "_score": 1.3862944, "_source" : { "user" : "kimchy", diff --git a/docs/reference/search/request/highlighting.asciidoc b/docs/reference/search/request/highlighting.asciidoc index d47c55656cd75..ad836c7c535e7 100644 --- a/docs/reference/search/request/highlighting.asciidoc +++ b/docs/reference/search/request/highlighting.asciidoc @@ -866,8 +866,6 @@ Response: "_index": "twitter", "_type": "_doc", "_id": "1", - "_seq_no": 1, - "_primary_term": 1, "_score": 1.601195, "_source": { "user": "test", @@ -927,8 +925,6 @@ Response: "_index": "twitter", "_type": "_doc", "_id": "1", - "_seq_no": 1, - "_primary_term": 1, "_score": 1.601195, "_source": { "user": "test", diff --git a/docs/reference/search/request/inner-hits.asciidoc b/docs/reference/search/request/inner-hits.asciidoc index 1318daefeed1d..bcd2c297e5da3 100644 --- a/docs/reference/search/request/inner-hits.asciidoc +++ b/docs/reference/search/request/inner-hits.asciidoc @@ -144,8 +144,6 @@ An example of a response snippet that could be generated from the above search r "_index": "test", "_type": "_doc", "_id": "1", - "_seq_no": 0, - "_primary_term": 1, "_score": 1.0, "_source": ..., "inner_hits": { @@ -278,8 +276,6 @@ Response not included in text but tested for completeness sake. "_index": "test", "_type": "_doc", "_id": "1", - "_seq_no": 0, - "_primary_term": 1, "_score": 1.0444684, "_source": ..., "inner_hits": { @@ -399,8 +395,6 @@ Which would look like: "_index": "test", "_type": "_doc", "_id": "1", - "_seq_no": 0, - "_primary_term": 1, "_score": 0.6931472, "_source": ..., "inner_hits": { @@ -516,8 +510,6 @@ An example of a response snippet that could be generated from the above search r "_index": "test", "_type": "_doc", "_id": "1", - "_seq_no": 0, - "_primary_term": 1, "_score": 1.0, "_source": { "number": 1, @@ -536,8 +528,6 @@ An example of a response snippet that could be generated from the above search r "_index": "test", "_type": "_doc", "_id": "2", - "_seq_no": 1, - "_primary_term": 1, "_score": 1.0, "_routing": "1", "_source": { diff --git a/docs/reference/search/suggesters/completion-suggest.asciidoc b/docs/reference/search/suggesters/completion-suggest.asciidoc index 826fd8f73dd59..191f01f17394d 100644 --- a/docs/reference/search/suggesters/completion-suggest.asciidoc +++ b/docs/reference/search/suggesters/completion-suggest.asciidoc @@ -195,8 +195,6 @@ returns this response: "_index": "music", "_type": "_doc", "_id": "1", - "_seq_no": 2, - "_primary_term": 1, "_score": 1.0, "_source": { "suggest": ["Nevermind", "Nirvana"] @@ -276,8 +274,6 @@ Which should look like: "_index": "music", "_type": "_doc", "_id": "1", - "_seq_no": 2, - "_primary_term": 1, "_score": 1.0, "_source": { "suggest": ["Nevermind", "Nirvana"] diff --git a/docs/reference/search/uri-request.asciidoc b/docs/reference/search/uri-request.asciidoc index 32ef816974e50..87e1da907fb7d 100644 --- a/docs/reference/search/uri-request.asciidoc +++ b/docs/reference/search/uri-request.asciidoc @@ -37,8 +37,6 @@ And here is a sample response: "_index" : "twitter", "_type" : "_doc", "_id" : "0", - "_seq_no": 0, - "_primary_term": 1, "_score": 1.3862944, "_source" : { "user" : "kimchy", From 46f772e1a7eeb2f722cb25fcf382d35236cff2ad Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Mon, 21 Jan 2019 13:26:08 +0100 Subject: [PATCH 24/29] Revert "remove seq_no_and_term" This reverts commit f6d3c086f0b9305b0ee15c594d28e8b5f436387a. --- .../join/query/HasChildQueryBuilderTests.java | 1 + .../elasticsearch/join/query/InnerHitsIT.java | 26 +++++++++---- .../rest-api-spec/test/11_parent_child.yml | 4 +- .../resources/rest-api-spec/api/search.json | 4 ++ .../200_top_hits_metric.yml | 1 + .../test/search/110_field_collapsing.yml | 4 +- .../test/search/300_sequence_numbers.yml | 16 ++++++++ .../action/search/ExpandSearchPhase.java | 1 + .../index/query/InnerHitBuilder.java | 21 ++++++++++ .../index/query/InnerHitContextBuilder.java | 1 + .../index/query/NestedQueryBuilder.java | 8 ++++ .../rest/action/search/RestSearchAction.java | 3 ++ .../search/DefaultSearchContext.java | 11 ++++++ .../elasticsearch/search/SearchService.java | 4 ++ .../metrics/TopHitsAggregationBuilder.java | 36 ++++++++++++++++-- .../metrics/TopHitsAggregatorFactory.java | 10 +++-- .../search/builder/SearchSourceBuilder.java | 38 ++++++++++++++++++- .../SeqNoPrimaryTermFetchSubPhase.java | 4 ++ .../internal/FilteredSearchContext.java | 10 +++++ .../search/internal/SearchContext.java | 6 +++ .../search/internal/SubSearchContext.java | 11 ++++++ .../action/search/ExpandSearchPhaseTests.java | 4 +- .../index/query/InnerHitBuilderTests.java | 7 ++++ .../index/query/NestedQueryBuilderTests.java | 16 ++++---- .../aggregations/metrics/TopHitsIT.java | 12 +++++- .../aggregations/metrics/TopHitsTests.java | 3 ++ .../search/RandomSearchRequestGenerator.java | 3 ++ .../elasticsearch/test/TestSearchContext.java | 10 +++++ 28 files changed, 247 insertions(+), 28 deletions(-) diff --git a/modules/parent-join/src/test/java/org/elasticsearch/join/query/HasChildQueryBuilderTests.java b/modules/parent-join/src/test/java/org/elasticsearch/join/query/HasChildQueryBuilderTests.java index 6e4e79d16e5a5..eea01d61386de 100644 --- a/modules/parent-join/src/test/java/org/elasticsearch/join/query/HasChildQueryBuilderTests.java +++ b/modules/parent-join/src/test/java/org/elasticsearch/join/query/HasChildQueryBuilderTests.java @@ -252,6 +252,7 @@ public void testFromJson() throws IOException { " \"from\" : 0,\n" + " \"size\" : 100,\n" + " \"version\" : false,\n" + + " \"seq_no_primary_term\" : false,\n" + " \"explain\" : false,\n" + " \"track_scores\" : false,\n" + " \"sort\" : [ {\n" + diff --git a/modules/parent-join/src/test/java/org/elasticsearch/join/query/InnerHitsIT.java b/modules/parent-join/src/test/java/org/elasticsearch/join/query/InnerHitsIT.java index b6718f6c8dbb3..89929985ea594 100644 --- a/modules/parent-join/src/test/java/org/elasticsearch/join/query/InnerHitsIT.java +++ b/modules/parent-join/src/test/java/org/elasticsearch/join/query/InnerHitsIT.java @@ -56,6 +56,8 @@ import static org.elasticsearch.index.query.QueryBuilders.matchQuery; import static org.elasticsearch.index.query.QueryBuilders.nestedQuery; import static org.elasticsearch.index.query.QueryBuilders.termQuery; +import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_PRIMARY_TERM; +import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_SEQ_NO; import static org.elasticsearch.join.query.JoinQueryBuilders.hasChildQuery; import static org.elasticsearch.join.query.JoinQueryBuilders.hasParentQuery; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; @@ -134,9 +136,10 @@ public void testSimpleParentChild() throws Exception { assertThat(innerHits.getAt(1).getId(), equalTo("c2")); assertThat(innerHits.getAt(1).getType(), equalTo("doc")); + final boolean seqNoAndTerm = randomBoolean(); response = client().prepareSearch("articles") .setQuery(hasChildQuery("comment", matchQuery("message", "elephant"), ScoreMode.None) - .innerHit(new InnerHitBuilder())) + .innerHit(new InnerHitBuilder().setSeqNoAndPrimaryTerm(seqNoAndTerm))) .get(); assertNoFailures(response); assertHitCount(response, 1); @@ -153,12 +156,21 @@ public void testSimpleParentChild() throws Exception { assertThat(innerHits.getAt(2).getId(), equalTo("c6")); assertThat(innerHits.getAt(2).getType(), equalTo("doc")); - assertThat(innerHits.getAt(0).getPrimaryTerm(), equalTo(1L)); - assertThat(innerHits.getAt(1).getPrimaryTerm(), equalTo(1L)); - assertThat(innerHits.getAt(2).getPrimaryTerm(), equalTo(1L)); - assertThat(innerHits.getAt(0).getSeqNo(), greaterThanOrEqualTo(0L)); - assertThat(innerHits.getAt(1).getSeqNo(), greaterThanOrEqualTo(0L)); - assertThat(innerHits.getAt(2).getSeqNo(), greaterThanOrEqualTo(0L)); + if (seqNoAndTerm) { + assertThat(innerHits.getAt(0).getPrimaryTerm(), equalTo(1L)); + assertThat(innerHits.getAt(1).getPrimaryTerm(), equalTo(1L)); + assertThat(innerHits.getAt(2).getPrimaryTerm(), equalTo(1L)); + assertThat(innerHits.getAt(0).getSeqNo(), greaterThanOrEqualTo(0L)); + assertThat(innerHits.getAt(1).getSeqNo(), greaterThanOrEqualTo(0L)); + assertThat(innerHits.getAt(2).getSeqNo(), greaterThanOrEqualTo(0L)); + } else { + assertThat(innerHits.getAt(0).getPrimaryTerm(), equalTo(UNASSIGNED_PRIMARY_TERM)); + assertThat(innerHits.getAt(1).getPrimaryTerm(), equalTo(UNASSIGNED_PRIMARY_TERM)); + assertThat(innerHits.getAt(2).getPrimaryTerm(), equalTo(UNASSIGNED_PRIMARY_TERM)); + assertThat(innerHits.getAt(0).getSeqNo(), equalTo(UNASSIGNED_SEQ_NO)); + assertThat(innerHits.getAt(1).getSeqNo(), equalTo(UNASSIGNED_SEQ_NO)); + assertThat(innerHits.getAt(2).getSeqNo(), equalTo(UNASSIGNED_SEQ_NO)); + } response = client().prepareSearch("articles") .setQuery( diff --git a/modules/parent-join/src/test/resources/rest-api-spec/test/11_parent_child.yml b/modules/parent-join/src/test/resources/rest-api-spec/test/11_parent_child.yml index 7d14cfdab8822..61af4ab1acb59 100644 --- a/modules/parent-join/src/test/resources/rest-api-spec/test/11_parent_child.yml +++ b/modules/parent-join/src/test/resources/rest-api-spec/test/11_parent_child.yml @@ -51,7 +51,9 @@ setup: - do: search: rest_total_hits_as_int: true - body: { "query" : { "has_child" : { "type" : "child", "query" : { "match_all" : {} }, "inner_hits" : {} } } } + body: { "query" : { "has_child" : + { "type" : "child", "query" : { "match_all" : {} }, "inner_hits" : { "seq_no_primary_term": true} } + } } - match: { hits.total: 1 } - match: { hits.hits.0._index: "test" } - match: { hits.hits.0._id: "1" } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/search.json b/rest-api-spec/src/main/resources/rest-api-spec/api/search.json index 5834ca623a99b..9ac02b1214a2f 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/search.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/search.json @@ -164,6 +164,10 @@ "type" : "boolean", "description" : "Specify whether to return document version as part of a hit" }, + "seq_no_primary_term": { + "type" : "boolean", + "description" : "Specify whether to return sequence number and primary term of the last modification of each hit" + }, "request_cache": { "type" : "boolean", "description" : "Specify if request cache should be used for this request or not, defaults to index level setting" diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/200_top_hits_metric.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/200_top_hits_metric.yml index e8ef0f172debf..775475e01a597 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/200_top_hits_metric.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/200_top_hits_metric.yml @@ -103,6 +103,7 @@ setup: users: top_hits: sort: "users.last.keyword" + seq_no_primary_term: true - match: { hits.total: 2 } - length: { aggregations.groups.buckets.0.users.hits.hits: 2 } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/110_field_collapsing.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/110_field_collapsing.yml index 5e56d02586e06..a85abb4e6e28b 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/110_field_collapsing.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/110_field_collapsing.yml @@ -418,7 +418,9 @@ setup: rest_total_hits_as_int: true index: test body: - collapse: { field: numeric_group, inner_hits: { name: sub_hits, size: 2, sort: [{ sort: asc }]} } + collapse: { field: numeric_group, inner_hits: { + name: sub_hits, seq_no_primary_term: true, size: 2, sort: [{ sort: asc }] + } } sort: [{ sort: desc }] - match: { hits.total: 6 } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/300_sequence_numbers.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/300_sequence_numbers.yml index ec04ab92fa6c9..5f30114df89f8 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/300_sequence_numbers.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/300_sequence_numbers.yml @@ -35,6 +35,7 @@ setup: query: match: foo: bar + seq_no_primary_term: true - match: {hits.total.value: 1} - match: {hits.hits.0._seq_no: 1} @@ -53,7 +54,22 @@ setup: query: match: foo: bar + seq_no_primary_term: true - match: {hits.total.value: 1} - match: {hits.hits.0._seq_no: 1} - gte: {hits.hits.0._primary_term: 1} + +--- +"sequence numbers are not returned if not requested": + - do: + search: + index: _all + body: + query: + match: + foo: bar + + - match: {hits.total.value: 1} + - is_false: hits.hits.0._seq_no + - is_false: hits.hits.0._primary_term diff --git a/server/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java b/server/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java index da481b7a4a8ee..10a85b723166c 100644 --- a/server/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java +++ b/server/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java @@ -153,6 +153,7 @@ private SearchSourceBuilder buildExpandSearchSourceBuilder(InnerHitBuilder optio groupSource.explain(options.isExplain()); groupSource.trackScores(options.isTrackScores()); groupSource.version(options.isVersion()); + groupSource.seqNoAndPrimaryTerm(options.isSeqNoAndPrimaryTerm()); if (innerCollapseBuilder != null) { groupSource.collapse(innerCollapseBuilder); } diff --git a/server/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java b/server/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java index 9eb2a7fb9affb..90275800d3769 100644 --- a/server/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java @@ -68,6 +68,7 @@ public final class InnerHitBuilder implements Writeable, ToXContentObject { PARSER.declareInt(InnerHitBuilder::setSize, SearchSourceBuilder.SIZE_FIELD); PARSER.declareBoolean(InnerHitBuilder::setExplain, SearchSourceBuilder.EXPLAIN_FIELD); PARSER.declareBoolean(InnerHitBuilder::setVersion, SearchSourceBuilder.VERSION_FIELD); + PARSER.declareBoolean(InnerHitBuilder::setSeqNoAndPrimaryTerm, SearchSourceBuilder.SEQ_NO_PRIMARY_TERM_FIELD); PARSER.declareBoolean(InnerHitBuilder::setTrackScores, SearchSourceBuilder.TRACK_SCORES_FIELD); PARSER.declareStringArray(InnerHitBuilder::setStoredFieldNames, SearchSourceBuilder.STORED_FIELDS_FIELD); PARSER.declareObjectArray(InnerHitBuilder::setDocValueFields, @@ -124,6 +125,7 @@ public final class InnerHitBuilder implements Writeable, ToXContentObject { private int size = 3; private boolean explain; private boolean version; + private boolean seqNoAndPrimaryTerm; private boolean trackScores; private StoredFieldsContext storedFieldsContext; @@ -154,6 +156,11 @@ public InnerHitBuilder(StreamInput in) throws IOException { size = in.readVInt(); explain = in.readBoolean(); version = in.readBoolean(); + if (in.getVersion().onOrAfter(Version.V_7_0_0)){ + seqNoAndPrimaryTerm = in.readBoolean(); + } else { + seqNoAndPrimaryTerm = false; + } trackScores = in.readBoolean(); storedFieldsContext = in.readOptionalWriteable(StoredFieldsContext::new); if (in.getVersion().before(Version.V_6_4_0)) { @@ -198,6 +205,9 @@ public void writeTo(StreamOutput out) throws IOException { out.writeVInt(size); out.writeBoolean(explain); out.writeBoolean(version); + if (out.getVersion().onOrAfter(Version.V_7_0_0)) { + out.writeBoolean(seqNoAndPrimaryTerm); + } out.writeBoolean(trackScores); out.writeOptionalWriteable(storedFieldsContext); if (out.getVersion().before(Version.V_6_4_0)) { @@ -298,6 +308,15 @@ public InnerHitBuilder setVersion(boolean version) { return this; } + public boolean isSeqNoAndPrimaryTerm() { + return seqNoAndPrimaryTerm; + } + + public InnerHitBuilder setSeqNoAndPrimaryTerm(boolean seqNoAndPrimaryTerm) { + this.seqNoAndPrimaryTerm = seqNoAndPrimaryTerm; + return this; + } + public boolean isTrackScores() { return trackScores; } @@ -435,6 +454,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(SearchSourceBuilder.FROM_FIELD.getPreferredName(), from); builder.field(SearchSourceBuilder.SIZE_FIELD.getPreferredName(), size); builder.field(SearchSourceBuilder.VERSION_FIELD.getPreferredName(), version); + builder.field(SearchSourceBuilder.SEQ_NO_PRIMARY_TERM_FIELD.getPreferredName(), seqNoAndPrimaryTerm); builder.field(SearchSourceBuilder.EXPLAIN_FIELD.getPreferredName(), explain); builder.field(SearchSourceBuilder.TRACK_SCORES_FIELD.getPreferredName(), trackScores); if (fetchSourceContext != null) { @@ -493,6 +513,7 @@ public boolean equals(Object o) { Objects.equals(size, that.size) && Objects.equals(explain, that.explain) && Objects.equals(version, that.version) && + Objects.equals(seqNoAndPrimaryTerm, that.seqNoAndPrimaryTerm) && Objects.equals(trackScores, that.trackScores) && Objects.equals(storedFieldsContext, that.storedFieldsContext) && Objects.equals(docValueFields, that.docValueFields) && diff --git a/server/src/main/java/org/elasticsearch/index/query/InnerHitContextBuilder.java b/server/src/main/java/org/elasticsearch/index/query/InnerHitContextBuilder.java index 1fe781f38ce27..9e6a766aed891 100644 --- a/server/src/main/java/org/elasticsearch/index/query/InnerHitContextBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/InnerHitContextBuilder.java @@ -78,6 +78,7 @@ protected void setupInnerHitsContext(QueryShardContext queryShardContext, innerHitsContext.size(innerHitBuilder.getSize()); innerHitsContext.explain(innerHitBuilder.isExplain()); innerHitsContext.version(innerHitBuilder.isVersion()); + innerHitsContext.seqNoAndPrimaryTerm(innerHitBuilder.isSeqNoAndPrimaryTerm()); innerHitsContext.trackScores(innerHitBuilder.isTrackScores()); if (innerHitBuilder.getStoredFieldsContext() != null) { innerHitsContext.storedFieldsContext(innerHitBuilder.getStoredFieldsContext()); diff --git a/server/src/main/java/org/elasticsearch/index/query/NestedQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/NestedQueryBuilder.java index d2b432e7c7ca1..3c3856e208f04 100644 --- a/server/src/main/java/org/elasticsearch/index/query/NestedQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/NestedQueryBuilder.java @@ -368,6 +368,14 @@ static final class NestedInnerHitSubContext extends InnerHitsContext.InnerHitSub this.childObjectMapper = childObjectMapper; } + @Override + public void seqNoAndPrimaryTerm(boolean seqNoAndPrimaryTerm) { + assert seqNoAndPrimaryTerm() == false; + if (seqNoAndPrimaryTerm) { + throw new UnsupportedOperationException("nested documents are not assigned sequence numbers"); + } + } + @Override public TopDocsAndMaxScore[] topDocs(SearchHit[] hits) throws IOException { Weight innerHitQueryWeight = createInnerHitQueryWeight(); diff --git a/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java b/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java index 3e3a1e02a174b..da773efed580d 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java @@ -201,6 +201,9 @@ private static void parseSearchSource(final SearchSourceBuilder searchSourceBuil if (request.hasParam("version")) { searchSourceBuilder.version(request.paramAsBoolean("version", null)); } + if (request.hasParam("seq_no_primary_term")) { + searchSourceBuilder.seqNoAndPrimaryTerm(request.paramAsBoolean("seq_no_primary_term", null)); + } if (request.hasParam("timeout")) { searchSourceBuilder.timeout(request.paramAsTime("timeout", null)); } diff --git a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java index 590c58b1f6615..4b82e23e42061 100644 --- a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java +++ b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java @@ -107,6 +107,7 @@ final class DefaultSearchContext extends SearchContext { private ScrollContext scrollContext; private boolean explain; private boolean version = false; // by default, we don't return versions + private boolean seqAndPrimaryTerm = false; private StoredFieldsContext storedFields; private ScriptFieldsContext scriptFields; private FetchSourceContext fetchSourceContext; @@ -719,6 +720,16 @@ public void version(boolean version) { this.version = version; } + @Override + public boolean seqNoAndPrimaryTerm() { + return seqAndPrimaryTerm; + } + + @Override + public void seqNoAndPrimaryTerm(boolean seqNoAndPrimaryTerm) { + this.seqAndPrimaryTerm = seqNoAndPrimaryTerm; + } + @Override public int[] docIdsToLoad() { return docIdsToLoad; diff --git a/server/src/main/java/org/elasticsearch/search/SearchService.java b/server/src/main/java/org/elasticsearch/search/SearchService.java index 92c5bcbb48734..ddec3637ed491 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchService.java +++ b/server/src/main/java/org/elasticsearch/search/SearchService.java @@ -902,6 +902,10 @@ private void parseSource(DefaultSearchContext context, SearchSourceBuilder sourc context.version(source.version()); } + if (source.seqNoAndPrimaryTerm() != null) { + context.seqNoAndPrimaryTerm(source.seqNoAndPrimaryTerm()); + } + if (source.stats() != null) { context.groupStats(source.stats()); } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregationBuilder.java index 771e5739e6641..ba51099d6bc00 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregationBuilder.java @@ -19,6 +19,7 @@ package org.elasticsearch.search.aggregations.metrics; +import org.elasticsearch.Version; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.Strings; @@ -66,6 +67,7 @@ public class TopHitsAggregationBuilder extends AbstractAggregationBuilder> sorts = null; private HighlightBuilder highlightBuilder; @@ -85,6 +87,7 @@ protected TopHitsAggregationBuilder(TopHitsAggregationBuilder clone, this.size = clone.size; this.explain = clone.explain; this.version = clone.version; + this.seqNoAndPrimaryTerm = clone.seqNoAndPrimaryTerm; this.trackScores = clone.trackScores; this.sorts = clone.sorts == null ? null : new ArrayList<>(clone.sorts); this.highlightBuilder = clone.highlightBuilder == null ? null : @@ -137,6 +140,9 @@ public TopHitsAggregationBuilder(StreamInput in) throws IOException { } trackScores = in.readBoolean(); version = in.readBoolean(); + if (in.getVersion().onOrAfter(Version.V_7_0_0)) { + seqNoAndPrimaryTerm = in.readBoolean(); + } } @Override @@ -173,6 +179,9 @@ protected void doWriteTo(StreamOutput out) throws IOException { } out.writeBoolean(trackScores); out.writeBoolean(version); + if (out.getVersion().onOrAfter(Version.V_7_0_0)) { + out.writeBoolean(seqNoAndPrimaryTerm); + } } /** @@ -526,6 +535,23 @@ public boolean version() { return version; } + /** + * Should each {@link org.elasticsearch.search.SearchHit} be returned with the + * sequence number and primary term of the last modification of the document. + */ + public TopHitsAggregationBuilder seqNoAndPrimaryTerm(Boolean seqNoAndPrimaryTerm) { + this.seqNoAndPrimaryTerm = seqNoAndPrimaryTerm; + return this; + } + + /** + * Indicates whether {@link org.elasticsearch.search.SearchHit}s should be returned with the + * sequence number and primary term of the last modification of the document. + */ + public Boolean seqNoAndPrimaryTerm() { + return seqNoAndPrimaryTerm; + } + /** * Applies when sorting, and controls if scores will be tracked as well. * Defaults to {@code false}. @@ -579,7 +605,7 @@ protected TopHitsAggregatorFactory doBuild(SearchContext context, AggregatorFact } else { optionalSort = SortBuilder.buildSort(sorts, context.getQueryShardContext()); } - return new TopHitsAggregatorFactory(name, from, size, explain, version, trackScores, optionalSort, + return new TopHitsAggregatorFactory(name, from, size, explain, version, seqNoAndPrimaryTerm, trackScores, optionalSort, highlightBuilder, storedFieldsContext, docValueFields, fields, fetchSourceContext, context, parent, subfactoriesBuilder, metaData); } @@ -590,6 +616,7 @@ protected XContentBuilder internalXContent(XContentBuilder builder, Params param builder.field(SearchSourceBuilder.FROM_FIELD.getPreferredName(), from); builder.field(SearchSourceBuilder.SIZE_FIELD.getPreferredName(), size); builder.field(SearchSourceBuilder.VERSION_FIELD.getPreferredName(), version); + builder.field(SearchSourceBuilder.SEQ_NO_PRIMARY_TERM_FIELD.getPreferredName(), seqNoAndPrimaryTerm); builder.field(SearchSourceBuilder.EXPLAIN_FIELD.getPreferredName(), explain); if (fetchSourceContext != null) { builder.field(SearchSourceBuilder._SOURCE_FIELD.getPreferredName(), fetchSourceContext); @@ -647,6 +674,8 @@ public static TopHitsAggregationBuilder parse(String aggregationName, XContentPa factory.size(parser.intValue()); } else if (SearchSourceBuilder.VERSION_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { factory.version(parser.booleanValue()); + } else if (SearchSourceBuilder.SEQ_NO_PRIMARY_TERM_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { + factory.seqNoAndPrimaryTerm(parser.booleanValue()); } else if (SearchSourceBuilder.EXPLAIN_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { factory.explain(parser.booleanValue()); } else if (SearchSourceBuilder.TRACK_SCORES_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { @@ -746,7 +775,7 @@ public static TopHitsAggregationBuilder parse(String aggregationName, XContentPa @Override protected int doHashCode() { return Objects.hash(explain, fetchSourceContext, docValueFields, storedFieldsContext, from, highlightBuilder, - scriptFields, size, sorts, trackScores, version); + scriptFields, size, sorts, trackScores, version, seqNoAndPrimaryTerm); } @Override @@ -762,7 +791,8 @@ protected boolean doEquals(Object obj) { && Objects.equals(size, other.size) && Objects.equals(sorts, other.sorts) && Objects.equals(trackScores, other.trackScores) - && Objects.equals(version, other.version); + && Objects.equals(version, other.version) + && Objects.equals(seqNoAndPrimaryTerm, other.seqNoAndPrimaryTerm); } @Override diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregatorFactory.java index 85d6d127c8892..7edaccb66d4bd 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregatorFactory.java @@ -44,6 +44,7 @@ class TopHitsAggregatorFactory extends AggregatorFactory sort; private final HighlightBuilder highlightBuilder; @@ -52,10 +53,9 @@ class TopHitsAggregatorFactory extends AggregatorFactory scriptFields; private final FetchSourceContext fetchSourceContext; - TopHitsAggregatorFactory(String name, int from, int size, boolean explain, boolean version, boolean trackScores, - Optional sort, HighlightBuilder highlightBuilder, StoredFieldsContext storedFieldsContext, - List docValueFields, List scriptFields, - FetchSourceContext fetchSourceContext, + TopHitsAggregatorFactory(String name, int from, int size, boolean explain, boolean version, boolean seqNoAndPrimaryTerm, + boolean trackScores, Optional sort, HighlightBuilder highlightBuilder, StoredFieldsContext storedFieldsContext, + List docValueFields, List scriptFields, FetchSourceContext fetchSourceContext, SearchContext context, AggregatorFactory parent, AggregatorFactories.Builder subFactories, Map metaData) throws IOException { super(name, context, parent, subFactories, metaData); @@ -63,6 +63,7 @@ class TopHitsAggregatorFactory extends AggregatorFactory> sorts; private boolean trackScores = false; @@ -247,6 +250,11 @@ public SearchSourceBuilder(StreamInput in) throws IOException { timeout = in.readOptionalTimeValue(); trackScores = in.readBoolean(); version = in.readOptionalBoolean(); + if (in.getVersion().onOrAfter(Version.V_7_0_0)) { + seqNoAndPrimaryTerm = in.readOptionalBoolean(); + } else { + seqNoAndPrimaryTerm = null; + } extBuilders = in.readNamedWriteableList(SearchExtBuilder.class); profile = in.readBoolean(); searchAfterBuilder = in.readOptionalWriteable(SearchAfterBuilder::new); @@ -310,6 +318,9 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalTimeValue(timeout); out.writeBoolean(trackScores); out.writeOptionalBoolean(version); + if (out.getVersion().onOrAfter(Version.V_7_0_0)) { + out.writeOptionalBoolean(seqNoAndPrimaryTerm); + } out.writeNamedWriteableList(extBuilders); out.writeBoolean(profile); out.writeOptionalWriteable(searchAfterBuilder); @@ -441,6 +452,23 @@ public Boolean version() { return version; } + /** + * Should each {@link org.elasticsearch.search.SearchHit} be returned with the + * sequence number and primary term of the last modification of the document. + */ + public SearchSourceBuilder seqNoAndPrimaryTerm(Boolean seqNoAndPrimaryTerm) { + this.seqNoAndPrimaryTerm = seqNoAndPrimaryTerm; + return this; + } + + /** + * Indicates whether {@link org.elasticsearch.search.SearchHit}s should be returned with the + * sequence number and primary term of the last modification of the document. + */ + public Boolean seqNoAndPrimaryTerm() { + return seqNoAndPrimaryTerm; + } + /** * An optional timeout to control how long search is allowed to take. */ @@ -999,6 +1027,7 @@ private SearchSourceBuilder shallowCopy(QueryBuilder queryBuilder, QueryBuilder rewrittenBuilder.trackScores = trackScores; rewrittenBuilder.trackTotalHitsUpTo = trackTotalHitsUpTo; rewrittenBuilder.version = version; + rewrittenBuilder.seqNoAndPrimaryTerm = seqNoAndPrimaryTerm; rewrittenBuilder.collapse = collapse; return rewrittenBuilder; } @@ -1038,6 +1067,8 @@ public void parseXContent(XContentParser parser, boolean checkTrailingTokens) th minScore = parser.floatValue(); } else if (VERSION_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { version = parser.booleanValue(); + } else if (SEQ_NO_PRIMARY_TERM_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { + seqNoAndPrimaryTerm = parser.booleanValue(); } else if (EXPLAIN_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { explain = parser.booleanValue(); } else if (TRACK_SCORES_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { @@ -1205,6 +1236,10 @@ public XContentBuilder innerToXContent(XContentBuilder builder, Params params) t builder.field(VERSION_FIELD.getPreferredName(), version); } + if (seqNoAndPrimaryTerm != null) { + builder.field(SEQ_NO_PRIMARY_TERM_FIELD.getPreferredName(), seqNoAndPrimaryTerm); + } + if (explain != null) { builder.field(EXPLAIN_FIELD.getPreferredName(), explain); } @@ -1523,7 +1558,7 @@ public int hashCode() { return Objects.hash(aggregations, explain, fetchSourceContext, docValueFields, storedFieldsContext, from, highlightBuilder, indexBoosts, minScore, postQueryBuilder, queryBuilder, rescoreBuilders, scriptFields, size, sorts, searchAfterBuilder, sliceBuilder, stats, suggestBuilder, terminateAfter, timeout, trackScores, version, - profile, extBuilders, collapse, trackTotalHitsUpTo); + seqNoAndPrimaryTerm, profile, extBuilders, collapse, trackTotalHitsUpTo); } @Override @@ -1558,6 +1593,7 @@ public boolean equals(Object obj) { && Objects.equals(timeout, other.timeout) && Objects.equals(trackScores, other.trackScores) && Objects.equals(version, other.version) + && Objects.equals(seqNoAndPrimaryTerm, other.seqNoAndPrimaryTerm) && Objects.equals(profile, other.profile) && Objects.equals(extBuilders, other.extBuilders) && Objects.equals(collapse, other.collapse) diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/SeqNoPrimaryTermFetchSubPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/SeqNoPrimaryTermFetchSubPhase.java index ea81610ea5bc5..527c88740af2c 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/SeqNoPrimaryTermFetchSubPhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/SeqNoPrimaryTermFetchSubPhase.java @@ -34,6 +34,10 @@ public final class SeqNoPrimaryTermFetchSubPhase implements FetchSubPhase { @Override public void hitsExecute(SearchContext context, SearchHit[] hits) throws IOException { + if (context.seqNoAndPrimaryTerm() == false || + (context.storedFieldsContext() != null && context.storedFieldsContext().fetchFields() == false)) { + return; + } hits = hits.clone(); // don't modify the incoming hits Arrays.sort(hits, Comparator.comparingInt(SearchHit::docId)); diff --git a/server/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java b/server/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java index 3a7fb9f823f3a..ecde28719cec6 100644 --- a/server/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java +++ b/server/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java @@ -422,6 +422,16 @@ public void version(boolean version) { in.version(version); } + @Override + public boolean seqNoAndPrimaryTerm() { + return in.seqNoAndPrimaryTerm(); + } + + @Override + public void seqNoAndPrimaryTerm(boolean seqNoAndPrimaryTerm) { + in.seqNoAndPrimaryTerm(seqNoAndPrimaryTerm); + } + @Override public int[] docIdsToLoad() { return in.docIdsToLoad(); diff --git a/server/src/main/java/org/elasticsearch/search/internal/SearchContext.java b/server/src/main/java/org/elasticsearch/search/internal/SearchContext.java index 5e8fec3195a65..bd6d9c501c8d1 100644 --- a/server/src/main/java/org/elasticsearch/search/internal/SearchContext.java +++ b/server/src/main/java/org/elasticsearch/search/internal/SearchContext.java @@ -309,6 +309,12 @@ public InnerHitsContext innerHits() { public abstract void version(boolean version); + /** indicates whether the sequence number and primary term of the last modification to each hit should be returned */ + public abstract boolean seqNoAndPrimaryTerm(); + + /** controls whether the sequence number and primary term of the last modification to each hit should be returned */ + public abstract void seqNoAndPrimaryTerm(boolean seqNoAndPrimaryTerm); + public abstract int[] docIdsToLoad(); public abstract int docIdsToLoadFrom(); diff --git a/server/src/main/java/org/elasticsearch/search/internal/SubSearchContext.java b/server/src/main/java/org/elasticsearch/search/internal/SubSearchContext.java index 8c8137f5e4345..fb4d233f10ee8 100644 --- a/server/src/main/java/org/elasticsearch/search/internal/SubSearchContext.java +++ b/server/src/main/java/org/elasticsearch/search/internal/SubSearchContext.java @@ -65,6 +65,7 @@ public class SubSearchContext extends FilteredSearchContext { private boolean explain; private boolean trackScores; private boolean version; + private boolean seqNoAndPrimaryTerm; public SubSearchContext(SearchContext context) { super(context); @@ -294,6 +295,16 @@ public void version(boolean version) { this.version = version; } + @Override + public boolean seqNoAndPrimaryTerm() { + return seqNoAndPrimaryTerm; + } + + @Override + public void seqNoAndPrimaryTerm(boolean seqNoAndPrimaryTerm) { + this.seqNoAndPrimaryTerm = seqNoAndPrimaryTerm; + } + @Override public int[] docIdsToLoad() { return docIdsToLoad; diff --git a/server/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java b/server/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java index 9378a2cdd86bb..328950e4f3569 100644 --- a/server/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/ExpandSearchPhaseTests.java @@ -241,6 +241,7 @@ public void run() throws IOException { public void testExpandRequestOptions() throws IOException { MockSearchPhaseContext mockSearchPhaseContext = new MockSearchPhaseContext(1); boolean version = randomBoolean(); + final boolean seqNoAndTerm = randomBoolean(); mockSearchPhaseContext.searchTransport = new SearchTransportService(null, null) { @Override @@ -249,13 +250,14 @@ void sendExecuteMultiSearch(MultiSearchRequest request, SearchTask task, ActionL assertTrue(request.requests().stream().allMatch((r) -> "foo".equals(r.preference()))); assertTrue(request.requests().stream().allMatch((r) -> "baz".equals(r.routing()))); assertTrue(request.requests().stream().allMatch((r) -> version == r.source().version())); + assertTrue(request.requests().stream().allMatch((r) -> seqNoAndTerm == r.source().seqNoAndPrimaryTerm())); assertTrue(request.requests().stream().allMatch((r) -> postFilter.equals(r.source().postFilter()))); } }; mockSearchPhaseContext.getRequest().source(new SearchSourceBuilder() .collapse( new CollapseBuilder("someField") - .setInnerHits(new InnerHitBuilder().setName("foobarbaz").setVersion(version)) + .setInnerHits(new InnerHitBuilder().setName("foobarbaz").setVersion(version).setSeqNoAndPrimaryTerm(seqNoAndTerm)) ) .postFilter(QueryBuilders.existsQuery("foo"))) .preference("foobar") diff --git a/server/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java index 5c76c77b5c888..257ee807419b6 100644 --- a/server/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java @@ -140,6 +140,11 @@ public void testEqualsAndHashcode() { } } + public static InnerHitBuilder randomNestedInnerHits() { + InnerHitBuilder innerHitBuilder = randomInnerHits(); + innerHitBuilder.setSeqNoAndPrimaryTerm(false); // not supported by nested queries + return innerHitBuilder; + } public static InnerHitBuilder randomInnerHits() { InnerHitBuilder innerHits = new InnerHitBuilder(); innerHits.setName(randomAlphaOfLengthBetween(1, 16)); @@ -147,6 +152,7 @@ public static InnerHitBuilder randomInnerHits() { innerHits.setSize(randomIntBetween(0, 32)); innerHits.setExplain(randomBoolean()); innerHits.setVersion(randomBoolean()); + innerHits.setSeqNoAndPrimaryTerm(randomBoolean()); innerHits.setTrackScores(randomBoolean()); if (randomBoolean()) { innerHits.setStoredFieldNames(randomListStuff(16, () -> randomAlphaOfLengthBetween(1, 16))); @@ -189,6 +195,7 @@ static InnerHitBuilder mutate(InnerHitBuilder original) throws IOException { modifiers.add(() -> copy.setSize(randomValueOtherThan(copy.getSize(), () -> randomIntBetween(0, 128)))); modifiers.add(() -> copy.setExplain(!copy.isExplain())); modifiers.add(() -> copy.setVersion(!copy.isVersion())); + modifiers.add(() -> copy.setSeqNoAndPrimaryTerm(!copy.isSeqNoAndPrimaryTerm())); modifiers.add(() -> copy.setTrackScores(!copy.isTrackScores())); modifiers.add(() -> copy.setName(randomValueOtherThan(copy.getName(), () -> randomAlphaOfLengthBetween(1, 16)))); modifiers.add(() -> { diff --git a/server/src/test/java/org/elasticsearch/index/query/NestedQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/NestedQueryBuilderTests.java index 76479791283b4..ac9ae8d0fa7fb 100644 --- a/server/src/test/java/org/elasticsearch/index/query/NestedQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/NestedQueryBuilderTests.java @@ -45,7 +45,7 @@ import java.util.Map; import static org.elasticsearch.index.IndexSettingsTests.newIndexMeta; -import static org.elasticsearch.index.query.InnerHitBuilderTests.randomInnerHits; +import static org.elasticsearch.index.query.InnerHitBuilderTests.randomNestedInnerHits; import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.instanceOf; @@ -267,7 +267,7 @@ public void testThatUnrecognizedFromStringThrowsException() { } public void testInlineLeafInnerHitsNestedQuery() throws Exception { - InnerHitBuilder leafInnerHits = randomInnerHits(); + InnerHitBuilder leafInnerHits = randomNestedInnerHits(); NestedQueryBuilder nestedQueryBuilder = new NestedQueryBuilder("path", new MatchAllQueryBuilder(), ScoreMode.None); nestedQueryBuilder.innerHit(leafInnerHits); Map innerHitBuilders = new HashMap<>(); @@ -276,7 +276,7 @@ public void testInlineLeafInnerHitsNestedQuery() throws Exception { } public void testInlineLeafInnerHitsNestedQueryViaBoolQuery() { - InnerHitBuilder leafInnerHits = randomInnerHits(); + InnerHitBuilder leafInnerHits = randomNestedInnerHits(); NestedQueryBuilder nestedQueryBuilder = new NestedQueryBuilder("path", new MatchAllQueryBuilder(), ScoreMode.None) .innerHit(leafInnerHits); BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder().should(nestedQueryBuilder); @@ -286,7 +286,7 @@ public void testInlineLeafInnerHitsNestedQueryViaBoolQuery() { } public void testInlineLeafInnerHitsNestedQueryViaConstantScoreQuery() { - InnerHitBuilder leafInnerHits = randomInnerHits(); + InnerHitBuilder leafInnerHits = randomNestedInnerHits(); NestedQueryBuilder nestedQueryBuilder = new NestedQueryBuilder("path", new MatchAllQueryBuilder(), ScoreMode.None) .innerHit(leafInnerHits); ConstantScoreQueryBuilder constantScoreQueryBuilder = new ConstantScoreQueryBuilder(nestedQueryBuilder); @@ -296,10 +296,10 @@ public void testInlineLeafInnerHitsNestedQueryViaConstantScoreQuery() { } public void testInlineLeafInnerHitsNestedQueryViaBoostingQuery() { - InnerHitBuilder leafInnerHits1 = randomInnerHits(); + InnerHitBuilder leafInnerHits1 = randomNestedInnerHits(); NestedQueryBuilder nestedQueryBuilder1 = new NestedQueryBuilder("path", new MatchAllQueryBuilder(), ScoreMode.None) .innerHit(leafInnerHits1); - InnerHitBuilder leafInnerHits2 = randomInnerHits(); + InnerHitBuilder leafInnerHits2 = randomNestedInnerHits(); NestedQueryBuilder nestedQueryBuilder2 = new NestedQueryBuilder("path", new MatchAllQueryBuilder(), ScoreMode.None) .innerHit(leafInnerHits2); BoostingQueryBuilder constantScoreQueryBuilder = new BoostingQueryBuilder(nestedQueryBuilder1, nestedQueryBuilder2); @@ -310,7 +310,7 @@ public void testInlineLeafInnerHitsNestedQueryViaBoostingQuery() { } public void testInlineLeafInnerHitsNestedQueryViaFunctionScoreQuery() { - InnerHitBuilder leafInnerHits = randomInnerHits(); + InnerHitBuilder leafInnerHits = randomNestedInnerHits(); NestedQueryBuilder nestedQueryBuilder = new NestedQueryBuilder("path", new MatchAllQueryBuilder(), ScoreMode.None) .innerHit(leafInnerHits); FunctionScoreQueryBuilder functionScoreQueryBuilder = new FunctionScoreQueryBuilder(nestedQueryBuilder); @@ -330,7 +330,7 @@ public void testBuildIgnoreUnmappedNestQuery() throws Exception { when(mapperService.getIndexSettings()).thenReturn(settings); when(searchContext.mapperService()).thenReturn(mapperService); - InnerHitBuilder leafInnerHits = randomInnerHits(); + InnerHitBuilder leafInnerHits = randomNestedInnerHits(); NestedQueryBuilder query1 = new NestedQueryBuilder("path", new MatchAllQueryBuilder(), ScoreMode.None); query1.innerHit(leafInnerHits); final Map innerHitBuilders = new HashMap<>(); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/TopHitsIT.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/TopHitsIT.java index bb275650712d7..4a5982b9dacfa 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/TopHitsIT.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/TopHitsIT.java @@ -31,6 +31,7 @@ import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.query.MatchAllQueryBuilder; import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.index.seqno.SequenceNumbers; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.script.MockScriptEngine; import org.elasticsearch.script.MockScriptPlugin; @@ -579,6 +580,7 @@ public void testFieldCollapsing() throws Exception { } public void testFetchFeatures() { + final boolean seqNoAndTerm = randomBoolean(); SearchResponse response = client().prepareSearch("idx") .setQuery(matchQuery("text", "text").queryName("test")) .addAggregation(terms("terms") @@ -594,6 +596,7 @@ public void testFetchFeatures() { new Script(ScriptType.INLINE, MockScriptEngine.NAME, "5", Collections.emptyMap())) .fetchSource("text", null) .version(true) + .seqNoAndPrimaryTerm(seqNoAndTerm) ) ) .get(); @@ -621,8 +624,13 @@ public void testFetchFeatures() { long version = hit.getVersion(); assertThat(version, equalTo(1L)); - assertThat(hit.getSeqNo(), greaterThanOrEqualTo(0L)); - assertThat(hit.getPrimaryTerm(), greaterThanOrEqualTo(1L)); + if (seqNoAndTerm) { + assertThat(hit.getSeqNo(), greaterThanOrEqualTo(0L)); + assertThat(hit.getPrimaryTerm(), greaterThanOrEqualTo(1L)); + } else { + assertThat(hit.getSeqNo(), equalTo(SequenceNumbers.UNASSIGNED_SEQ_NO)); + assertThat(hit.getPrimaryTerm(), equalTo(SequenceNumbers.UNASSIGNED_PRIMARY_TERM)); + } assertThat(hit.getMatchedQueries()[0], equalTo("test")); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/TopHitsTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/TopHitsTests.java index caf1d4ef4ee47..6ec1ea1cad301 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/TopHitsTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/TopHitsTests.java @@ -54,6 +54,9 @@ protected final TopHitsAggregationBuilder createTestAggregatorBuilder() { if (randomBoolean()) { factory.version(randomBoolean()); } + if (randomBoolean()) { + factory.seqNoAndPrimaryTerm(randomBoolean()); + } if (randomBoolean()) { factory.trackScores(randomBoolean()); } diff --git a/test/framework/src/main/java/org/elasticsearch/search/RandomSearchRequestGenerator.java b/test/framework/src/main/java/org/elasticsearch/search/RandomSearchRequestGenerator.java index 5d96cd37b054d..6ec2732aaf915 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/RandomSearchRequestGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/search/RandomSearchRequestGenerator.java @@ -145,6 +145,9 @@ public static SearchSourceBuilder randomSearchSourceBuilder( if (randomBoolean()) { builder.version(randomBoolean()); } + if (randomBoolean()) { + builder.seqNoAndPrimaryTerm(randomBoolean()); + } if (randomBoolean()) { builder.trackScores(randomBoolean()); } diff --git a/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java b/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java index 18edb5ec3790a..6a17f65790368 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java +++ b/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java @@ -504,6 +504,16 @@ public boolean version() { public void version(boolean version) { } + @Override + public boolean seqNoAndPrimaryTerm() { + return false; + } + + @Override + public void seqNoAndPrimaryTerm(boolean seqNoAndPrimaryTerm) { + + } + @Override public int[] docIdsToLoad() { return new int[0]; From 8ce9840937b88ea9f365795c995de2c12dc9e12c Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Mon, 21 Jan 2019 13:35:44 +0100 Subject: [PATCH 25/29] fix multi_cluster --- .../resources/rest-api-spec/test/multi_cluster/10_basic.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/10_basic.yml b/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/10_basic.yml index 1cb601fc0e848..5acf84139bbf4 100644 --- a/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/10_basic.yml +++ b/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/10_basic.yml @@ -30,7 +30,7 @@ rest_total_hits_as_int: true index: test_index,my_remote_cluster:test_index body: - seq_no_and_primary_term: true + seq_no_primary_term: true aggs: cluster: terms: From a7a71d61d874a78c8ba0b477379ddab658360b26 Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Mon, 21 Jan 2019 15:26:32 +0100 Subject: [PATCH 26/29] feedback --- .../java/org/elasticsearch/index/query/InnerHitBuilder.java | 2 +- .../search/fetch/subphase/SeqNoPrimaryTermFetchSubPhase.java | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java b/server/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java index 90275800d3769..f5be9650b8d5c 100644 --- a/server/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java @@ -526,7 +526,7 @@ public boolean equals(Object o) { @Override public int hashCode() { - return Objects.hash(name, ignoreUnmapped, from, size, explain, version, trackScores, + return Objects.hash(name, ignoreUnmapped, from, size, explain, version, seqNoAndPrimaryTerm, trackScores, storedFieldsContext, docValueFields, scriptFields, fetchSourceContext, sorts, highlightBuilder, innerCollapseBuilder); } diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/SeqNoPrimaryTermFetchSubPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/SeqNoPrimaryTermFetchSubPhase.java index 527c88740af2c..31a6328ff9574 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/SeqNoPrimaryTermFetchSubPhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/SeqNoPrimaryTermFetchSubPhase.java @@ -34,8 +34,7 @@ public final class SeqNoPrimaryTermFetchSubPhase implements FetchSubPhase { @Override public void hitsExecute(SearchContext context, SearchHit[] hits) throws IOException { - if (context.seqNoAndPrimaryTerm() == false || - (context.storedFieldsContext() != null && context.storedFieldsContext().fetchFields() == false)) { + if (context.seqNoAndPrimaryTerm() == false) { return; } From c3ec471126228d0400a919458289669233b5b043 Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Mon, 21 Jan 2019 17:01:31 +0100 Subject: [PATCH 27/29] add documentation --- .../metrics/tophits-aggregation.asciidoc | 1 + .../docs/concurrency-control.asciidoc | 2 +- docs/reference/search/request-body.asciidoc | 2 +- .../search/request/inner-hits.asciidoc | 1 + .../request/version-and-seq-no.asciidoc | 34 +++++++++++++++++++ .../reference/search/request/version.asciidoc | 16 --------- 6 files changed, 38 insertions(+), 18 deletions(-) create mode 100644 docs/reference/search/request/version-and-seq-no.asciidoc delete mode 100644 docs/reference/search/request/version.asciidoc diff --git a/docs/reference/aggregations/metrics/tophits-aggregation.asciidoc b/docs/reference/aggregations/metrics/tophits-aggregation.asciidoc index c3b6a9bad4cfc..1ea38fec9657b 100644 --- a/docs/reference/aggregations/metrics/tophits-aggregation.asciidoc +++ b/docs/reference/aggregations/metrics/tophits-aggregation.asciidoc @@ -25,6 +25,7 @@ The top_hits aggregation returns regular search hits, because of this many per h * <> * <> * <> +* <> ==== Example diff --git a/docs/reference/docs/concurrency-control.asciidoc b/docs/reference/docs/concurrency-control.asciidoc index e695e6b5127c9..780a9c7cf76fc 100644 --- a/docs/reference/docs/concurrency-control.asciidoc +++ b/docs/reference/docs/concurrency-control.asciidoc @@ -87,7 +87,7 @@ returns: Note: The <> can return the `_seq_no` and `_primary_term` -for each search hit by requesting the `_seq_no` and `_primary_term` <>. +for each search hit by setting <>. The sequence number and the primary term uniquely identify a change. By noting down the sequence number and primary term returned, you can make sure to only change the diff --git a/docs/reference/search/request-body.asciidoc b/docs/reference/search/request-body.asciidoc index 9970c4cc6223f..dac7622aab8ed 100644 --- a/docs/reference/search/request-body.asciidoc +++ b/docs/reference/search/request-body.asciidoc @@ -213,7 +213,7 @@ include::request/preference.asciidoc[] include::request/explain.asciidoc[] -include::request/version.asciidoc[] +include::request/version-and-seq-no.asciidoc[] include::request/index-boost.asciidoc[] diff --git a/docs/reference/search/request/inner-hits.asciidoc b/docs/reference/search/request/inner-hits.asciidoc index bcd2c297e5da3..b287b1609703e 100644 --- a/docs/reference/search/request/inner-hits.asciidoc +++ b/docs/reference/search/request/inner-hits.asciidoc @@ -76,6 +76,7 @@ Inner hits also supports the following per document features: * <> * <> * <> +* <> [[nested-inner-hits]] ==== Nested inner hits diff --git a/docs/reference/search/request/version-and-seq-no.asciidoc b/docs/reference/search/request/version-and-seq-no.asciidoc new file mode 100644 index 0000000000000..2bca4c985b290 --- /dev/null +++ b/docs/reference/search/request/version-and-seq-no.asciidoc @@ -0,0 +1,34 @@ +[[search-request-seq-no-primary-term]] +=== Sequence Numbers and Primary Term + +Returns the sequence number and primary term of the last modification to each search hit. +See <> for more details. + +[source,js] +-------------------------------------------------- +GET /_search +{ + "seq_no_primary_term": true, + "query" : { + "term" : { "user" : "kimchy" } + } +} +-------------------------------------------------- +// CONSOLE + +[[search-request-version]] +=== Version + +Returns a version for each search hit. + +[source,js] +-------------------------------------------------- +GET /_search +{ + "version": true, + "query" : { + "term" : { "user" : "kimchy" } + } +} +-------------------------------------------------- +// CONSOLE diff --git a/docs/reference/search/request/version.asciidoc b/docs/reference/search/request/version.asciidoc deleted file mode 100644 index 57c6ce27feb91..0000000000000 --- a/docs/reference/search/request/version.asciidoc +++ /dev/null @@ -1,16 +0,0 @@ -[[search-request-version]] -=== Version - -Returns a version for each search hit. - -[source,js] --------------------------------------------------- -GET /_search -{ - "version": true, - "query" : { - "term" : { "user" : "kimchy" } - } -} --------------------------------------------------- -// CONSOLE From 6c884114e496d9db25381a2cbb5bc67eed654c2c Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Mon, 21 Jan 2019 18:13:01 +0100 Subject: [PATCH 28/29] java doc defaults --- .../main/java/org/elasticsearch/search/SearchHit.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/search/SearchHit.java b/server/src/main/java/org/elasticsearch/search/SearchHit.java index c88c4098cca72..74093ae6a7d3b 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchHit.java +++ b/server/src/main/java/org/elasticsearch/search/SearchHit.java @@ -180,12 +180,17 @@ public void setPrimaryTerm(long primaryTerm) { this.primaryTerm = primaryTerm; } - /** returns the sequence number of the last modification to the document */ + /** + * returns the sequence number of the last modification to the document, or {@link SequenceNumbers#UNASSIGNED_SEQ_NO} + * if not requested. + **/ public long getSeqNo() { return this.seqNo; } - /** returns the primary term of the last modification to the document */ + /** + * returns the primary term of the last modification to the document, or {@link SequenceNumbers#UNASSIGNED_PRIMARY_TERM} + * if not requested. */ public long getPrimaryTerm() { return this.primaryTerm; } From 40b5656477bd6e537d09c5361abc689afc1cdae8 Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Tue, 22 Jan 2019 09:40:39 +0100 Subject: [PATCH 29/29] fix 300_sequence_numbers.yml --- .../resources/rest-api-spec/test/search/300_sequence_numbers.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/300_sequence_numbers.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/300_sequence_numbers.yml index 5f30114df89f8..9e838d1c58f77 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/300_sequence_numbers.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/300_sequence_numbers.yml @@ -70,6 +70,5 @@ setup: match: foo: bar - - match: {hits.total.value: 1} - is_false: hits.hits.0._seq_no - is_false: hits.hits.0._primary_term