Expose sequence number and primary terms in search responses (#37639)

Users may require the sequence number and primary terms to perform optimistic concurrency control operations. Currently, you can get the sequence number via the `docvalues_fields` API but the primary term is not accessible because it is maintained by the `SeqNoFieldMapper` and the infrastructure can't find it.

This commit adds a dedicated sub fetch phase to return both numbers that is connected to a new `seq_no_primary_term` parameter.

Author: bleskes

docs/reference/aggregations/metrics/tophits-aggregation.asciidoc

@@ -25,6 +25,7 @@ The top_hits aggregation returns regular search hits, because of this many per h
 * <<search-request-script-fields,Script fields>>
 * <<search-request-docvalue-fields,Doc value fields>>
 * <<search-request-version,Include versions>>
+* <<search-request-seq-no-primary-term,Include Sequence Numbers and Primary Terms>>
 
 ==== Example
 

docs/reference/docs/concurrency-control.asciidoc

@@ -87,7 +87,7 @@ returns:
 
 
 Note: The <<search-search,Search API>> can return the `_seq_no` and `_primary_term`
-for each search hit by requesting the `_seq_no` and `_primary_term` <<search-request-docvalue-fields,Doc Value Fields>>.
+for each search hit by setting <<search-request-seq-no-primary-term,`seq_no_primary_term` parameter>>.
 
 The sequence number and the primary term uniquely identify a change. By noting down 
 the sequence number and primary term returned, you can make sure to only change the

docs/reference/search/request-body.asciidoc

@@ -205,7 +205,7 @@ include::request/preference.asciidoc[]
 
 include::request/explain.asciidoc[]
 
-include::request/version.asciidoc[]
+include::request/version-and-seq-no.asciidoc[]
 
 include::request/index-boost.asciidoc[]
 

docs/reference/search/request/inner-hits.asciidoc

@@ -76,6 +76,7 @@ Inner hits also supports the following per document features:
 * <<search-request-script-fields,Script fields>>
 * <<search-request-docvalue-fields,Doc value fields>>
 * <<search-request-version,Include versions>>
+* <<search-request-seq-no-primary-term,Include Sequence Numbers and Primary Terms>>
 
 [[nested-inner-hits]]
 ==== Nested inner hits

docs/reference/search/request/version-and-seq-no.asciidoc

@@ -0,0 +1,34 @@
+[[search-request-seq-no-primary-term]]
+=== Sequence Numbers and Primary Term
+
+Returns the sequence number and primary term of the last modification to each search hit.
+See <<optimistic-concurrency-control>> for more details.
+
+[source,js]
+--------------------------------------------------
+GET /_search
+{
+    "seq_no_primary_term": true,
+    "query" : {
+        "term" : { "user" : "kimchy" }
+    }
+}
+--------------------------------------------------
+// CONSOLE
+
+[[search-request-version]]
+=== Version
+
+Returns a version for each search hit.
+
+[source,js]
+--------------------------------------------------
+GET /_search
+{
+    "version": true,
+    "query" : {
+        "term" : { "user" : "kimchy" }
+    }
+}
+--------------------------------------------------
+// CONSOLE

docs/reference/search/request/version.asciidoc

@@ -256,6 +256,7 @@ public void testFromJson() throws IOException {
                 "      \"from\" : 0,\n" +
                 "      \"size\" : 100,\n" +
                 "      \"version\" : false,\n" +
+                "      \"seq_no_primary_term\" : false,\n" +
                 "      \"explain\" : false,\n" +
                 "      \"track_scores\" : false,\n" +
                 "      \"sort\" : [ {\n" +

modules/parent-join/src/test/java/org/elasticsearch/join/query/HasChildQueryBuilderTests.java

@@ -56,6 +56,8 @@
 import static org.elasticsearch.index.query.QueryBuilders.matchQuery;
 import static org.elasticsearch.index.query.QueryBuilders.nestedQuery;
 import static org.elasticsearch.index.query.QueryBuilders.termQuery;
+import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_PRIMARY_TERM;
+import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_SEQ_NO;
 import static org.elasticsearch.join.query.JoinQueryBuilders.hasChildQuery;
 import static org.elasticsearch.join.query.JoinQueryBuilders.hasParentQuery;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
@@ -66,6 +68,7 @@
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.hasId;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.notNullValue;
 import static org.hamcrest.Matchers.nullValue;
 
@@ -140,9 +143,10 @@ public void testSimpleParentChild() throws Exception {
         assertThat(innerHits.getAt(1).getId(), equalTo("c2"));
         assertThat(innerHits.getAt(1).getType(), equalTo(legacy() ? "comment" : "doc"));
 
+        final boolean seqNoAndTerm = randomBoolean();
         response = client().prepareSearch("articles")
             .setQuery(hasChildQuery("comment", matchQuery("message", "elephant"), ScoreMode.None)
-                .innerHit(new InnerHitBuilder()))
+                .innerHit(new InnerHitBuilder().setSeqNoAndPrimaryTerm(seqNoAndTerm)))
             .get();
         assertNoFailures(response);
         assertHitCount(response, 1);
@@ -159,6 +163,22 @@ public void testSimpleParentChild() throws Exception {
         assertThat(innerHits.getAt(2).getId(), equalTo("c6"));
         assertThat(innerHits.getAt(2).getType(), equalTo(legacy() ? "comment" : "doc"));
 
+        if (seqNoAndTerm) {
+            assertThat(innerHits.getAt(0).getPrimaryTerm(), equalTo(1L));
+            assertThat(innerHits.getAt(1).getPrimaryTerm(), equalTo(1L));
+            assertThat(innerHits.getAt(2).getPrimaryTerm(), equalTo(1L));
+            assertThat(innerHits.getAt(0).getSeqNo(), greaterThanOrEqualTo(0L));
+            assertThat(innerHits.getAt(1).getSeqNo(), greaterThanOrEqualTo(0L));
+            assertThat(innerHits.getAt(2).getSeqNo(), greaterThanOrEqualTo(0L));
+        } else {
+            assertThat(innerHits.getAt(0).getPrimaryTerm(), equalTo(UNASSIGNED_PRIMARY_TERM));
+            assertThat(innerHits.getAt(1).getPrimaryTerm(), equalTo(UNASSIGNED_PRIMARY_TERM));
+            assertThat(innerHits.getAt(2).getPrimaryTerm(), equalTo(UNASSIGNED_PRIMARY_TERM));
+            assertThat(innerHits.getAt(0).getSeqNo(), equalTo(UNASSIGNED_SEQ_NO));
+            assertThat(innerHits.getAt(1).getSeqNo(), equalTo(UNASSIGNED_SEQ_NO));
+            assertThat(innerHits.getAt(2).getSeqNo(), equalTo(UNASSIGNED_SEQ_NO));
+        }
+
         response = client().prepareSearch("articles")
             .setQuery(
                 hasChildQuery("comment", matchQuery("message", "fox"), ScoreMode.None).innerHit(

modules/parent-join/src/test/java/org/elasticsearch/join/query/InnerHitsIT.java

@@ -20,7 +20,6 @@
 package org.elasticsearch.join.query;
 
 import com.carrotsearch.randomizedtesting.generators.RandomPicks;
-
 import org.apache.lucene.search.BooleanClause;
 import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.ConstantScoreQuery;
@@ -240,6 +239,7 @@ public void testFromJson() throws IOException {
                 "      \"from\" : 0,\n" +
                 "      \"size\" : 100,\n" +
                 "      \"version\" : false,\n" +
+                "      \"seq_no_primary_term\" : false,\n" +
                 "      \"explain\" : false,\n" +
                 "      \"track_scores\" : false,\n" +
                 "      \"sort\" : [ {\n" +

modules/parent-join/src/test/java/org/elasticsearch/join/query/LegacyHasChildQueryBuilderTests.java

@@ -1,4 +1,7 @@
 setup:
+  - skip:
+      version: " - 5.5.99"
+      reason:  parent-join was added in 5.6.
   - do:
       indices.create:
         index: test
@@ -11,30 +14,30 @@ setup:
                     relations:
                       parent: child
 
+  - do:
+      index:
+        index: test
+        type:  doc
+        id:    1
+        body:  {"foo": "bar", "join_field": {"name" : "parent"} }
+
+  - do:
+      index:
+        index: test
+        type:  doc
+        id:    2
+        routing: 1
+        body:  {"bar": "baz", "join_field": { "name" : "child", "parent": "1"} }
+
+  - do:
+      indices.refresh: {}
+
 ---
 "Parent/child inner hits":
     - skip:
         version: " - 5.5.99"
         reason:  parent-join was added in 5.6.
 
-    - do:
-        index:
-          index: test
-          type:  doc
-          id:    1
-          body:  {"foo": "bar", "join_field": {"name" : "parent"} }
-
-    - do:
-        index:
-          index: test
-          type:  doc
-          id:    2
-          routing: 1
-          body:  {"bar": "baz", "join_field": { "name" : "child", "parent": "1"} }
-
-    - do:
-        indices.refresh: {}
-
     - do:
         search:
           body: { "query" : { "has_child" : { "type" : "child", "query" : { "match_all" : {} }, "inner_hits" : {} } } }
@@ -44,3 +47,24 @@ setup:
     - match: { hits.hits.0.inner_hits.child.hits.hits.0._index: "test"}
     - match: { hits.hits.0.inner_hits.child.hits.hits.0._id: "2" }
     - is_false: hits.hits.0.inner_hits.child.hits.hits.0._nested
+
+---
+"Parent/child inner hits with seq no":
+  - skip:
+      version: " - 6.6.99"
+      reason: support was added in 6.7
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        body: { "query" : { "has_child" :
+            { "type" : "child", "query" : { "match_all" : {} }, "inner_hits" : { "seq_no_primary_term": true} }
+        } }
+  - match: { hits.total: 1 }
+  - match: { hits.hits.0._index: "test" }
+  - match: { hits.hits.0._id: "1" }
+  - match: { hits.hits.0.inner_hits.child.hits.hits.0._index: "test"}
+  - match: { hits.hits.0.inner_hits.child.hits.hits.0._id: "2" }
+  - is_false: hits.hits.0.inner_hits.child.hits.hits.0._nested
+  - gte: { hits.hits.0.inner_hits.child.hits.hits.0._seq_no: 0 }
+  - gte: { hits.hits.0.inner_hits.child.hits.hits.0._primary_term: 1 }