HLRC: Add rollup search (#36334)

Relates to #29827

Author: nik9000

client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java

@@ -361,8 +361,15 @@ static Request update(UpdateRequest updateRequest) throws IOException {
         return request;
     }
 
-    static Request search(SearchRequest searchRequest) throws IOException {
-        Request request = new Request(HttpPost.METHOD_NAME, endpoint(searchRequest.indices(), searchRequest.types(), "_search"));
+    /**
+     * Convert a {@linkplain SearchRequest} into a {@linkplain Request}.
+     * @param searchRequest the request to convert
+     * @param searchEndpoint the name of the search endpoint. {@literal _search}
+     *    for standard searches and {@literal _rollup_search} for rollup
+     *    searches.
+     */
+    static Request search(SearchRequest searchRequest, String searchEndpoint) throws IOException {
+        Request request = new Request(HttpPost.METHOD_NAME, endpoint(searchRequest.indices(), searchRequest.types(), searchEndpoint));
 
         Params params = new Params(request);
         addSearchRequestParams(params, searchRequest);

client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java

@@ -1087,7 +1087,12 @@ public final void deleteAsync(DeleteRequest deleteRequest, ActionListener<Delete
      * @return the response
      */
     public final SearchResponse search(SearchRequest searchRequest, RequestOptions options) throws IOException {
-        return performRequestAndParseEntity(searchRequest, RequestConverters::search, options, SearchResponse::fromXContent, emptySet());
+        return performRequestAndParseEntity(
+                searchRequest,
+                r -> RequestConverters.search(r, "_search"),
+                options,
+                SearchResponse::fromXContent,
+                emptySet());
     }
 
     /**
@@ -1098,7 +1103,12 @@ public final SearchResponse search(SearchRequest searchRequest, RequestOptions o
      */
     @Deprecated
     public final SearchResponse search(SearchRequest searchRequest, Header... headers) throws IOException {
-        return performRequestAndParseEntity(searchRequest, RequestConverters::search, SearchResponse::fromXContent, emptySet(), headers);
+        return performRequestAndParseEntity(
+                searchRequest,
+                r -> RequestConverters.search(r, "_search"),
+                SearchResponse::fromXContent,
+                emptySet(),
+                headers);
     }
 
     /**
@@ -1109,7 +1119,12 @@ public final SearchResponse search(SearchRequest searchRequest, Header... header
      * @param listener the listener to be notified upon request completion
      */
     public final void searchAsync(SearchRequest searchRequest, RequestOptions options, ActionListener<SearchResponse> listener) {
-        performRequestAsyncAndParseEntity(searchRequest, RequestConverters::search, options, SearchResponse::fromXContent, listener,
+        performRequestAsyncAndParseEntity(
+                searchRequest,
+                r -> RequestConverters.search(r, "_search"),
+                options,
+                SearchResponse::fromXContent,
+                listener,
                 emptySet());
     }
 
@@ -1121,8 +1136,13 @@ public final void searchAsync(SearchRequest searchRequest, RequestOptions option
      */
     @Deprecated
     public final void searchAsync(SearchRequest searchRequest, ActionListener<SearchResponse> listener, Header... headers) {
-        performRequestAsyncAndParseEntity(searchRequest, RequestConverters::search, SearchResponse::fromXContent, listener,
-                emptySet(), headers);
+        performRequestAsyncAndParseEntity(
+                searchRequest,
+                r -> RequestConverters.search(r, "_search"),
+                SearchResponse::fromXContent,
+                listener,
+                emptySet(),
+                headers);
     }
 
     /**

client/rest-high-level/src/main/java/org/elasticsearch/client/RollupClient.java

@@ -20,6 +20,8 @@
 package org.elasticsearch.client;
 
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.client.core.AcknowledgedResponse;
 import org.elasticsearch.client.rollup.DeleteRollupJobRequest;
 import org.elasticsearch.client.rollup.GetRollupIndexCapsRequest;
@@ -224,6 +226,42 @@ public void getRollupJobAsync(GetRollupJobRequest request, RequestOptions option
             listener, Collections.emptySet());
     }
 
+    /**
+     * Perform a rollup search.
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/rollup-search.html">
+     * the docs</a> for more.
+     * @param request the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @return the response
+     * @throws IOException in case there is a problem sending the request or parsing back the response
+     */
+    public SearchResponse search(SearchRequest request, RequestOptions options) throws IOException {
+        return restHighLevelClient.performRequestAndParseEntity(
+                request,
+                RollupRequestConverters::search,
+                options,
+                SearchResponse::fromXContent,
+                Collections.emptySet());
+    }
+
+    /**
+     * Perform a rollup search.
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/rollup-search.html">
+     * the docs</a> for more.
+     * @param request the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @param listener the listener to be notified upon request completion
+     */
+    public void searchAsync(SearchRequest request, RequestOptions options, ActionListener<SearchResponse> listener) {
+        restHighLevelClient.performRequestAsyncAndParseEntity(
+                request,
+                RollupRequestConverters::search,
+                options,
+                SearchResponse::fromXContent,
+                listener,
+                Collections.emptySet());
+    }
+
     /**
      * Get the Rollup Capabilities of a target (non-rollup) index or pattern
      * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/master/rollup-get-rollup-caps.html">

client/rest-high-level/src/main/java/org/elasticsearch/client/RollupRequestConverters.java

@@ -22,6 +22,7 @@
 import org.apache.http.client.methods.HttpGet;
 import org.apache.http.client.methods.HttpPost;
 import org.apache.http.client.methods.HttpPut;
+import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.client.rollup.DeleteRollupJobRequest;
 import org.elasticsearch.client.rollup.GetRollupCapsRequest;
 import org.elasticsearch.client.rollup.GetRollupIndexCapsRequest;
@@ -83,6 +84,20 @@ static Request getJob(final GetRollupJobRequest getRollupJobRequest) {
         return new Request(HttpGet.METHOD_NAME, endpoint);
     }
 
+    static Request search(final SearchRequest request) throws IOException {
+        if (request.types().length > 0) {
+            /*
+             * Ideally we'd check this with the standard validation framework
+             * but we don't have a special request for rollup search so that'd
+             * be difficult. 
+             */
+            ValidationException ve = new ValidationException();
+            ve.addValidationError("types are not allowed in rollup search");
+            throw ve;
+        }
+        return RequestConverters.search(request, "_rollup_search");
+    }
+
     static Request getRollupCaps(final GetRollupCapsRequest getRollupCapsRequest) throws IOException {
         String endpoint = new RequestConverters.EndpointBuilder()
             .addPathPartAsIs("_xpack", "rollup", "data")

client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java

@@ -908,14 +908,16 @@ public void testGlobalPipelineOnBulkRequest() throws IOException {
     }
 
     public void testSearchNullSource() throws IOException {
+        String searchEndpoint = randomFrom("_" + randomAlphaOfLength(5));
         SearchRequest searchRequest = new SearchRequest();
-        Request request = RequestConverters.search(searchRequest);
+        Request request = RequestConverters.search(searchRequest, searchEndpoint);
         assertEquals(HttpPost.METHOD_NAME, request.getMethod());
-        assertEquals("/_search", request.getEndpoint());
+        assertEquals("/" + searchEndpoint, request.getEndpoint());
         assertNull(request.getEntity());
     }
 
     public void testSearch() throws Exception {
+        String searchEndpoint = randomFrom("_" + randomAlphaOfLength(5));
         String[] indices = randomIndicesNames(0, 5);
         SearchRequest searchRequest = new SearchRequest(indices);
 
@@ -976,7 +978,7 @@ public void testSearch() throws Exception {
             searchRequest.source(searchSourceBuilder);
         }
 
-        Request request = RequestConverters.search(searchRequest);
+        Request request = RequestConverters.search(searchRequest, searchEndpoint);
         StringJoiner endpoint = new StringJoiner("/", "/", "");
         String index = String.join(",", indices);
         if (Strings.hasLength(index)) {
@@ -986,7 +988,7 @@ public void testSearch() throws Exception {
         if (Strings.hasLength(type)) {
             endpoint.add(type);
         }
-        endpoint.add("_search");
+        endpoint.add(searchEndpoint);
         assertEquals(HttpPost.METHOD_NAME, request.getMethod());
         assertEquals(endpoint.toString(), request.getEndpoint());
         assertEquals(expectedParams, request.getParameters());

client/rest-high-level/src/test/java/org/elasticsearch/client/RollupIT.java

@@ -54,11 +54,13 @@
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval;
+import org.elasticsearch.search.aggregations.metrics.NumericMetricsAggregation;
 import org.elasticsearch.search.aggregations.metrics.avg.AvgAggregationBuilder;
 import org.elasticsearch.search.aggregations.metrics.max.MaxAggregationBuilder;
 import org.elasticsearch.search.aggregations.metrics.min.MinAggregationBuilder;
 import org.elasticsearch.search.aggregations.metrics.sum.SumAggregationBuilder;
 import org.elasticsearch.search.aggregations.metrics.valuecount.ValueCountAggregationBuilder;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.junit.Before;
 
 import java.util.Arrays;
@@ -70,6 +72,7 @@
 import java.util.Set;
 
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
+import static org.hamcrest.Matchers.closeTo;
 import static org.hamcrest.Matchers.either;
 import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
@@ -245,6 +248,33 @@ public void testPutAndGetRollupJob() throws Exception {
         }
     }
 
+    public void testSearch() throws Exception {
+        testPutAndGetRollupJob();
+        SearchRequest search = new SearchRequest(rollupIndex);
+        search.source(new SearchSourceBuilder()
+                .size(0)
+                .aggregation(new AvgAggregationBuilder("avg").field("value")));
+        SearchResponse response = highLevelClient().rollup().search(search, RequestOptions.DEFAULT);
+        assertEquals(0, response.getFailedShards());
+        assertEquals(0, response.getHits().getTotalHits());
+        NumericMetricsAggregation.SingleValue avg = response.getAggregations().get("avg");
+        assertThat(avg.value(), closeTo(sum / numDocs, 0.00000001));
+    }
+
+    public void testSearchWithType() throws Exception {
+        SearchRequest search = new SearchRequest(rollupIndex);
+        search.types("a", "b", "c");
+        search.source(new SearchSourceBuilder()
+                .size(0)
+                .aggregation(new AvgAggregationBuilder("avg").field("value")));
+        try {
+            highLevelClient().rollup().search(search, RequestOptions.DEFAULT);
+            fail("types are not allowed but didn't fail");
+        } catch (ValidationException e) {
+            assertEquals("Validation Failed: 1: types are not allowed in rollup search;", e.getMessage());
+        }
+    }
+
     public void testGetMissingRollupJob() throws Exception {
         GetRollupJobRequest getRollupJobRequest = new GetRollupJobRequest("missing");
         RollupClient rollupClient = highLevelClient().rollup();

client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/RollupDocumentationIT.java

@@ -27,6 +27,8 @@
 import org.elasticsearch.action.bulk.BulkRequest;
 import org.elasticsearch.action.bulk.BulkResponse;
 import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.client.ESRestHighLevelClientTestCase;
 import org.elasticsearch.client.RequestOptions;
@@ -60,6 +62,9 @@
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval;
+import org.elasticsearch.search.aggregations.metrics.NumericMetricsAggregation;
+import org.elasticsearch.search.aggregations.metrics.max.MaxAggregationBuilder;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.junit.Before;
 
 import java.io.IOException;
@@ -72,6 +77,7 @@
 import java.util.concurrent.TimeUnit;
 
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
+import static org.hamcrest.Matchers.closeTo;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.isOneOf;
@@ -89,7 +95,7 @@ public void setUpDocs() throws IOException {
                 .field("timestamp", String.format(Locale.ROOT, "2018-01-01T00:%02d:00Z", i))
                 .field("hostname", 0)
                 .field("datacenter", 0)
-                .field("temperature", 0)
+                .field("temperature", i)
                 .field("voltage", 0)
                 .field("load", 0)
                 .field("net_in", 0)
@@ -330,6 +336,56 @@ public void onFailure(Exception e) {
         assertTrue(latch.await(30L, TimeUnit.SECONDS));
     }
 
+    public void testSearch() throws Exception {
+        // Setup a rollup index to query
+        testCreateRollupJob();
+
+        RestHighLevelClient client = highLevelClient();
+
+        // tag::search-request
+        SearchRequest request = new SearchRequest();
+        request.source(new SearchSourceBuilder()
+            .size(0)
+            .aggregation(new MaxAggregationBuilder("max_temperature")
+                .field("temperature")));
+        // end::search-request
+
+        // tag::search-execute
+        SearchResponse response =
+            client.rollup().search(request, RequestOptions.DEFAULT);
+        // end::search-execute
+
+        // tag::search-response
+        NumericMetricsAggregation.SingleValue maxTemperature =
+                response.getAggregations().get("max_temperature");
+        assertThat(maxTemperature.value(), closeTo(49.0, .00001));
+        // end::search-response
+
+        ActionListener<SearchResponse> listener;
+        // tag::search-execute-listener
+        listener = new ActionListener<SearchResponse>() {
+            @Override
+            public void onResponse(SearchResponse response) {
+                 // <1>
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                // <2>
+            }
+        };
+        // end::search-execute-listener
+
+        final CountDownLatch latch = new CountDownLatch(1);
+        listener = new LatchedActionListener<>(listener, latch);
+
+        // tag::search-execute-async
+        client.rollup().searchAsync(request, RequestOptions.DEFAULT, listener); // <1>
+        // end::search-execute-async
+
+        assertTrue(latch.await(30L, TimeUnit.SECONDS));
+    }
+
     @SuppressWarnings("unused")
     public void testGetRollupCaps() throws Exception {
         RestHighLevelClient client = highLevelClient();

docs/java-rest/high-level/rollup/search.asciidoc

@@ -0,0 +1,45 @@
+--
+:api: search
+:request: SearchRequest
+:response: SearchResponse
+--
+
+[id="{upid}-{api}"]
+=== Rollup Search API
+
+The Rollup Search endpoint allows searching rolled-up data using the standard
+query DSL. The Rollup Search endpoint is needed because, internally,
+rolled-up documents utilize a different document structure than the original
+data. The Rollup Search endpoint rewrites standard query DSL into a format that
+matches the rollup documents, then takes the response and rewrites it back to
+what a client would expect given the original query.
+
+[id="{upid}-{api}-request"]
+==== Request
+
+Rollup Search uses the same +{request}+ that is used by the <<{mainid}-search>>
+but it is mostly for aggregations you should set the `size` to 0 and add
+aggregations like this:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests-file}[{api}-request]
+--------------------------------------------------
+
+NOTE:: Rollup Search is limited in many ways because only some query elements
+can be translated into queries against the rollup indices. See the main
+{ref}/rollup-search.html[Rollup Search] documentation for more.
+
+include::../execution.asciidoc[]
+
+[id="{upid}-{api}-response"]
+==== Response
+
+Rollup Search returns the same +{response}+ that is used by the
+<<{mainid}-search>> and everything can be accessed in exactly the same way.
+This will access the aggregation built by the example request above:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests-file}[{api}-response]
+--------------------------------------------------