Add ranking evaluation API to High Level Rest Client

client/rest-high-level/src/main/java/org/elasticsearch/client/Request.java

@@ -63,6 +63,7 @@
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.VersionType;
+import org.elasticsearch.index.rankeval.RankEvalRequest;
 import org.elasticsearch.rest.action.search.RestSearchAction;
 import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
 
@@ -71,6 +72,7 @@
 import java.nio.charset.Charset;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
@@ -484,6 +486,16 @@ static Request existsAlias(GetAliasesRequest getAliasesRequest) {
         return new Request(HttpHead.METHOD_NAME, endpoint, params.getParams(), null);
     }
 
+    static Request rankEval(RankEvalRequest rankEvalRequest) throws IOException {
+        // TODO maybe indices should be propery of RankEvalRequest and not of the spec
+        List<String> indices = rankEvalRequest.getRankEvalSpec().getIndices();
+        String endpoint = endpoint(indices.toArray(new String[indices.size()]), Strings.EMPTY_ARRAY, "_rank_eval");
+        HttpEntity entity = null;
+        entity = createEntity(rankEvalRequest.getRankEvalSpec(), REQUEST_BODY_CONTENT_TYPE);
+        return new Request(HttpGet.METHOD_NAME, endpoint, Collections.emptyMap(), entity);
+
+    }
+
     private static HttpEntity createEntity(ToXContent toXContent, XContentType xContentType) throws IOException {
         BytesRef source = XContentHelper.toXContent(toXContent, xContentType, false).toBytesRef();
         return new ByteArrayEntity(source.bytes, source.offset, source.length, createContentType(xContentType));

client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java

@@ -54,6 +54,8 @@
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.index.rankeval.RankEvalRequest;
+import org.elasticsearch.index.rankeval.RankEvalResponse;
 import org.elasticsearch.plugins.spi.NamedXContentProvider;
 import org.elasticsearch.rest.BytesRestResponse;
 import org.elasticsearch.rest.RestStatus;
@@ -467,6 +469,27 @@ public final void clearScrollAsync(ClearScrollRequest clearScrollRequest,
                 listener, emptySet(), headers);
     }
 
+    /**
+     * Executes a request using the Ranking Evaluation API.
+     *
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/search-rank-eval.html">Ranking Evaluation API
+     * on elastic.co</a>
+     */
+    public final RankEvalResponse rankEval(RankEvalRequest rankEvalRequest, Header... headers) throws IOException {
+        return performRequestAndParseEntity(rankEvalRequest, Request::rankEval, RankEvalResponse::fromXContent, emptySet(), headers);
+    }
+
+    /**
+     * Asynchronously executes a request using the Ranking Evaluation API.
+     *
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/search-rank-eval.html">Ranking Evaluation API
+     * on elastic.co</a>
+     */
+    public final void rankEvalAsync(RankEvalRequest rankEvalRequest, ActionListener<RankEvalResponse> listener, Header... headers) {
+        performRequestAsyncAndParseEntity(rankEvalRequest, Request::rankEval, RankEvalResponse::fromXContent, listener, emptySet(),
+                headers);
+    }
+
     protected final <Req extends ActionRequest, Resp> Resp performRequestAndParseEntity(Req request,
                                                                             CheckedFunction<Req, Request, IOException> requestConverter,
                                                                             CheckedFunction<XContentParser, Resp, IOException> entityParser,

client/rest-high-level/src/test/java/org/elasticsearch/client/RankEvalIT.java

@@ -0,0 +1,120 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client;
+
+import org.apache.http.entity.ContentType;
+import org.apache.http.entity.StringEntity;
+import org.elasticsearch.index.query.MatchAllQueryBuilder;
+import org.elasticsearch.index.rankeval.EvalQueryQuality;
+import org.elasticsearch.index.rankeval.PrecisionAtK;
+import org.elasticsearch.index.rankeval.RankEvalRequest;
+import org.elasticsearch.index.rankeval.RankEvalResponse;
+import org.elasticsearch.index.rankeval.RankEvalSpec;
+import org.elasticsearch.index.rankeval.RatedDocument;
+import org.elasticsearch.index.rankeval.RatedRequest;
+import org.elasticsearch.index.rankeval.RatedSearchHit;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map.Entry;
+import java.util.Set;
+
+import static org.elasticsearch.index.rankeval.EvaluationMetric.filterUnknownDocuments;
+
+public class RankEvalIT extends ESRestHighLevelClientTestCase {
+
+    @Before
+    public void indexDocuments() throws IOException {
+        StringEntity doc = new StringEntity("{\"text\":\"berlin\"}", ContentType.APPLICATION_JSON);
+        client().performRequest("PUT", "/index/doc/1", Collections.emptyMap(), doc);
+        doc = new StringEntity("{\"text\":\"amsterdam\"}", ContentType.APPLICATION_JSON);
+        client().performRequest("PUT", "/index/doc/2", Collections.emptyMap(), doc);
+        client().performRequest("PUT", "/index/doc/3", Collections.emptyMap(), doc);
+        client().performRequest("PUT", "/index/doc/4", Collections.emptyMap(), doc);
+        client().performRequest("PUT", "/index/doc/5", Collections.emptyMap(), doc);
+        client().performRequest("PUT", "/index/doc/6", Collections.emptyMap(), doc);
+        client().performRequest("POST", "/index/_refresh");
+    }
+
+    /**
+     * Test cases retrieves all six documents indexed above and checks the Prec@10
+     * calculation where all unlabeled documents are treated as not relevant.
+     */
+    public void testRankEvalRequest() throws IOException {
+        SearchSourceBuilder testQuery = new SearchSourceBuilder();
+        testQuery.query(new MatchAllQueryBuilder());
+        RatedRequest amsterdamRequest = new RatedRequest("amsterdam_query", createRelevant("index" , "2", "3", "4", "5"), testQuery);
+        RatedRequest berlinRequest = new RatedRequest("berlin_query", createRelevant("index", "1"), testQuery);
+        List<RatedRequest> specifications = new ArrayList<>();
+        specifications.add(amsterdamRequest);
+        specifications.add(berlinRequest);
+        PrecisionAtK metric = new PrecisionAtK(1, false, 10);
+        RankEvalSpec spec = new RankEvalSpec(specifications, metric);
+        spec.addIndices(Collections.singletonList("index"));
+
+        RankEvalResponse response = execute(new RankEvalRequest(spec), highLevelClient()::rankEval, highLevelClient()::rankEvalAsync);
+        // the expected Prec@ for the first query is 4/6 and the expected Prec@ for the second is 1/6, divided by 2 to get the average
+        double expectedPrecision = (1.0 / 6.0 + 4.0 / 6.0) / 2.0;
+        assertEquals(expectedPrecision, response.getEvaluationResult(), Double.MIN_VALUE);
+        Set<Entry<String, EvalQueryQuality>> entrySet = response.getPartialResults().entrySet();
+        assertEquals(2, entrySet.size());
+        for (Entry<String, EvalQueryQuality> entry : entrySet) {
+            EvalQueryQuality quality = entry.getValue();
+            if (entry.getKey() == "amsterdam_query") {
+                assertEquals(2, filterUnknownDocuments(quality.getHitsAndRatings()).size());
+                List<RatedSearchHit> hitsAndRatings = quality.getHitsAndRatings();
+                assertEquals(6, hitsAndRatings.size());
+                for (RatedSearchHit hit : hitsAndRatings) {
+                    String id = hit.getSearchHit().getId();
+                    if (id.equals("1") || id.equals("6")) {
+                        assertFalse(hit.getRating().isPresent());
+                    } else {
+                        assertEquals(1, hit.getRating().get().intValue());
+                    }
+                }
+            }
+            if (entry.getKey() == "berlin_query") {
+                assertEquals(5, filterUnknownDocuments(quality.getHitsAndRatings()).size());
+                List<RatedSearchHit> hitsAndRatings = quality.getHitsAndRatings();
+                assertEquals(6, hitsAndRatings.size());
+                for (RatedSearchHit hit : hitsAndRatings) {
+                    String id = hit.getSearchHit().getId();
+                    if (id.equals("1")) {
+                        assertEquals(1, hit.getRating().get().intValue());
+                    } else {
+                        assertFalse(hit.getRating().isPresent());
+                    }
+                }
+            }
+        }
+    }
+
+    private static List<RatedDocument> createRelevant(String indexName, String... docs) {
+        List<RatedDocument> relevant = new ArrayList<>();
+        for (String doc : docs) {
+            relevant.add(new RatedDocument(indexName, doc, 1));
+        }
+        return relevant;
+    }
+}

client/rest-high-level/src/test/java/org/elasticsearch/client/RequestTests.java

@@ -70,6 +70,11 @@
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.VersionType;
 import org.elasticsearch.index.query.TermQueryBuilder;
+import org.elasticsearch.index.rankeval.PrecisionAtK;
+import org.elasticsearch.index.rankeval.RankEvalRequest;
+import org.elasticsearch.index.rankeval.RankEvalSpec;
+import org.elasticsearch.index.rankeval.RatedRequest;
+import org.elasticsearch.index.rankeval.RestRankEvalAction;
 import org.elasticsearch.rest.action.search.RestSearchAction;
 import org.elasticsearch.search.Scroll;
 import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
@@ -89,6 +94,8 @@
 import java.lang.reflect.Constructor;
 import java.lang.reflect.Modifier;
 import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Locale;
@@ -1021,6 +1028,26 @@ public void testExistsAliasNoAliasNoIndex() {
         IllegalArgumentException iae = expectThrows(IllegalArgumentException.class, () -> Request.existsAlias(getAliasesRequest));
         assertEquals("existsAlias requires at least an alias or an index", iae.getMessage());
     }
+
+    public void testRankEval() throws Exception {
+        RankEvalSpec spec = new RankEvalSpec(
+                Collections.singletonList(new RatedRequest("queryId", Collections.emptyList(), new SearchSourceBuilder())),
+                new PrecisionAtK());
+        String[] indices = randomIndicesNames(0, 5);
+        spec.addIndices(Arrays.asList(indices));
+        RankEvalRequest rankEvalRequest = new RankEvalRequest(spec);
+
+        Request request = Request.rankEval(rankEvalRequest);
+        StringJoiner endpoint = new StringJoiner("/", "/", "");
+        String index = String.join(",", indices);
+        if (Strings.hasLength(index)) {
+            endpoint.add(index);
+        }
+        endpoint.add(RestRankEvalAction.ENDPOINT);
+        assertEquals(endpoint.toString(), request.getEndpoint());
+        assertEquals(Collections.emptyMap(), request.getParameters());
+        assertToXContentBody(spec, request.getEntity());
+    }
 
     private static void assertToXContentBody(ToXContent expectedBody, HttpEntity actualEntity) throws IOException {
         BytesReference expectedBytes = XContentHelper.toXContent(expectedBody, REQUEST_BODY_CONTENT_TYPE, false);

client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java

@@ -64,6 +64,7 @@
 import org.elasticsearch.index.rankeval.DiscountedCumulativeGain;
 import org.elasticsearch.index.rankeval.EvaluationMetric;
 import org.elasticsearch.index.rankeval.MeanReciprocalRank;
+import org.elasticsearch.index.rankeval.MetricDetail;
 import org.elasticsearch.index.rankeval.PrecisionAtK;
 import org.elasticsearch.join.aggregations.ChildrenAggregationBuilder;
 import org.elasticsearch.rest.RestStatus;
@@ -656,7 +657,7 @@ public void testDefaultNamedXContents() {
 
     public void testProvidedNamedXContents() {
         List<NamedXContentRegistry.Entry> namedXContents = RestHighLevelClient.getProvidedNamedXContents();
-        assertEquals(5, namedXContents.size());
+        assertEquals(7, namedXContents.size());
         Map<Class<?>, Integer> categories = new HashMap<>();
         List<String> names = new ArrayList<>();
         for (NamedXContentRegistry.Entry namedXContent : namedXContents) {
@@ -666,14 +667,17 @@ public void testProvidedNamedXContents() {
                 categories.put(namedXContent.categoryClass, counter + 1);
             }
         }
-        assertEquals(2, categories.size());
+        assertEquals(3, categories.size());
         assertEquals(Integer.valueOf(2), categories.get(Aggregation.class));
         assertTrue(names.contains(ChildrenAggregationBuilder.NAME));
         assertTrue(names.contains(MatrixStatsAggregationBuilder.NAME));
         assertEquals(Integer.valueOf(3), categories.get(EvaluationMetric.class));
         assertTrue(names.contains(PrecisionAtK.NAME));
         assertTrue(names.contains(DiscountedCumulativeGain.NAME));
         assertTrue(names.contains(MeanReciprocalRank.NAME));
+        assertEquals(Integer.valueOf(2), categories.get(MetricDetail.class));
+        assertTrue(names.contains(PrecisionAtK.NAME));
+        assertTrue(names.contains(MeanReciprocalRank.NAME));
     }
 
     private static class TrackingActionListener implements ActionListener<Integer> {

docs/reference/search/rank-eval.asciidoc

@@ -283,8 +283,10 @@ that shows potential errors of individual queries. The response has the followin
                     }, [...]
                 ],
                 "metric_details": { <6>
-                    "relevant_docs_retrieved": 6,
-                    "docs_retrieved": 10
+                    "precision" : {
+                        "relevant_docs_retrieved": 6,
+                        "docs_retrieved": 10
+                    }
                 }
             },
             "my_query_id2" : { [...] }

modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGain.java

@@ -164,7 +164,7 @@ private static double computeDCG(List<Integer> ratings) {
     private static final ParseField K_FIELD = new ParseField("k");
     private static final ParseField NORMALIZE_FIELD = new ParseField("normalize");
     private static final ParseField UNKNOWN_DOC_RATING_FIELD = new ParseField("unknown_doc_rating");
-    private static final ConstructingObjectParser<DiscountedCumulativeGain, Void> PARSER = new ConstructingObjectParser<>("dcg_at",
+    private static final ConstructingObjectParser<DiscountedCumulativeGain, Void> PARSER = new ConstructingObjectParser<>("dcg_at", true,
             args -> {
                 Boolean normalized = (Boolean) args[0];
                 Integer optK = (Integer) args[2];