From c0f92108f3611ef34d76ae024a147d930950e103 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Tue, 10 Apr 2018 17:41:18 +0200 Subject: [PATCH 1/2] Add tests with aliases to rank-eval The ranking evaluation requests so far were not tested against aliases but they should run regardless of the targeted index is a real index or an alias. This change adds cases for this to the integration and rest tests. Relates to #29011 --- .../index/rankeval/RankEvalRequestIT.java | 55 +++++++----- .../rest-api-spec/test/rank_eval/10_basic.yml | 90 +++++++++---------- 2 files changed, 77 insertions(+), 68 deletions(-) diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java index dc0bbddeb62b1..1fa801aa98caf 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java @@ -20,6 +20,7 @@ package org.elasticsearch.index.rankeval; import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest.AliasActions; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.index.IndexNotFoundException; @@ -40,9 +41,14 @@ import java.util.Set; import static org.elasticsearch.index.rankeval.EvaluationMetric.filterUnknownDocuments; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.instanceOf; public class RankEvalRequestIT extends ESIntegTestCase { + + private static final String TEST_INDEX = "test"; + private static final String INDEX_ALIAS = "alias0"; + @Override protected Collection> transportClientPlugins() { return Arrays.asList(RankEvalPlugin.class); @@ -55,20 +61,23 @@ protected Collection> nodePlugins() { @Before public void setup() { - createIndex("test"); + createIndex(TEST_INDEX); ensureGreen(); - client().prepareIndex("test", "testtype").setId("1") + client().prepareIndex(TEST_INDEX, "testtype").setId("1") .setSource("text", "berlin", "title", "Berlin, Germany", "population", 3670622).get(); - client().prepareIndex("test", "testtype").setId("2").setSource("text", "amsterdam", "population", 851573).get(); - client().prepareIndex("test", "testtype").setId("3").setSource("text", "amsterdam", "population", 851573).get(); - client().prepareIndex("test", "testtype").setId("4").setSource("text", "amsterdam", "population", 851573).get(); - client().prepareIndex("test", "testtype").setId("5").setSource("text", "amsterdam", "population", 851573).get(); - client().prepareIndex("test", "testtype").setId("6").setSource("text", "amsterdam", "population", 851573).get(); + client().prepareIndex(TEST_INDEX, "testtype").setId("2").setSource("text", "amsterdam", "population", 851573).get(); + client().prepareIndex(TEST_INDEX, "testtype").setId("3").setSource("text", "amsterdam", "population", 851573).get(); + client().prepareIndex(TEST_INDEX, "testtype").setId("4").setSource("text", "amsterdam", "population", 851573).get(); + client().prepareIndex(TEST_INDEX, "testtype").setId("5").setSource("text", "amsterdam", "population", 851573).get(); + client().prepareIndex(TEST_INDEX, "testtype").setId("6").setSource("text", "amsterdam", "population", 851573).get(); // add another index for testing closed indices etc... client().prepareIndex("test2", "testtype").setId("7").setSource("text", "amsterdam", "population", 851573).get(); refresh(); + + // set up an alias that can also be used in tests + assertAcked(client().admin().indices().prepareAliases().addAliasAction(AliasActions.add().index(TEST_INDEX).alias(INDEX_ALIAS))); } /** @@ -98,7 +107,9 @@ public void testPrecisionAtRequest() { RankEvalAction.INSTANCE, new RankEvalRequest()); builder.setRankEvalSpec(task); - RankEvalResponse response = client().execute(RankEvalAction.INSTANCE, builder.request().indices("test")) + // String indexToUse = randomBoolean() ? TEST_INDEX : INDEX_ALIAS; + String indexToUse = INDEX_ALIAS; + RankEvalResponse response = client().execute(RankEvalAction.INSTANCE, builder.request().indices(indexToUse)) .actionGet(); // the expected Prec@ for the first query is 4/6 and the expected Prec@ for the // second is 1/6, divided by 2 to get the average @@ -140,7 +151,7 @@ public void testPrecisionAtRequest() { metric = new PrecisionAtK(1, false, 3); task = new RankEvalSpec(specifications, metric); - builder = new RankEvalRequestBuilder(client(), RankEvalAction.INSTANCE, new RankEvalRequest(task, new String[] { "test" })); + builder = new RankEvalRequestBuilder(client(), RankEvalAction.INSTANCE, new RankEvalRequest(task, new String[] { TEST_INDEX })); response = client().execute(RankEvalAction.INSTANCE, builder.request()).actionGet(); // if we look only at top 3 documente, the expected P@3 for the first query is @@ -160,19 +171,19 @@ public void testDCGRequest() { List specifications = new ArrayList<>(); List ratedDocs = Arrays.asList( - new RatedDocument("test", "1", 3), - new RatedDocument("test", "2", 2), - new RatedDocument("test", "3", 3), - new RatedDocument("test", "4", 0), - new RatedDocument("test", "5", 1), - new RatedDocument("test", "6", 2)); + new RatedDocument(TEST_INDEX, "1", 3), + new RatedDocument(TEST_INDEX, "2", 2), + new RatedDocument(TEST_INDEX, "3", 3), + new RatedDocument(TEST_INDEX, "4", 0), + new RatedDocument(TEST_INDEX, "5", 1), + new RatedDocument(TEST_INDEX, "6", 2)); specifications.add(new RatedRequest("amsterdam_query", ratedDocs, testQuery)); DiscountedCumulativeGain metric = new DiscountedCumulativeGain(false, null, 10); RankEvalSpec task = new RankEvalSpec(specifications, metric); RankEvalRequestBuilder builder = new RankEvalRequestBuilder(client(), RankEvalAction.INSTANCE, - new RankEvalRequest(task, new String[] { "test" })); + new RankEvalRequest(task, new String[] { TEST_INDEX })); RankEvalResponse response = client().execute(RankEvalAction.INSTANCE, builder.request()).actionGet(); assertEquals(DiscountedCumulativeGainTests.EXPECTED_DCG, response.getEvaluationResult(), 10E-14); @@ -181,7 +192,7 @@ public void testDCGRequest() { metric = new DiscountedCumulativeGain(false, null, 3); task = new RankEvalSpec(specifications, metric); - builder = new RankEvalRequestBuilder(client(), RankEvalAction.INSTANCE, new RankEvalRequest(task, new String[] { "test" })); + builder = new RankEvalRequestBuilder(client(), RankEvalAction.INSTANCE, new RankEvalRequest(task, new String[] { TEST_INDEX })); response = client().execute(RankEvalAction.INSTANCE, builder.request()).actionGet(); assertEquals(12.39278926071437, response.getEvaluationResult(), 10E-14); @@ -200,7 +211,7 @@ public void testMRRRequest() { RankEvalSpec task = new RankEvalSpec(specifications, metric); RankEvalRequestBuilder builder = new RankEvalRequestBuilder(client(), RankEvalAction.INSTANCE, - new RankEvalRequest(task, new String[] { "test" })); + new RankEvalRequest(task, new String[] { TEST_INDEX })); RankEvalResponse response = client().execute(RankEvalAction.INSTANCE, builder.request()).actionGet(); // the expected reciprocal rank for the amsterdam_query is 1/5 @@ -213,7 +224,7 @@ public void testMRRRequest() { metric = new MeanReciprocalRank(1, 3); task = new RankEvalSpec(specifications, metric); - builder = new RankEvalRequestBuilder(client(), RankEvalAction.INSTANCE, new RankEvalRequest(task, new String[] { "test" })); + builder = new RankEvalRequestBuilder(client(), RankEvalAction.INSTANCE, new RankEvalRequest(task, new String[] { TEST_INDEX })); response = client().execute(RankEvalAction.INSTANCE, builder.request()).actionGet(); // limiting to top 3 results, the amsterdam_query has no relevant document in it @@ -244,7 +255,7 @@ public void testBadQuery() { RankEvalSpec task = new RankEvalSpec(specifications, new PrecisionAtK()); RankEvalRequestBuilder builder = new RankEvalRequestBuilder(client(), RankEvalAction.INSTANCE, - new RankEvalRequest(task, new String[] { "test" })); + new RankEvalRequest(task, new String[] { TEST_INDEX })); builder.setRankEvalSpec(task); RankEvalResponse response = client().execute(RankEvalAction.INSTANCE, builder.request()).actionGet(); @@ -264,7 +275,7 @@ public void testIndicesOptions() { specifications.add(new RatedRequest("amsterdam_query", relevantDocs, amsterdamQuery)); RankEvalSpec task = new RankEvalSpec(specifications, new PrecisionAtK()); - RankEvalRequest request = new RankEvalRequest(task, new String[] { "test", "test2" }); + RankEvalRequest request = new RankEvalRequest(task, new String[] { TEST_INDEX, "test2" }); request.setRankEvalSpec(task); RankEvalResponse response = client().execute(RankEvalAction.INSTANCE, request).actionGet(); @@ -322,7 +333,7 @@ public void testIndicesOptions() { private static List createRelevant(String... docs) { List relevant = new ArrayList<>(); for (String doc : docs) { - relevant.add(new RatedDocument("test", doc, TestRatingEnum.RELEVANT.ordinal())); + relevant.add(new RatedDocument(TEST_INDEX, doc, TestRatingEnum.RELEVANT.ordinal())); } return relevant; } diff --git a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yml b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yml index fcf5f945a06ae..3900b1f32baa7 100644 --- a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yml +++ b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yml @@ -1,10 +1,4 @@ ---- -"Response format": - - - skip: - version: " - 6.2.99" - reason: response format was updated in 6.3 - +setup: - do: indices.create: index: foo @@ -43,8 +37,21 @@ - do: indices.refresh: {} + - do: + indices.put_alias: + index: foo + name: alias + +--- +"Response format": + + - skip: + version: " - 6.2.99" + reason: response format was updated in 6.3 + - do: rank_eval: + index: foo, body: { "requests" : [ { @@ -84,52 +91,43 @@ - match: { details.berlin_query.hits.0.hit._id: "doc1" } - match: { details.berlin_query.hits.0.rating: 1} - match: { details.berlin_query.hits.1.hit._id: "doc4" } - - is_false: details.berlin_query.hits.1.rating + - is_false: details.berlin_query.hits.1.rating --- -"Mean Reciprocal Rank": - - - skip: - version: " - 6.2.99" - reason: response format was updated in 6.3 +"Alias resolution": - do: - indices.create: - index: foo - body: - settings: - index: - number_of_shards: 1 - - do: - index: - index: foo - type: bar - id: doc1 - body: { "text": "berlin" } + rank_eval: + index: alias + body: { + "requests" : [ + { + "id": "amsterdam_query", + "request": { "query": { "match" : {"text" : "amsterdam" }}}, + "ratings": [ + {"_index": "foo", "_id": "doc1", "rating": 0}, + {"_index": "foo", "_id": "doc2", "rating": 1}, + {"_index": "foo", "_id": "doc3", "rating": 1}] + }, + { + "id" : "berlin_query", + "request": { "query": { "match" : { "text" : "berlin" } }, "size" : 10 }, + "ratings": [{"_index": "foo", "_id": "doc1", "rating": 1}] + } + ], + "metric" : { "precision": { "ignore_unlabeled" : true }} + } - - do: - index: - index: foo - type: bar - id: doc2 - body: { "text": "amsterdam" } + - match: { quality_level: 1} + - match: { details.amsterdam_query.quality_level: 1.0} + - match: { details.berlin_query.quality_level: 1.0} - - do: - index: - index: foo - type: bar - id: doc3 - body: { "text": "amsterdam" } - - - do: - index: - index: foo - type: bar - id: doc4 - body: { "text": "something about amsterdam and berlin" } +--- +"Mean Reciprocal Rank": - - do: - indices.refresh: {} + - skip: + version: " - 6.2.99" + reason: response format was updated in 6.3 - do: rank_eval: From 8b823da84ed682ae1fdb29e75ecc42106c0ee8cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Thu, 19 Apr 2018 14:15:53 +0200 Subject: [PATCH 2/2] Correcting test --- .../org/elasticsearch/index/rankeval/RankEvalRequestIT.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java index 1fa801aa98caf..f0809e1780767 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java @@ -107,8 +107,7 @@ public void testPrecisionAtRequest() { RankEvalAction.INSTANCE, new RankEvalRequest()); builder.setRankEvalSpec(task); - // String indexToUse = randomBoolean() ? TEST_INDEX : INDEX_ALIAS; - String indexToUse = INDEX_ALIAS; + String indexToUse = randomBoolean() ? TEST_INDEX : INDEX_ALIAS; RankEvalResponse response = client().execute(RankEvalAction.INSTANCE, builder.request().indices(indexToUse)) .actionGet(); // the expected Prec@ for the first query is 4/6 and the expected Prec@ for the