Experiment: Module for computing prec@ on queries

core/src/main/java/org/elasticsearch/client/transport/support/TransportProxyClient.java

@@ -32,6 +32,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.transport.TransportService;
 
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 

modules/rank-eval/build.gradle

@@ -0,0 +1,30 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+esplugin {
+  description 'The Rank Eval module adds APIs to evaluate ranking quality.'
+  classname 'org.elasticsearch.index.rankeval.RankEvalPlugin'
+}
+
+integTest {
+  cluster {
+    setting 'script.inline', 'true'
+    setting 'script.stored', 'true'
+  }
+}

modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/EvalQueryQuality.java

@@ -0,0 +1,45 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.rankeval;
+
+import java.util.Collection;
+
+/** Returned for each search intent and search specification combination. Summarises the document ids found that were not
+ * annotated and the average precision of result sets in each particular combination based on the annotations given. 
+ * */
+public class EvalQueryQuality {
+    private double qualityLevel;
+
+    private Collection<String> unknownDocs;
+
+    public EvalQueryQuality (double qualityLevel, Collection<String> unknownDocs) {
+       this.qualityLevel = qualityLevel;
+       this.unknownDocs = unknownDocs;
+    }
+
+    public Collection<String> getUnknownDocs() {
+        return unknownDocs;
+    }
+
+    public double getQualityLevel() {
+        return qualityLevel;          
+    }
+
+}

modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/Evaluator.java

@@ -0,0 +1,28 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.rankeval;
+
+import org.elasticsearch.common.io.stream.NamedWriteable;
+import org.elasticsearch.search.SearchHit;
+
+public interface Evaluator extends NamedWriteable {
+
+    public Object evaluate(SearchHit[] hits, RatedQuery intent);
+}

modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java

@@ -0,0 +1,144 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.rankeval;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.search.SearchHit;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import javax.naming.directory.SearchResult;
+
+/**
+ * Evaluate Precision at N, N being the number of search results to consider for precision calculation.
+ * 
+ * Documents of unkonwn quality are ignored in the precision at n computation and returned by document id.
+ * */
+public class PrecisionAtN implements RankedListQualityMetric {
+
+    /** Number of results to check against a given set of relevant results. */
+    private int n;
+
+    public static final String NAME = "precisionatn";
+
+    public PrecisionAtN(StreamInput in) throws IOException {
+        n = in.readInt();
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeInt(n);
+    }
+
+    @Override
+    public String getWriteableName() {
+        return NAME;
+    }
+
+    /**
+     * Initialises n with 10
+     * */
+    public PrecisionAtN() {
+        this.n = 10;
+    }
+
+    /**
+     * @param n number of top results to check against a given set of relevant results.
+     * */
+    public PrecisionAtN(int n) {
+        this.n= n;
+    }
+
+    /**
+     * Return number of search results to check for quality.
+     * */
+    public int getN() {
+        return n;
+    }
+
+    /** Compute precisionAtN based on provided relevant document IDs.
+     * @return precision at n for above {@link SearchResult} list.
+     **/
+    @Override
+    public EvalQueryQuality evaluate(SearchHit[] hits, RatedQuery intent) {
+        Map<String, Integer> ratedDocIds = intent.getRatedDocuments();
+
+        Collection<String> relevantDocIds = new ArrayList<>(); 
+        for (Entry<String, Integer> entry : ratedDocIds.entrySet()) {
+            if (Rating.RELEVANT.equals(RatingMapping.mapTo(entry.getValue()))) {
+                relevantDocIds.add(entry.getKey());
+            }
+        }
+
+        Collection<String> irrelevantDocIds = new ArrayList<>(); 
+        for (Entry<String, Integer> entry : ratedDocIds.entrySet()) {
+            if (Rating.IRRELEVANT.equals(RatingMapping.mapTo(entry.getValue()))) {
+                irrelevantDocIds.add(entry.getKey());
+            }
+        }
+
+        int good = 0;
+        int bad = 0;
+        Collection<String> unknownDocIds = new ArrayList<String>();
+        for (int i = 0; (i < n && i < hits.length); i++) {
+            String id = hits[i].getId();
+            if (relevantDocIds.contains(id)) {
+                good++;
+            } else if (irrelevantDocIds.contains(id)) {
+                bad++;
+            } else {
+                unknownDocIds.add(id);
+            }
+        }
+
+        double precision = (double) good / (good + bad);
+
+        return new EvalQueryQuality(precision, unknownDocIds);
+    }
+
+    public enum Rating {
+        RELEVANT, IRRELEVANT;
+    }
+
+    /**
+     * Needed to get the enum accross serialisation boundaries.
+     * */
+    public static class RatingMapping {
+        public static Integer mapFrom(Rating rating) {
+            if (Rating.RELEVANT.equals(rating)) {
+                return 0;
+            }
+            return 1;
+        }
+
+        public static Rating mapTo(Integer rating) {
+            if (rating == 0) {
+                return Rating.RELEVANT;
+            }
+            return Rating.IRRELEVANT;
+        }
+    }
+
+}

modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/QuerySpec.java

@@ -0,0 +1,114 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.rankeval;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Defines a QA specification: All end user supplied query intents will be mapped to the search request specified in this search request
+ * template and executed against the targetIndex given. Any filters that should be applied in the target system can be specified as well.
+ *
+ * The resulting document lists can then be compared against what was specified in the set of rated documents as part of a QAQuery.  
+ * */
+public class QuerySpec implements Writeable {
+
+    private int specId = 0;
+    private SearchSourceBuilder testRequest;
+    private List<String> indices = new ArrayList<>();
+    private List<String> types = new ArrayList<>();
+
+    public QuerySpec(
+            int specId, SearchSourceBuilder testRequest, List<String> indices, List<String> types) {
+        this.specId = specId;
+        this.testRequest = testRequest;
+        this.indices = indices;
+        this.types = types;
+    }
+
+    public QuerySpec(StreamInput in) throws IOException {
+        this.specId = in.readInt();
+        testRequest = new SearchSourceBuilder(in);
+        int indicesSize = in.readInt();
+        indices = new ArrayList<String>(indicesSize);
+        for (int i = 0; i < indicesSize; i++) {
+            this.indices.add(in.readString());
+        }
+        int typesSize = in.readInt();
+        types = new ArrayList<String>(typesSize);
+        for (int i = 0; i < typesSize; i++) {
+            this.types.add(in.readString());
+        }
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeInt(specId);
+        testRequest.writeTo(out);
+        out.writeInt(indices.size());
+        for (String index : indices) {
+            out.writeString(index);
+        }
+        out.writeInt(types.size());
+        for (String type : types) {
+            out.writeString(type);
+        }
+    }
+
+    public SearchSourceBuilder getTestRequest() {
+        return testRequest;
+    }
+
+    public void setTestRequest(SearchSourceBuilder testRequest) {
+        this.testRequest = testRequest;
+    }
+
+    public List<String> getIndices() {
+        return indices;
+    }
+
+    public void setIndices(List<String> indices) {
+        this.indices = indices;
+    }
+
+    public List<String> getTypes() {
+        return types;
+    }
+
+    public void setTypes(List<String> types) {
+        this.types = types;
+    }
+
+    /** Returns a user supplied spec id for easier referencing. */
+    public int getSpecId() {
+        return specId;
+    }
+
+    /** Sets a user supplied spec id for easier referencing. */
+    public void setSpecId(int specId) {
+        this.specId = specId;
+    }
+}