Add "optimized" as a sort parameter

Author: mayya-sharipova

docs/reference/search/request/sort.asciidoc

@@ -236,6 +236,54 @@ POST /index_long,index_double/_search
 To avoid overflow, the conversion to `date_nanos` cannot be applied on dates before
 1970 and after 2262 as nanoseconds are represented as longs.
 
+
+==== Sort Optimization
+experimental[]
+
+Sort on `long`, `date` and `date_nanos` fields can be optionally optimized.
+The optimization instead of sorting all field values, will underneath use the
+<<query-dsl-distance-feature-query, distance_feature>>
+query that can efficiently skip non-competitive hits.
+For the optimization to work, the field must be indexed and
+doc-values must be activated.
+Depending on the data this could bring significant speedups, while
+some performance regression is also possible.
+
+The optimization is disabled by default. To enable optimization,
+set the `optimized` parameter to `true`.
+This example shows how to enable optimization on a `long` field:
+
+[source,js]
+--------------------------------------------------
+PUT /index_long
+{
+    "mappings": {
+        "properties": {
+            "my_long": { "type": "long" }
+        }
+    }
+}
+--------------------------------------------------
+// CONSOLE
+
+[source,js]
+--------------------------------------------------
+POST /index_long/_search
+{
+    "sort" : [
+        {
+            "my_long" : {
+                "optimized": true,
+                "order" : "asc"
+            }
+        }
+    ]
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[continued]
+
+
 [[nested-sorting]]
 ==== Sorting within nested objects.
 

server/src/main/java/org/elasticsearch/search/query/QueryPhase.java

@@ -356,6 +356,8 @@ private static Query tryRewriteLongSort(SearchContext searchContext, IndexReader
         if (searchContext.collapse() != null) return null;
         if (searchContext.trackScores()) return null;
         if (searchContext.aggregations() != null) return null;
+        if (searchContext.sort().optimized == null) return null;
+        if (searchContext.sort().optimized[0] == false) return null;
         Sort sort = searchContext.sort().sort;
         SortField sortField = sort.getSort()[0];
         if (SortField.Type.LONG.equals(IndexSortConfig.getSortFieldType(sortField)) == false) return null;

server/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java

@@ -60,6 +60,7 @@ public class FieldSortBuilder extends SortBuilder<FieldSortBuilder> {
     public static final ParseField SORT_MODE = new ParseField("mode");
     public static final ParseField UNMAPPED_TYPE = new ParseField("unmapped_type");
     public static final ParseField NUMERIC_TYPE = new ParseField("numeric_type");
+    public static final ParseField OPTIMIZED_FIELD = new ParseField("optimized");
 
     /**
      * special field name to sort by index order
@@ -86,6 +87,8 @@ public class FieldSortBuilder extends SortBuilder<FieldSortBuilder> {
 
     private NestedSortBuilder nestedSort;
 
+    private boolean optimized = false;
+
     /** Copy constructor. */
     public FieldSortBuilder(FieldSortBuilder template) {
         this(template.fieldName);
@@ -101,6 +104,7 @@ public FieldSortBuilder(FieldSortBuilder template) {
             this.setNestedSort(template.getNestedSort());
         }
         this.numericType = template.numericType;
+        this.optimized(template.optimized);
     }
 
     /**
@@ -131,6 +135,9 @@ public FieldSortBuilder(StreamInput in) throws IOException {
         if (in.getVersion().onOrAfter(Version.V_7_2_0)) {
             numericType = in.readOptionalString();
         }
+        if (in.getVersion().onOrAfter(Version.V_8_0_0)) {
+            optimized = in.readOptionalBoolean();
+        }
     }
 
     @Override
@@ -146,6 +153,9 @@ public void writeTo(StreamOutput out) throws IOException {
         if (out.getVersion().onOrAfter(Version.V_7_2_0)) {
             out.writeOptionalString(numericType);
         }
+        if (out.getVersion().onOrAfter(Version.V_8_0_0)) {
+            out.writeOptionalBoolean(optimized);
+        }
     }
 
     /** Returns the document field this sort should be based on. */
@@ -316,6 +326,20 @@ public FieldSortBuilder setNumericType(String numericType) {
         return this;
     }
 
+    /**
+     * Sets if the sort should be optimized
+     * Applicable only for numeric long, date and date_nano fields.
+     */
+    public FieldSortBuilder optimized(boolean optimized) {
+        this.optimized = optimized;
+        return this;
+    }
+
+    /** Returns if the sort is optimized. */
+    public boolean optimized() {
+        return optimized;
+    }
+
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
@@ -342,6 +366,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         if (numericType != null) {
             builder.field(NUMERIC_TYPE.getPreferredName(), numericType);
         }
+        if (optimized) {
+            builder.field(OPTIMIZED_FIELD.getPreferredName(), optimized);
+        }
         builder.endObject();
         builder.endObject();
         return builder;
@@ -425,7 +452,7 @@ public SortFieldAndFormat build(QueryShardContext context) throws IOException {
             } else {
                 field = fieldData.sortField(missing, localSortMode, nested, reverse);
             }
-            return new SortFieldAndFormat(field, fieldType.docValueFormat(null, null));
+            return new SortFieldAndFormat(field, fieldType.docValueFormat(null, null), optimized);
         }
     }
 
@@ -444,13 +471,14 @@ public boolean equals(Object other) {
                 && Objects.equals(this.nestedPath, builder.nestedPath) && Objects.equals(this.missing, builder.missing)
                 && Objects.equals(this.order, builder.order) && Objects.equals(this.sortMode, builder.sortMode)
                 && Objects.equals(this.unmappedType, builder.unmappedType) && Objects.equals(this.nestedSort, builder.nestedSort))
-                && Objects.equals(this.numericType, builder.numericType);
+                && Objects.equals(this.numericType, builder.numericType)
+                && this.optimized == builder.optimized;
     }
 
     @Override
     public int hashCode() {
         return Objects.hash(this.fieldName, this.nestedFilter, this.nestedPath, this.nestedSort, this.missing, this.order, this.sortMode,
-            this.unmappedType, this.numericType);
+            this.unmappedType, this.numericType, this.optimized);
     }
 
     @Override
@@ -488,6 +516,7 @@ public static FieldSortBuilder fromXContent(XContentParser parser, String fieldN
         }, NESTED_FILTER_FIELD);
         PARSER.declareObject(FieldSortBuilder::setNestedSort, (p, c) -> NestedSortBuilder.fromXContent(p), NESTED_FIELD);
         PARSER.declareString((b, v) -> b.setNumericType(v), NUMERIC_TYPE);
+        PARSER.declareBoolean(FieldSortBuilder::optimized, OPTIMIZED_FIELD);
     }
 
     @Override

server/src/main/java/org/elasticsearch/search/sort/SortAndFormats.java

@@ -25,6 +25,7 @@ public final class SortAndFormats {
 
     public final Sort sort;
     public final DocValueFormat[] formats;
+    public final boolean[] optimized;
 
     public SortAndFormats(Sort sort, DocValueFormat[] formats) {
         if (sort.getSort().length != formats.length) {
@@ -33,6 +34,17 @@ public SortAndFormats(Sort sort, DocValueFormat[] formats) {
         }
         this.sort = sort;
         this.formats = formats;
+        this.optimized = null;
+    }
+
+    public SortAndFormats(Sort sort, DocValueFormat[] formats, boolean[] optimized) {
+        if (sort.getSort().length != formats.length) {
+            throw new IllegalArgumentException("Number of sort field mismatch: "
+                + sort.getSort().length + " != " + formats.length);
+        }
+        this.sort = sort;
+        this.formats = formats;
+        this.optimized = optimized;
     }
 
 }

server/src/main/java/org/elasticsearch/search/sort/SortBuilder.java

@@ -143,10 +143,13 @@ private static void parseCompoundSortField(XContentParser parser, List<SortBuild
     public static Optional<SortAndFormats> buildSort(List<SortBuilder<?>> sortBuilders, QueryShardContext context) throws IOException {
         List<SortField> sortFields = new ArrayList<>(sortBuilders.size());
         List<DocValueFormat> sortFormats = new ArrayList<>(sortBuilders.size());
+        boolean[] optimized = new boolean[sortBuilders.size()];
+        int i = 0;
         for (SortBuilder<?> builder : sortBuilders) {
             SortFieldAndFormat sf = builder.build(context);
             sortFields.add(sf.field);
             sortFormats.add(sf.format);
+            optimized[i++] = sf.optimized;
         }
         if (!sortFields.isEmpty()) {
             // optimize if we just sort on score non reversed, we don't really
@@ -165,7 +168,8 @@ public static Optional<SortAndFormats> buildSort(List<SortBuilder<?>> sortBuilde
             if (sort) {
                 return Optional.of(new SortAndFormats(
                         new Sort(sortFields.toArray(new SortField[sortFields.size()])),
-                        sortFormats.toArray(new DocValueFormat[sortFormats.size()])));
+                        sortFormats.toArray(new DocValueFormat[sortFormats.size()]),
+                        optimized));
             }
         }
         return Optional.empty();

server/src/main/java/org/elasticsearch/search/sort/SortFieldAndFormat.java

@@ -27,10 +27,18 @@ public final class SortFieldAndFormat {
 
     public final SortField field;
     public final DocValueFormat format;
+    public final boolean optimized;
 
     public SortFieldAndFormat(SortField field, DocValueFormat format) {
         this.field = Objects.requireNonNull(field);
         this.format = Objects.requireNonNull(format);
+        this.optimized = false;
+    }
+
+    public SortFieldAndFormat(SortField field, DocValueFormat format, boolean optimized) {
+        this.field = Objects.requireNonNull(field);
+        this.format = Objects.requireNonNull(format);
+        this.optimized = optimized;
     }
 
 }

server/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java

@@ -673,7 +673,8 @@ public void testNumericLongOrDateSortOptimization() throws Exception {
         final SortField sortFieldLong = new SortField(fieldNameLong, SortField.Type.LONG);
         sortFieldLong.setMissingValue(Long.MAX_VALUE);
         final Sort longSort = new Sort(sortFieldLong);
-        SortAndFormats sortAndFormats = new SortAndFormats(longSort, new DocValueFormat[]{DocValueFormat.RAW});
+        boolean[] optimized = new boolean[]{true};
+        SortAndFormats sortAndFormats = new SortAndFormats(longSort, new DocValueFormat[]{DocValueFormat.RAW}, optimized);
         searchContext.sort(sortAndFormats);
         searchContext.parsedQuery(new ParsedQuery(new MatchAllDocsQuery()));
         searchContext.setTask(new SearchTask(123L, "", "", "", null, Collections.emptyMap()));
@@ -685,22 +686,25 @@ public void testNumericLongOrDateSortOptimization() throws Exception {
         final SortField sortFieldDate = new SortField(fieldNameDate, SortField.Type.LONG);
         DocValueFormat dateFormat = fieldTypeDate.docValueFormat(null, null);
         final Sort longDateSort = new Sort(sortFieldLong, sortFieldDate);
-        sortAndFormats = new SortAndFormats(longDateSort, new DocValueFormat[]{DocValueFormat.RAW, dateFormat});
+        optimized = new boolean[]{true, false};
+        sortAndFormats = new SortAndFormats(longDateSort, new DocValueFormat[]{DocValueFormat.RAW, dateFormat}, optimized);
         searchContext.sort(sortAndFormats);
         QueryPhase.execute(searchContext, searcher, checkCancelled -> {});
         assertSortResults(searchContext.queryResult().topDocs().topDocs, (long) numDocs, true);
 
         // 3. Test a sort on date field
         sortFieldDate.setMissingValue(Long.MAX_VALUE);
         final Sort dateSort = new Sort(sortFieldDate);
-        sortAndFormats = new SortAndFormats(dateSort, new DocValueFormat[]{dateFormat});
+        optimized = new boolean[]{true};
+        sortAndFormats = new SortAndFormats(dateSort, new DocValueFormat[]{dateFormat}, optimized);
         searchContext.sort(sortAndFormats);
         QueryPhase.execute(searchContext, searcher, checkCancelled -> {});
         assertSortResults(searchContext.queryResult().topDocs().topDocs, (long) numDocs, false);
 
         // 4. Test a sort on date field + long field
         final Sort dateLongSort = new Sort(sortFieldDate, sortFieldLong);
-        sortAndFormats = new SortAndFormats(dateLongSort, new DocValueFormat[]{dateFormat, DocValueFormat.RAW});
+        optimized = new boolean[]{true, false};
+        sortAndFormats = new SortAndFormats(dateLongSort, new DocValueFormat[]{dateFormat, DocValueFormat.RAW}, optimized);
         searchContext.sort(sortAndFormats);
         QueryPhase.execute(searchContext, searcher, checkCancelled -> {});
         assertSortResults(searchContext.queryResult().topDocs().topDocs, (long) numDocs, true);
@@ -785,7 +789,8 @@ public void testNumericLongSortOptimizationDocsHaveTheSameValue() throws Excepti
         final SortField sortFieldLong = new SortField(fieldNameLong, SortField.Type.LONG);
         sortFieldLong.setMissingValue(Long.MAX_VALUE);
         final Sort longSort = new Sort(sortFieldLong);
-        SortAndFormats sortAndFormats = new SortAndFormats(longSort, new DocValueFormat[]{DocValueFormat.RAW});
+        boolean[] optimized = new boolean[]{true};
+        SortAndFormats sortAndFormats = new SortAndFormats(longSort, new DocValueFormat[]{DocValueFormat.RAW}, optimized);
         searchContext.sort(sortAndFormats);
         searchContext.parsedQuery(new ParsedQuery(new MatchAllDocsQuery()));
         searchContext.setTask(new SearchTask(123L, "", "", "", null, Collections.emptyMap()));