Add support for ignore_unmapped to geo sort (#31153)

Adds support for `ignore_unmapped` parameter in geo distance sorting,
which is functionally equivalent to specifying an `unmapped_type` in
the field sort.

Closes #28152

Author: imotov

docs/reference/search/request/sort.asciidoc

@@ -288,8 +288,9 @@ GET /_search
                 "pin.location" : [-70, 40],
                 "order" : "asc",
                 "unit" : "km",
-		"mode" : "min",
-		"distance_type" : "arc"
+                "mode" : "min",
+                "distance_type" : "arc",
+                "ignore_unmapped": true
             }
         }
     ],
@@ -317,6 +318,12 @@ GET /_search
 
     The unit to use when computing sort values. The default is `m` (meters).
 
+
+`ignore_unmapped`::
+
+    Indicates if the unmapped field should be treated as a missing value. Setting it to `true` is equivalent to specifying
+    an `unmapped_type` in the field sort. The default is `false` (unmapped field are causing the search to fail).
+
 NOTE: geo distance sorting does not support configurable missing values: the
 distance will always be considered equal to +Infinity+ when a document does not
 have values for the field that is used for distance computation.

server/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java

@@ -81,6 +81,7 @@ public class GeoDistanceSortBuilder extends SortBuilder<GeoDistanceSortBuilder>
     private static final ParseField DISTANCE_TYPE_FIELD = new ParseField("distance_type");
     private static final ParseField VALIDATION_METHOD_FIELD = new ParseField("validation_method");
     private static final ParseField SORTMODE_FIELD = new ParseField("mode", "sort_mode");
+    private static final ParseField IGNORE_UNMAPPED = new ParseField("ignore_unmapped");
 
     private final String fieldName;
     private final List<GeoPoint> points = new ArrayList<>();
@@ -97,6 +98,8 @@ public class GeoDistanceSortBuilder extends SortBuilder<GeoDistanceSortBuilder>
 
     private GeoValidationMethod validation = DEFAULT_VALIDATION;
 
+    private boolean ignoreUnmapped = false;
+
     /**
      * Constructs a new distance based sort on a geo point like field.
      *
@@ -152,6 +155,7 @@ public GeoDistanceSortBuilder(String fieldName, String ... geohashes) {
         this.nestedPath = original.nestedPath;
         this.validation = original.validation;
         this.nestedSort = original.nestedSort;
+        this.ignoreUnmapped = original.ignoreUnmapped;
     }
 
     /**
@@ -171,6 +175,9 @@ public GeoDistanceSortBuilder(StreamInput in) throws IOException {
             nestedSort = in.readOptionalWriteable(NestedSortBuilder::new);
         }
         validation = GeoValidationMethod.readFromStream(in);
+        if (in.getVersion().onOrAfter(Version.V_6_4_0)) {
+            ignoreUnmapped = in.readBoolean();
+        }
     }
 
     @Override
@@ -187,6 +194,9 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeOptionalWriteable(nestedSort);
         }
         validation.writeTo(out);
+        if (out.getVersion().onOrAfter(Version.V_6_4_0)) {
+            out.writeBoolean(ignoreUnmapped);
+        }
     }
 
     /**
@@ -374,6 +384,18 @@ public GeoDistanceSortBuilder setNestedSort(final NestedSortBuilder nestedSort)
         return this;
     }
 
+    /**
+     * Returns true if unmapped geo fields should be treated as located at an infinite distance
+     */
+    public boolean ignoreUnmapped() {
+        return ignoreUnmapped;
+    }
+
+    public GeoDistanceSortBuilder ignoreUnmapped(boolean ignoreUnmapped) {
+        this.ignoreUnmapped = ignoreUnmapped;
+        return this;
+    }
+
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         builder.startObject();
@@ -403,6 +425,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             builder.field(NESTED_FIELD.getPreferredName(), nestedSort);
         }
         builder.field(VALIDATION_METHOD_FIELD.getPreferredName(), validation);
+        builder.field(IGNORE_UNMAPPED.getPreferredName(), ignoreUnmapped);
 
         builder.endObject();
         builder.endObject();
@@ -434,14 +457,15 @@ public boolean equals(Object object) {
                 Objects.equals(nestedFilter, other.nestedFilter) &&
                 Objects.equals(nestedPath, other.nestedPath) &&
                 Objects.equals(validation, other.validation) &&
-                Objects.equals(nestedSort, other.nestedSort);
+                Objects.equals(nestedSort, other.nestedSort) &&
+                ignoreUnmapped == other.ignoreUnmapped;
     }
 
     @Override
     public int hashCode() {
         return Objects.hash(this.fieldName, this.points, this.geoDistance,
                 this.unit, this.sortMode, this.order, this.nestedFilter,
-                this.nestedPath, this.validation, this.nestedSort);
+                this.nestedPath, this.validation, this.nestedSort, this.ignoreUnmapped);
     }
 
     /**
@@ -465,6 +489,7 @@ public static GeoDistanceSortBuilder fromXContent(XContentParser parser, String
         String nestedPath = null;
         NestedSortBuilder nestedSort = null;
         GeoValidationMethod validation = null;
+        boolean ignoreUnmapped = false;
 
         XContentParser.Token token;
         String currentName = parser.currentName();
@@ -509,6 +534,8 @@ public static GeoDistanceSortBuilder fromXContent(XContentParser parser, String
                 } else if (NESTED_PATH_FIELD.match(currentName, parser.getDeprecationHandler())) {
                     DEPRECATION_LOGGER.deprecated("[nested_path] has been deprecated in favour of the [nested] parameter");
                     nestedPath = parser.text();
+                } else if (IGNORE_UNMAPPED.match(currentName, parser.getDeprecationHandler())) {
+                    ignoreUnmapped = parser.booleanValue();
                 } else if (token == Token.VALUE_STRING){
                     if (fieldName != null && fieldName.equals(currentName) == false) {
                         throw new ParsingException(
@@ -554,6 +581,7 @@ public static GeoDistanceSortBuilder fromXContent(XContentParser parser, String
         if (validation != null) {
             result.validation(validation);
         }
+        result.ignoreUnmapped(ignoreUnmapped);
         return result;
     }
 
@@ -596,8 +624,11 @@ public SortFieldAndFormat build(QueryShardContext context) throws IOException {
 
         MappedFieldType fieldType = context.fieldMapper(fieldName);
         if (fieldType == null) {
-            throw new IllegalArgumentException("failed to find mapper for [" + fieldName
-                + "] for geo distance based sort");
+            if (ignoreUnmapped) {
+                fieldType = context.getMapperService().unmappedFieldType("geo_point");
+            } else {
+                throw new IllegalArgumentException("failed to find mapper for [" + fieldName + "] for geo distance based sort");
+            }
         }
         final IndexGeoPointFieldData geoIndexFieldData = context.getForField(fieldType);
 

server/src/test/java/org/elasticsearch/search/sort/GeoDistanceIT.java

@@ -49,6 +49,7 @@
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.hasId;
 import static org.hamcrest.Matchers.closeTo;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
 
 
 public class GeoDistanceIT extends ESIntegTestCase {
@@ -406,4 +407,57 @@ public void testGeoDistanceFilter() throws IOException {
         assertHitCount(result, 1);
     }
 
+    public void testDistanceSortingWithUnmappedField() throws Exception {
+        XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().startObject().startObject("type1").startObject("properties")
+            .startObject("locations").field("type", "geo_point");
+        xContentBuilder.endObject().endObject().endObject().endObject();
+        assertAcked(prepareCreate("test1").addMapping("type1", xContentBuilder));
+        assertAcked(prepareCreate("test2"));
+        ensureGreen();
+
+        client().prepareIndex("test1", "type1", "1")
+            .setSource(jsonBuilder().startObject().array("names", "Times Square", "Tribeca").startArray("locations")
+                // to NY: 5.286 km
+                .startObject().field("lat", 40.759011).field("lon", -73.9844722).endObject()
+                // to NY: 0.4621 km
+                .startObject().field("lat", 40.718266).field("lon", -74.007819).endObject().endArray().endObject())
+            .execute().actionGet();
+
+        client().prepareIndex("test2", "type1", "2")
+            .setSource(jsonBuilder().startObject().array("names", "Wall Street", "Soho").endObject())
+            .execute().actionGet();
+
+        refresh();
+
+        // Order: Asc
+        SearchResponse searchResponse = client().prepareSearch("test1", "test2").setQuery(matchAllQuery())
+            .addSort(SortBuilders.geoDistanceSort("locations", 40.7143528, -74.0059731).ignoreUnmapped(true).order(SortOrder.ASC)).execute()
+            .actionGet();
+
+        assertHitCount(searchResponse, 2);
+        assertOrderedSearchHits(searchResponse, "1", "2");
+        assertThat(((Number) searchResponse.getHits().getAt(0).getSortValues()[0]).doubleValue(), closeTo(462.1d, 10d));
+        assertThat(((Number) searchResponse.getHits().getAt(1).getSortValues()[0]).doubleValue(), equalTo(Double.POSITIVE_INFINITY));
+
+        // Order: Desc
+        searchResponse = client().prepareSearch("test1", "test2").setQuery(matchAllQuery())
+            .addSort(
+                SortBuilders.geoDistanceSort("locations", 40.7143528, -74.0059731).ignoreUnmapped(true).order(SortOrder.DESC)
+            ).execute()
+            .actionGet();
+
+        // Doc with missing geo point is first, is consistent with 0.20.x
+        assertHitCount(searchResponse, 2);
+        assertOrderedSearchHits(searchResponse, "2", "1");
+        assertThat(((Number) searchResponse.getHits().getAt(0).getSortValues()[0]).doubleValue(), equalTo(Double.POSITIVE_INFINITY));
+        assertThat(((Number) searchResponse.getHits().getAt(1).getSortValues()[0]).doubleValue(), closeTo(5286d, 10d));
+
+        // Make sure that by default the unmapped fields continue to fail
+        searchResponse = client().prepareSearch("test1", "test2").setQuery(matchAllQuery())
+            .addSort( SortBuilders.geoDistanceSort("locations", 40.7143528, -74.0059731).order(SortOrder.DESC)).execute()
+            .actionGet();
+        assertThat(searchResponse.getFailedShards(), greaterThan(0));
+        assertHitCount(searchResponse, 1);
+    }
+
 }

server/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderTests.java

@@ -47,6 +47,7 @@
 import org.elasticsearch.index.query.RangeQueryBuilder;
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.MultiValueMode;
+import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.test.geo.RandomGeoGenerator;
 
 import java.io.IOException;
@@ -121,6 +122,9 @@ public static GeoDistanceSortBuilder randomGeoDistanceSortBuilder() {
                 }
             }
         }
+        if (randomBoolean()) {
+            result.ignoreUnmapped(result.ignoreUnmapped() == false);
+        }
         return result;
     }
 
@@ -154,7 +158,7 @@ private static GeoDistance geoDistance(GeoDistance original) {
     @Override
     protected GeoDistanceSortBuilder mutate(GeoDistanceSortBuilder original) throws IOException {
         GeoDistanceSortBuilder result = new GeoDistanceSortBuilder(original);
-        int parameter = randomIntBetween(0, 7);
+        int parameter = randomIntBetween(0, 8);
         switch (parameter) {
         case 0:
             while (Arrays.deepEquals(original.points(), result.points())) {
@@ -194,6 +198,9 @@ protected GeoDistanceSortBuilder mutate(GeoDistanceSortBuilder original) throws
         case 7:
             result.validation(randomValueOtherThan(result.validation(), () -> randomFrom(GeoValidationMethod.values())));
             break;
+        case 8:
+            result.ignoreUnmapped(result.ignoreUnmapped() == false);
+            break;
         }
         return result;
     }