Add geo_shape mapper supporting doc-values in Spatial Plugin (#55037) (#55500)

After #53562, the `geo_shape` field mapper is registered within
a module. This opens the door for introducing a new `geo_shape`
field mapper into the Spatial Plugin that has doc-values support.

This is very much an extension of server's GeoShapeFieldMapper,
but with the addition of the doc values implementation.

Author: talevy

distribution/build.gradle

@@ -317,6 +317,12 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
           excludePlatforms = []
         }
         from(buildModules) {
+          // geo registers the geo_shape mapper that is overridden by
+          // the geo_shape mapper registered in the x-pack-spatial plugin
+          if (oss == false) {
+            exclude "**/geo/**"
+          }
+
           for (String excludePlatform : excludePlatforms) {
             exclude "**/platform/${excludePlatform}/**"
           }

modules/geo/build.gradle

@@ -21,3 +21,7 @@ esplugin {
   description 'Placeholder plugin for geospatial features in ES. only registers geo_shape field mapper for now'
   classname 'org.elasticsearch.geo.GeoPlugin'
 }
+
+artifacts {
+  restTests(new File(projectDir, "src/test/resources/rest-api-spec/test"))
+}

modules/geo/src/main/java/org/elasticsearch/geo/GeoPlugin.java

@@ -19,7 +19,6 @@
 
 package org.elasticsearch.geo;
 
-import org.elasticsearch.index.mapper.AbstractGeometryFieldMapper;
 import org.elasticsearch.index.mapper.GeoShapeFieldMapper;
 import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.plugins.MapperPlugin;
@@ -32,6 +31,6 @@ public class GeoPlugin extends Plugin implements MapperPlugin {
 
     @Override
     public Map<String, Mapper.TypeParser> getMappers() {
-        return Collections.singletonMap(GeoShapeFieldMapper.CONTENT_TYPE, new AbstractGeometryFieldMapper.TypeParser());
+        return Collections.singletonMap(GeoShapeFieldMapper.CONTENT_TYPE, new GeoShapeFieldMapper.TypeParser());
     }
 }

server/src/main/java/org/elasticsearch/common/io/stream/ByteBufferStreamInput.java

@@ -110,6 +110,14 @@ public long readLong() throws IOException {
         }
     }
 
+    public void position(int newPosition) throws IOException {
+        buffer.position(newPosition);
+    }
+
+    public int position() throws IOException {
+        return buffer.position();
+    }
+
     @Override
     public void reset() throws IOException {
         buffer.reset();

server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java

@@ -185,7 +185,9 @@ protected void setupFieldType(BuilderContext context) {
 
     protected static final String DEPRECATED_PARAMETERS_KEY = "deprecated_parameters";
 
-    public static class TypeParser implements Mapper.TypeParser {
+    public abstract static class TypeParser implements Mapper.TypeParser {
+        protected abstract Builder newBuilder(String name, Map<String, Object> params);
+
         protected boolean parseXContentParameters(String name, Map.Entry<String, Object> entry, Map<String, Object> params)
                 throws MapperParsingException {
             if (DeprecatedParameters.parse(name, entry.getKey(), entry.getValue(),
@@ -195,13 +197,6 @@ protected boolean parseXContentParameters(String name, Map.Entry<String, Object>
             return false;
         }
 
-        protected Builder newBuilder(String name, Map<String, Object> params) {
-            if (params.containsKey(DEPRECATED_PARAMETERS_KEY)) {
-                return new LegacyGeoShapeFieldMapper.Builder(name, (DeprecatedParameters)params.get(DEPRECATED_PARAMETERS_KEY));
-            }
-            return new GeoShapeFieldMapper.Builder(name);
-        }
-
         @Override
         public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
             Map<String, Object> params = new HashMap<>();

server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java

@@ -26,6 +26,8 @@
 import org.elasticsearch.geometry.Geometry;
 import org.elasticsearch.index.query.VectorGeoShapeQueryProcessor;
 
+import java.util.Map;
+
 /**
  * FieldMapper for indexing {@link LatLonShape}s.
  * <p>
@@ -76,7 +78,7 @@ protected void setupFieldType(BuilderContext context) {
         }
     }
 
-    public static final class GeoShapeFieldType extends AbstractGeometryFieldType<Geometry, Geometry> {
+    public static class GeoShapeFieldType extends AbstractGeometryFieldType<Geometry, Geometry> {
         public GeoShapeFieldType() {
             super();
         }
@@ -96,6 +98,18 @@ public String typeName() {
         }
     }
 
+    public static final class TypeParser extends AbstractGeometryFieldMapper.TypeParser {
+
+        @Override
+        protected AbstractGeometryFieldMapper.Builder newBuilder(String name, Map<String, Object> params) {
+            if (params.containsKey(DEPRECATED_PARAMETERS_KEY)) {
+                return new LegacyGeoShapeFieldMapper.Builder(name,
+                    (LegacyGeoShapeFieldMapper.DeprecatedParameters)params.get(DEPRECATED_PARAMETERS_KEY));
+            }
+            return new GeoShapeFieldMapper.Builder(name);
+        }
+    }
+
     public GeoShapeFieldMapper(String simpleName, MappedFieldType fieldType, MappedFieldType defaultFieldType,
                                Explicit<Boolean> ignoreMalformed, Explicit<Boolean> coerce,
                                Explicit<Boolean> ignoreZValue, Settings indexSettings,

server/src/main/java/org/elasticsearch/index/mapper/GeoShapeIndexer.java

@@ -49,7 +49,7 @@
 /**
  * Utility class that converts geometries into Lucene-compatible form for indexing in a geo_shape field.
  */
-public final class GeoShapeIndexer implements AbstractGeometryFieldMapper.Indexer<Geometry, Geometry> {
+public class GeoShapeIndexer implements AbstractGeometryFieldMapper.Indexer<Geometry, Geometry> {
 
     private final boolean orientation;
     private final String name;

server/src/main/java/org/elasticsearch/search/aggregations/support/MissingValues.java

@@ -57,7 +57,7 @@ public String toString() {
         };
     }
 
-    static SortedBinaryDocValues replaceMissing(final SortedBinaryDocValues values, final BytesRef missing) {
+    public static SortedBinaryDocValues replaceMissing(final SortedBinaryDocValues values, final BytesRef missing) {
         return new SortedBinaryDocValues() {
 
             private int count;

test/framework/src/main/java/org/elasticsearch/test/TestGeoShapeFieldMapperPlugin.java

@@ -18,7 +18,6 @@
  */
 package org.elasticsearch.test;
 
-import org.elasticsearch.index.mapper.AbstractGeometryFieldMapper;
 import org.elasticsearch.index.mapper.GeoShapeFieldMapper;
 import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.plugins.MapperPlugin;
@@ -40,7 +39,7 @@ public class TestGeoShapeFieldMapperPlugin extends Plugin implements MapperPlugi
     @Override
     public Map<String, Mapper.TypeParser> getMappers() {
         Map<String, Mapper.TypeParser> mappers = new LinkedHashMap<>();
-        mappers.put(GeoShapeFieldMapper.CONTENT_TYPE, new AbstractGeometryFieldMapper.TypeParser());
+        mappers.put(GeoShapeFieldMapper.CONTENT_TYPE, new GeoShapeFieldMapper.TypeParser());
         return Collections.unmodifiableMap(mappers);
     }
 }

x-pack/plugin/spatial/build.gradle

@@ -12,15 +12,24 @@ esplugin {
 dependencies {
   compileOnly project(path: xpackModule('core'), configuration: 'default')
   testCompile project(path: xpackModule('core'), configuration: 'testArtifacts')
-  testCompile project(path: ':modules:geo', configuration: 'runtime')
+  compile project(path: ':modules:geo', configuration: 'default')
+  restTestConfig project(path: ':modules:geo', configuration: 'restTests')
+}
+
+restResources {
+  restApi {
+    includeCore '_common', 'indices', 'index', 'search'
+  }
+  restTests {
+    includeCore 'geo_shape'
+  }
+}
+
+testClusters.integTest {
+  testDistribution = 'DEFAULT'
 }
 
 licenseHeaders {
   // This class was sourced from apache lucene's sandbox module tests
   excludes << 'org/apache/lucene/geo/XShapeTestUtil.java'
 }
-
-// xpack modules are installed in real clusters as the meta plugin, so
-// installing them as individual plugins for integ tests doesn't make sense,
-// so we disable integ tests
-integTest.enabled = false