elastic · jpountz · Dec 9, 2013
diff --git a/docs/reference/mapping/types/geo-point-type.asciidoc b/docs/reference/mapping/types/geo-point-type.asciidoc
@@ -156,6 +156,47 @@ is `true`)
 |`normalize_lon` |Set to `true` to normalize longitude
 |=======================================================================
 
+[float]
+==== Field data
+
+By default, geo points use the `array` format which loads geo points into two
+parallel double arrays, making sure there is no precision loss. However, this
+can require a non-negligible amount of memory (16 bytes per document) which is
+why Elasticsearch also provides a field data implementation with lossy
+compression called `compressed`:
+
+[source,js]
+--------------------------------------------------
+{
+    "pin" : {
+        "properties" : {
+            "location" : {
+                "type" : "geo_point",
+                "fielddata" : {
+                    "format" : "compressed",
+                    "precision" : "1cm"
+                }
+            }
+        }
+    }
+}
+--------------------------------------------------
+
+This field data format comes with a `precision` option which allows to
+configure how much precision can be traded for memory. The default value is
+`1cm`. The following table presents values of the memory savings given various
+precisions:
+
+|=============================================
+| Precision | Bytes per point | Size reduction
+|       1km |               4 |            75%
+|        3m |               6 |          62.5%
+|       1cm |               8 |            50%
+|       1mm |              10 |          37.5%
+|=============================================
+
+Precision can be changed on a live index by using the update mapping API.
+
 [float]
 ==== Usage in Scripts
 

diff --git a/src/main/java/org/elasticsearch/common/unit/DistanceUnit.java b/src/main/java/org/elasticsearch/common/unit/DistanceUnit.java
@@ -248,7 +248,7 @@ public static class Distance implements Comparable<Distance> {
         public final double value;
         public final DistanceUnit unit;
 
-        private Distance(double value, DistanceUnit unit) {
+        public Distance(double value, DistanceUnit unit) {
             super();
             this.value = value;
             this.unit = unit;

diff --git a/src/main/java/org/elasticsearch/index/fielddata/IndexFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/IndexFieldData.java
@@ -175,7 +175,7 @@ protected final Object missingObject(Object missingValue, boolean reversed) {
 
     interface Builder {
 
-        IndexFieldData build(Index index, @IndexSettings Settings indexSettings, FieldMapper.Names fieldNames, FieldDataType type, IndexFieldDataCache cache);
+        IndexFieldData build(Index index, @IndexSettings Settings indexSettings, FieldMapper<?> mapper, IndexFieldDataCache cache);
     }
 
     public interface WithOrdinals<FD extends AtomicFieldData.WithOrdinals> extends IndexFieldData<FD> {

diff --git a/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java b/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java
@@ -85,6 +85,7 @@ public class IndexFieldDataService extends AbstractIndexComponent {
                 .put(Tuple.tuple("long", "array"), new PackedArrayIndexFieldData.Builder().setNumericType(IndexNumericFieldData.NumericType.LONG))
                 .put(Tuple.tuple("long", "doc_values"), new DocValuesIndexFieldData.Builder().numericType(IndexNumericFieldData.NumericType.LONG))
                 .put(Tuple.tuple("geo_point", "array"), new GeoPointDoubleArrayIndexFieldData.Builder())
+                .put(Tuple.tuple("geo_point", "compressed"), new GeoPointCompressedIndexFieldData.Builder())
                 .immutableMap();
     }
 
@@ -177,7 +178,7 @@ public <IFD extends IndexFieldData<?>> IFD getForField(FieldMapper<?> mapper) {
                         throw new ElasticSearchIllegalArgumentException("cache type not supported [" + cacheType + "] for field [" + fieldNames.fullName() + "]");
                     }
 
-                    fieldData = builder.build(index, indexSettings, fieldNames, type, cache);
+                    fieldData = builder.build(index, indexSettings, mapper, cache);
                     loadedFieldData.put(fieldNames.indexName(), fieldData);
                 }
             }

diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractGeoPointIndexFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractGeoPointIndexFieldData.java
@@ -0,0 +1,145 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata.plain;
+
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.BytesRefIterator;
+import org.apache.lucene.util.CharsRef;
+import org.apache.lucene.util.UnicodeUtil;
+import org.elasticsearch.ElasticSearchIllegalArgumentException;
+import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.geo.GeoPoint;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.fielddata.*;
+import org.elasticsearch.index.fielddata.fieldcomparator.SortMode;
+import org.elasticsearch.index.mapper.FieldMapper.Names;
+
+import java.io.IOException;
+
+abstract class AbstractGeoPointIndexFieldData extends AbstractIndexFieldData<AtomicGeoPointFieldData<ScriptDocValues>> implements IndexGeoPointFieldData<AtomicGeoPointFieldData<ScriptDocValues>> {
+
+    protected static class Empty extends AtomicGeoPointFieldData<ScriptDocValues> {
+
+        private final int numDocs;
+
+        Empty(int numDocs) {
+            this.numDocs = numDocs;
+        }
+
+        @Override
+        public boolean isMultiValued() {
+            return false;
+        }
+
+        @Override
+        public boolean isValuesOrdered() {
+            return false;
+        }
+
+        @Override
+        public long getNumberUniqueValues() {
+            return 0;
+        }
+
+        @Override
+        public long getMemorySizeInBytes() {
+            return 0;
+        }
+
+        @Override
+        public BytesValues getBytesValues(boolean needsHashes) {
+            return BytesValues.EMPTY;
+        }
+
+        @Override
+        public GeoPointValues getGeoPointValues() {
+            return GeoPointValues.EMPTY;
+        }
+
+        @Override
+        public ScriptDocValues getScriptValues() {
+            return ScriptDocValues.EMPTY;
+        }
+
+        @Override
+        public int getNumDocs() {
+            return numDocs;
+        }
+
+        @Override
+        public void close() {
+            // no-op
+        }
+    }
+
+    protected static class GeoPointEnum {
+
+        private final BytesRefIterator termsEnum;
+        private final GeoPoint next;
+        private final CharsRef spare;
+
+        protected GeoPointEnum(BytesRefIterator termsEnum) {
+            this.termsEnum = termsEnum;
+            next = new GeoPoint();
+            spare = new CharsRef();
+        }
+
+        public GeoPoint next() throws IOException {
+            final BytesRef term = termsEnum.next();
+            if (term == null) {
+                return null;
+            }
+            UnicodeUtil.UTF8toUTF16(term, spare);
+            int commaIndex = -1;
+            for (int i = 0; i < spare.length; i++) {
+                if (spare.chars[spare.offset + i] == ',') { // safes a string creation 
+                    commaIndex = i;
+                    break;
+                }
+            }
+            if (commaIndex == -1) {
+                assert false;
+                return next.reset(0, 0);
+            }
+            final double lat = Double.parseDouble(new String(spare.chars, spare.offset, (commaIndex - spare.offset)));
+            final double lon = Double.parseDouble(new String(spare.chars, (spare.offset + (commaIndex + 1)), spare.length - ((commaIndex + 1) - spare.offset)));
+            return next.reset(lat, lon);
+        }
+
+    }
+
+    public AbstractGeoPointIndexFieldData(Index index, Settings indexSettings, Names fieldNames, FieldDataType fieldDataType, IndexFieldDataCache cache) {
+        super(index, indexSettings, fieldNames, fieldDataType, cache);
+    }
+
+    @Override
+    public boolean valuesOrdered() {
+        // because we might have single values? we can dynamically update a flag to reflect that
+        // based on the atomic field data loaded
+        return false;
+    }
+
+    @Override
+    public final XFieldComparatorSource comparatorSource(@Nullable Object missingValue, SortMode sortMode) {
+        throw new ElasticSearchIllegalArgumentException("can't sort on geo_point field without using specific sorting feature, like geo_distance");
+    }
+
+}
diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/DocValuesIndexFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/DocValuesIndexFieldData.java
@@ -24,10 +24,10 @@
 import org.elasticsearch.ElasticSearchIllegalArgumentException;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.Index;
-import org.elasticsearch.index.fielddata.FieldDataType;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.IndexFieldDataCache;
 import org.elasticsearch.index.fielddata.IndexNumericFieldData.NumericType;
+import org.elasticsearch.index.mapper.FieldMapper;
 import org.elasticsearch.index.mapper.FieldMapper.Names;
 import org.elasticsearch.index.mapper.internal.IdFieldMapper;
 import org.elasticsearch.index.mapper.internal.TimestampFieldMapper;
@@ -93,8 +93,9 @@ public Builder numericType(NumericType type) {
         }
 
         @Override
-        public IndexFieldData<?> build(Index index, Settings indexSettings, Names fieldNames, FieldDataType type, IndexFieldDataCache cache) {
-            final Settings fdSettings = type.getSettings();
+        public IndexFieldData<?> build(Index index, Settings indexSettings, FieldMapper<?> mapper, IndexFieldDataCache cache) {
+            final FieldMapper.Names fieldNames = mapper.names();
+            final Settings fdSettings = mapper.fieldDataType().getSettings();
             final Map<String, Settings> filter = fdSettings.getGroups("filter");
             if (filter != null && !filter.isEmpty()) {
                 throw new ElasticSearchIllegalArgumentException("Doc values field data doesn't support filters [" + fieldNames.name() + "]");

diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/DoubleArrayIndexFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/DoubleArrayIndexFieldData.java
@@ -43,8 +43,8 @@ public class DoubleArrayIndexFieldData extends AbstractIndexFieldData<DoubleArra
     public static class Builder implements IndexFieldData.Builder {
 
         @Override
-        public IndexFieldData<?> build(Index index, @IndexSettings Settings indexSettings, FieldMapper.Names fieldNames, FieldDataType type, IndexFieldDataCache cache) {
-            return new DoubleArrayIndexFieldData(index, indexSettings, fieldNames, type, cache);
+        public IndexFieldData<?> build(Index index, @IndexSettings Settings indexSettings, FieldMapper<?> mapper, IndexFieldDataCache cache) {
+            return new DoubleArrayIndexFieldData(index, indexSettings, mapper.names(), mapper.fieldDataType(), cache);
         }
     }
 

diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/FSTBytesIndexFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/FSTBytesIndexFieldData.java
@@ -43,8 +43,8 @@ public class FSTBytesIndexFieldData extends AbstractBytesIndexFieldData<FSTBytes
     public static class Builder implements IndexFieldData.Builder {
 
         @Override
-        public IndexFieldData<FSTBytesAtomicFieldData> build(Index index, @IndexSettings Settings indexSettings, FieldMapper.Names fieldNames, FieldDataType type, IndexFieldDataCache cache) {
-            return new FSTBytesIndexFieldData(index, indexSettings, fieldNames, type, cache);
+        public IndexFieldData<FSTBytesAtomicFieldData> build(Index index, @IndexSettings Settings indexSettings, FieldMapper<?> mapper, IndexFieldDataCache cache) {
+            return new FSTBytesIndexFieldData(index, indexSettings, mapper.names(), mapper.fieldDataType(), cache);
         }
     }
 

diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/FloatArrayIndexFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/FloatArrayIndexFieldData.java
@@ -43,8 +43,8 @@ public class FloatArrayIndexFieldData extends AbstractIndexFieldData<FloatArrayA
     public static class Builder implements IndexFieldData.Builder {
 
         @Override
-        public IndexFieldData<?> build(Index index, @IndexSettings Settings indexSettings, FieldMapper.Names fieldNames, FieldDataType type, IndexFieldDataCache cache) {
-            return new FloatArrayIndexFieldData(index, indexSettings, fieldNames, type, cache);
+        public IndexFieldData<?> build(Index index, @IndexSettings Settings indexSettings, FieldMapper<?> mapper, IndexFieldDataCache cache) {
+            return new FloatArrayIndexFieldData(index, indexSettings, mapper.names(), mapper.fieldDataType(), cache);
         }
     }