Index ids in binary form. (#25352)

Indexing ids in binary form should help with indexing speed since we would
have to compare fewer bytes upon sorting, should help with memory usage of
the live version map since keys will be shorter, and might help with disk
usage depending on how efficient the terms dictionary is at compressing
terms.

Since we can only expect base64 ids in the auto-generated case, this PR tries
to use an encoding that makes the binary id equal to the base64-decoded id in
the majority of cases (253 out of 256). It also specializes numeric ids, since
this seems to be common when content that is stored in Elasticsearch comes
from another database that uses eg. auto-increment ids.

Another option could be to require base64 ids all the time. It would make things
simpler but I'm not sure users would welcome this requirement.

This PR should bring some benefits, but I expect it to be mostly useful when
coupled with something like #24615.

Closes #18154

Author: jpountz

core/src/main/java/org/elasticsearch/index/fielddata/UidIndexFieldData.java

@@ -19,17 +19,13 @@
 
 package org.elasticsearch.index.fielddata;
 
-import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.index.MultiDocValues;
-import org.apache.lucene.index.SortedSetDocValues;
 import org.apache.lucene.search.SortField;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.BytesRefBuilder;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
 import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource;
-import org.elasticsearch.index.fielddata.plain.AbstractAtomicOrdinalsFieldData;
 import org.elasticsearch.index.mapper.UidFieldMapper;
 import org.elasticsearch.search.MultiValueMode;
 
@@ -42,16 +38,14 @@
  *  already using: this is just a view.
  *  TODO: Remove fielddata access on _uid and _id, or add doc values to _id.
  */
-public final class UidIndexFieldData implements IndexOrdinalsFieldData {
+public final class UidIndexFieldData implements IndexFieldData<AtomicFieldData> {
 
     private final Index index;
-    private final String type;
     private final BytesRef prefix;
-    private final IndexOrdinalsFieldData idFieldData;
+    private final IndexFieldData<?> idFieldData;
 
-    public UidIndexFieldData(Index index, String type, IndexOrdinalsFieldData idFieldData) {
+    public UidIndexFieldData(Index index, String type, IndexFieldData<?> idFieldData) {
         this.index = index;
-        this.type = type;
         BytesRefBuilder prefix = new BytesRefBuilder();
         prefix.append(new BytesRef(type));
         prefix.append((byte) '#');
@@ -76,12 +70,12 @@ public SortField sortField(Object missingValue, MultiValueMode sortMode, Nested
     }
 
     @Override
-    public AtomicOrdinalsFieldData load(LeafReaderContext context) {
+    public AtomicFieldData load(LeafReaderContext context) {
         return new UidAtomicFieldData(prefix, idFieldData.load(context));
     }
 
     @Override
-    public AtomicOrdinalsFieldData loadDirect(LeafReaderContext context) throws Exception {
+    public AtomicFieldData loadDirect(LeafReaderContext context) throws Exception {
         return new UidAtomicFieldData(prefix, idFieldData.loadDirect(context));
     }
 
@@ -90,39 +84,19 @@ public void clear() {
         idFieldData.clear();
     }
 
-    @Override
-    public IndexOrdinalsFieldData loadGlobal(DirectoryReader indexReader) {
-        return new UidIndexFieldData(index, type, idFieldData.loadGlobal(indexReader));
-    }
-
-    @Override
-    public IndexOrdinalsFieldData localGlobalDirect(DirectoryReader indexReader) throws Exception {
-        return new UidIndexFieldData(index, type, idFieldData.localGlobalDirect(indexReader));
-    }
-
-    @Override
-    public MultiDocValues.OrdinalMap getOrdinalMap() {
-        return idFieldData.getOrdinalMap();
-    }
-
-    static final class UidAtomicFieldData implements AtomicOrdinalsFieldData {
+    static final class UidAtomicFieldData implements AtomicFieldData {
 
         private final BytesRef prefix;
-        private final AtomicOrdinalsFieldData idFieldData;
+        private final AtomicFieldData idFieldData;
 
-        UidAtomicFieldData(BytesRef prefix, AtomicOrdinalsFieldData idFieldData) {
+        UidAtomicFieldData(BytesRef prefix, AtomicFieldData idFieldData) {
             this.prefix = prefix;
             this.idFieldData = idFieldData;
         }
 
         @Override
         public ScriptDocValues<?> getScriptValues() {
-            return AbstractAtomicOrdinalsFieldData.DEFAULT_SCRIPT_FUNCTION.apply(getOrdinalsValues());
-        }
-
-        @Override
-        public SortedBinaryDocValues getBytesValues() {
-            return FieldData.toString(getOrdinalsValues());
+            return new ScriptDocValues.Strings(getBytesValues());
         }
 
         @Override
@@ -136,54 +110,30 @@ public void close() {
         }
 
         @Override
-        public SortedSetDocValues getOrdinalsValues() {
-            SortedSetDocValues idValues = idFieldData.getOrdinalsValues();
-            return new SortedSetDocValues() {
+        public SortedBinaryDocValues getBytesValues() {
+            SortedBinaryDocValues idValues = idFieldData.getBytesValues();
+            return new SortedBinaryDocValues() {
 
                 private final BytesRefBuilder scratch = new BytesRefBuilder();
 
                 @Override
-                public int nextDoc() throws IOException {
-                    return idValues.nextDoc();
+                public boolean advanceExact(int doc) throws IOException {
+                    return idValues.advanceExact(doc);
                 }
 
                 @Override
-                public int docID() {
-                    return idValues.docID();
+                public int docValueCount() {
+                    return idValues.docValueCount();
                 }
 
                 @Override
-                public long cost() {
-                    return idValues.cost();
-                }
-
-                @Override
-                public int advance(int target) throws IOException {
-                    return idValues.advance(target);
-                }
-
-                @Override
-                public boolean advanceExact(int target) throws IOException {
-                    return idValues.advanceExact(target);
-                }
-
-                @Override
-                public long nextOrd() throws IOException {
-                    return idValues.nextOrd();
-                }
-
-                @Override
-                public BytesRef lookupOrd(long ord) throws IOException {
-                    scratch.setLength(0);
-                    scratch.append(prefix);
-                    scratch.append(idValues.lookupOrd(ord));
+                public BytesRef nextValue() throws IOException {
+                    BytesRef nextID = idValues.nextValue();
+                    scratch.copyBytes(prefix);
+                    scratch.append(nextID);
                     return scratch.get();
                 }
 
-                @Override
-                public long getValueCount() {
-                    return idValues.getValueCount();
-                }
             };
         }
 

core/src/main/java/org/elasticsearch/index/fieldvisitor/FieldsVisitor.java

@@ -105,6 +105,8 @@ public void postProcess(MapperService mapperService) {
     public void binaryField(FieldInfo fieldInfo, byte[] value) throws IOException {
         if (SourceFieldMapper.NAME.equals(fieldInfo.name)) {
             source = new BytesArray(value);
+        } else if (IdFieldMapper.NAME.equals(fieldInfo.name)) {
+            id = Uid.decodeId(value);
         } else {
             addValue(fieldInfo.name, new BytesRef(value));
         }
@@ -114,10 +116,14 @@ public void binaryField(FieldInfo fieldInfo, byte[] value) throws IOException {
     public void stringField(FieldInfo fieldInfo, byte[] bytes) throws IOException {
         final String value = new String(bytes, StandardCharsets.UTF_8);
         if (UidFieldMapper.NAME.equals(fieldInfo.name)) {
+            // 5.x-only
+            // TODO: Remove when we are on 7.x
             Uid uid = Uid.createUid(value);
             type = uid.type();
             id = uid.id();
         } else if (IdFieldMapper.NAME.equals(fieldInfo.name)) {
+            // only applies to 5.x indices that have single_type = true
+            // TODO: Remove when we are on 7.x
             id = value;
         } else {
             addValue(fieldInfo.name, value);

core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java

@@ -436,7 +436,13 @@ private static ParseContext nestedContext(ParseContext context, ObjectMapper map
             if (idField != null) {
                 // We just need to store the id as indexed field, so that IndexWriter#deleteDocuments(term) can then
                 // delete it when the root document is deleted too.
-                nestedDoc.add(new Field(IdFieldMapper.NAME, idField.stringValue(), IdFieldMapper.Defaults.NESTED_FIELD_TYPE));
+                if (idField.stringValue() != null) {
+                    // backward compat with 5.x
+                    // TODO: Remove on 7.0
+                    nestedDoc.add(new Field(IdFieldMapper.NAME, idField.stringValue(), IdFieldMapper.Defaults.NESTED_FIELD_TYPE));
+                } else {
+                    nestedDoc.add(new Field(IdFieldMapper.NAME, idField.binaryValue(), IdFieldMapper.Defaults.NESTED_FIELD_TYPE));
+                }
             } else {
                 throw new IllegalStateException("The root document of a nested document should have an id field");
             }

core/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java

@@ -22,15 +22,28 @@
 import org.apache.lucene.document.Field;
 import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.search.Query;
+import org.apache.lucene.search.SortField;
 import org.apache.lucene.search.TermInSetQuery;
-import org.elasticsearch.common.Nullable;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.Version;
+import org.elasticsearch.common.lucene.BytesRefs;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.fielddata.AtomicFieldData;
 import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
+import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource;
+import org.elasticsearch.index.fielddata.IndexFieldDataCache;
+import org.elasticsearch.index.fielddata.ScriptDocValues;
+import org.elasticsearch.index.fielddata.SortedBinaryDocValues;
 import org.elasticsearch.index.fielddata.plain.PagedBytesIndexFieldData;
 import org.elasticsearch.index.query.QueryShardContext;
+import org.elasticsearch.indices.breaker.CircuitBreakerService;
+import org.elasticsearch.search.MultiValueMode;
 
 import java.io.IOException;
 import java.util.Arrays;
@@ -109,15 +122,31 @@ public boolean isSearchable() {
         }
 
         @Override
-        public Query termQuery(Object value, @Nullable QueryShardContext context) {
+        public Query termQuery(Object value, QueryShardContext context) {
             return termsQuery(Arrays.asList(value), context);
         }
 
         @Override
-        public Query termsQuery(List<?> values, @Nullable QueryShardContext context) {
+        public Query termsQuery(List<?> values, QueryShardContext context) {
             if (indexOptions() != IndexOptions.NONE) {
-                // 6.x index, _id is indexed
-                return super.termsQuery(values, context);
+                failIfNotIndexed();
+                BytesRef[] bytesRefs = new BytesRef[values.size()];
+                final boolean is5xIndex = context.indexVersionCreated().before(Version.V_6_0_0_alpha3);
+                for (int i = 0; i < bytesRefs.length; i++) {
+                    BytesRef id;
+                    if (is5xIndex) {
+                        // 5.x index with index.mapping.single_type = true
+                        id = BytesRefs.toBytesRef(values.get(i));
+                    } else {
+                        Object idObject = values.get(i);
+                        if (idObject instanceof BytesRef) {
+                            idObject = ((BytesRef) idObject).utf8ToString();
+                        }
+                        id = Uid.encodeId(idObject.toString());
+                    }
+                    bytesRefs[i] = id;
+                }
+                return new TermInSetQuery(name(), bytesRefs);
             }
             // 5.x index, _uid is indexed
             return new TermInSetQuery(UidFieldMapper.NAME, Uid.createUidsForTypesAndIds(context.queryTypes(), values));
@@ -128,13 +157,106 @@ public IndexFieldData.Builder fielddataBuilder() {
             if (indexOptions() == IndexOptions.NONE) {
                 throw new IllegalArgumentException("Fielddata access on the _uid field is disallowed");
             }
-            return new PagedBytesIndexFieldData.Builder(
+            final IndexFieldData.Builder fieldDataBuilder = new PagedBytesIndexFieldData.Builder(
                     TextFieldMapper.Defaults.FIELDDATA_MIN_FREQUENCY,
                     TextFieldMapper.Defaults.FIELDDATA_MAX_FREQUENCY,
                     TextFieldMapper.Defaults.FIELDDATA_MIN_SEGMENT_SIZE);
+            return new IndexFieldData.Builder() {
+                @Override
+                public IndexFieldData<?> build(IndexSettings indexSettings, MappedFieldType fieldType, IndexFieldDataCache cache,
+                        CircuitBreakerService breakerService, MapperService mapperService) {
+                    final IndexFieldData<?> fieldData = fieldDataBuilder.build(indexSettings, fieldType, cache, breakerService, mapperService);
+                    if (indexSettings.getIndexVersionCreated().before(Version.V_6_0_0_alpha3)) {
+                        // ids were indexed as utf-8
+                        return fieldData;
+                    }
+                    return new IndexFieldData<AtomicFieldData>() {
+
+                        @Override
+                        public Index index() {
+                            return fieldData.index();
+                        }
+
+                        @Override
+                        public String getFieldName() {
+                            return fieldData.getFieldName();
+                        }
+
+                        @Override
+                        public AtomicFieldData load(LeafReaderContext context) {
+                            return wrap(fieldData.load(context));
+                        }
+
+                        @Override
+                        public AtomicFieldData loadDirect(LeafReaderContext context) throws Exception {
+                            return wrap(fieldData.loadDirect(context));
+                        }
+
+                        @Override
+                        public SortField sortField(Object missingValue, MultiValueMode sortMode, Nested nested, boolean reverse) {
+                            XFieldComparatorSource source = new BytesRefFieldComparatorSource(this, missingValue, sortMode, nested);
+                            return new SortField(getFieldName(), source, reverse);
+                        }
+
+                        @Override
+                        public void clear() {
+                            fieldData.clear();
+                        }
+
+                    };
+                }
+            };
         }
     }
 
+    private static AtomicFieldData wrap(AtomicFieldData in) {
+        return new AtomicFieldData() {
+
+            @Override
+            public void close() {
+                in.close();
+            }
+
+            @Override
+            public long ramBytesUsed() {
+                return in.ramBytesUsed();
+            }
+
+            @Override
+            public ScriptDocValues<?> getScriptValues() {
+                return new ScriptDocValues.Strings(getBytesValues());
+            }
+
+            @Override
+            public SortedBinaryDocValues getBytesValues() {
+                SortedBinaryDocValues inValues = in.getBytesValues();
+                return new SortedBinaryDocValues() {
+
+                    @Override
+                    public BytesRef nextValue() throws IOException {
+                        BytesRef encoded = inValues.nextValue();
+                        return new BytesRef(Uid.decodeId(
+                                Arrays.copyOfRange(encoded.bytes, encoded.offset, encoded.offset + encoded.length)));
+                    }
+
+                    @Override
+                    public int docValueCount() {
+                        final int count = inValues.docValueCount();
+                        // If the count is not 1 then the impl is not correct as the binary representation
+                        // does not preserve order. But id fields only have one value per doc so we are good.
+                        assert count == 1;
+                        return inValues.docValueCount();
+                    }
+
+                    @Override
+                    public boolean advanceExact(int doc) throws IOException {
+                        return inValues.advanceExact(doc);
+                    }
+                };
+            }
+        };
+    }
+
     static MappedFieldType defaultFieldType(IndexSettings indexSettings) {
         MappedFieldType defaultFieldType = Defaults.FIELD_TYPE.clone();
         if (indexSettings.isSingleType()) {
@@ -166,8 +288,12 @@ public void postParse(ParseContext context) throws IOException {}
     @Override
     protected void parseCreateField(ParseContext context, List<IndexableField> fields) throws IOException {
         if (fieldType.indexOptions() != IndexOptions.NONE || fieldType.stored()) {
-            Field id = new Field(NAME, context.sourceToParse().id(), fieldType);
-            fields.add(id);
+            if (context.mapperService().getIndexSettings().getIndexVersionCreated().onOrAfter(Version.V_6_0_0_alpha3)) {
+                BytesRef id = Uid.encodeId(context.sourceToParse().id());
+                fields.add(new Field(NAME, id, fieldType));
+            } else {
+                fields.add(new Field(NAME, context.sourceToParse().id(), fieldType));
+            }
         }
     }