Skip to content

Commit d859600

Browse files
jpountzjpountz
committed
Add a doc value format to binary fields. (#30860)
This will be necessary for the `docvalue_fields` option to work correctly once we use the field's doc-value format to format doc-value fields. Binary values are formatted as base64-encoded strings.
1 parent 97b3b2d commit d859600

File tree

5 files changed

+77
-3
lines changed

5 files changed

+77
-3
lines changed

‎server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,8 @@
4040
import org.elasticsearch.index.fielddata.plain.BytesBinaryDVIndexFieldData;
4141
import org.elasticsearch.index.query.QueryShardContext;
4242
import org.elasticsearch.index.query.QueryShardException;
43+
import org.elasticsearch.search.DocValueFormat;
44+
import org.joda.time.DateTimeZone;
4345

4446
import java.io.IOException;
4547
import java.util.Base64;
@@ -104,6 +106,10 @@ public String typeName() {
104106
return CONTENT_TYPE;
105107
}
106108

109+
@Override
110+
public DocValueFormat docValueFormat(String format, DateTimeZone timeZone) {
111+
return DocValueFormat.BINARY;
112+
}
107113

108114
@Override
109115
public BytesReference valueForDisplay(Object value) {

‎server/src/main/java/org/elasticsearch/search/DocValueFormat.java‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@
3939
import java.text.NumberFormat;
4040
import java.text.ParseException;
4141
import java.util.Arrays;
42+
import java.util.Base64;
4243
import java.util.Locale;
4344
import java.util.Objects;
4445
import java.util.function.LongSupplier;
@@ -121,6 +122,50 @@ public BytesRef parseBytesRef(String value) {
121122
}
122123
};
123124

125+
DocValueFormat BINARY = new DocValueFormat() {
126+
127+
@Override
128+
public String getWriteableName() {
129+
return "binary";
130+
}
131+
132+
@Override
133+
public void writeTo(StreamOutput out) throws IOException {
134+
}
135+
136+
@Override
137+
public Object format(long value) {
138+
throw new UnsupportedOperationException();
139+
}
140+
141+
@Override
142+
public Object format(double value) {
143+
throw new UnsupportedOperationException();
144+
}
145+
146+
@Override
147+
public String format(BytesRef value) {
148+
return Base64.getEncoder()
149+
.withoutPadding()
150+
.encodeToString(Arrays.copyOfRange(value.bytes, value.offset, value.offset + value.length));
151+
}
152+
153+
@Override
154+
public long parseLong(String value, boolean roundUp, LongSupplier now) {
155+
throw new UnsupportedOperationException();
156+
}
157+
158+
@Override
159+
public double parseDouble(String value, boolean roundUp, LongSupplier now) {
160+
throw new UnsupportedOperationException();
161+
}
162+
163+
@Override
164+
public BytesRef parseBytesRef(String value) {
165+
return new BytesRef(Base64.getDecoder().decode(value));
166+
}
167+
};
168+
124169
final class DateTime implements DocValueFormat {
125170

126171
public static final String NAME = "date_time";

‎server/src/main/java/org/elasticsearch/search/SearchModule.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -646,6 +646,7 @@ private void registerValueFormats() {
646646
registerValueFormat(DocValueFormat.GEOHASH.getWriteableName(), in -> DocValueFormat.GEOHASH);
647647
registerValueFormat(DocValueFormat.IP.getWriteableName(), in -> DocValueFormat.IP);
648648
registerValueFormat(DocValueFormat.RAW.getWriteableName(), in -> DocValueFormat.RAW);
649+
registerValueFormat(DocValueFormat.BINARY.getWriteableName(), in -> DocValueFormat.BINARY);
649650
}
650651

651652
/**

‎server/src/test/java/org/elasticsearch/search/DocValueFormatTests.java‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ public void testSerialization() throws Exception {
4444
entries.add(new Entry(DocValueFormat.class, DocValueFormat.GEOHASH.getWriteableName(), in -> DocValueFormat.GEOHASH));
4545
entries.add(new Entry(DocValueFormat.class, DocValueFormat.IP.getWriteableName(), in -> DocValueFormat.IP));
4646
entries.add(new Entry(DocValueFormat.class, DocValueFormat.RAW.getWriteableName(), in -> DocValueFormat.RAW));
47+
entries.add(new Entry(DocValueFormat.class, DocValueFormat.BINARY.getWriteableName(), in -> DocValueFormat.BINARY));
4748
NamedWriteableRegistry registry = new NamedWriteableRegistry(entries);
4849

4950
BytesStreamOutput out = new BytesStreamOutput();
@@ -82,6 +83,11 @@ public void testSerialization() throws Exception {
8283
out.writeNamedWriteable(DocValueFormat.RAW);
8384
in = new NamedWriteableAwareStreamInput(out.bytes().streamInput(), registry);
8485
assertSame(DocValueFormat.RAW, in.readNamedWriteable(DocValueFormat.class));
86+
87+
out = new BytesStreamOutput();
88+
out.writeNamedWriteable(DocValueFormat.BINARY);
89+
in = new NamedWriteableAwareStreamInput(out.bytes().streamInput(), registry);
90+
assertSame(DocValueFormat.BINARY, in.readNamedWriteable(DocValueFormat.class));
8591
}
8692

8793
public void testRawFormat() {
@@ -96,6 +102,14 @@ public void testRawFormat() {
96102
assertEquals("abc", DocValueFormat.RAW.format(new BytesRef("abc")));
97103
}
98104

105+
public void testBinaryFormat() {
106+
assertEquals("", DocValueFormat.BINARY.format(new BytesRef()));
107+
assertEquals("KmQ", DocValueFormat.BINARY.format(new BytesRef(new byte[] {42, 100})));
108+
109+
assertEquals(new BytesRef(), DocValueFormat.BINARY.parseBytesRef(""));
110+
assertEquals(new BytesRef(new byte[] {42, 100}), DocValueFormat.BINARY.parseBytesRef("KmQ"));
111+
}
112+
99113
public void testBooleanFormat() {
100114
assertEquals(false, DocValueFormat.BOOLEAN.format(0));
101115
assertEquals(true, DocValueFormat.BOOLEAN.format(1));

‎server/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919

2020
package org.elasticsearch.search.fields;
2121

22+
import org.apache.lucene.util.BytesRef;
2223
import org.elasticsearch.action.index.IndexRequestBuilder;
2324
import org.elasticsearch.action.search.SearchRequestBuilder;
2425
import org.elasticsearch.action.search.SearchResponse;
@@ -719,7 +720,7 @@ public void testSingleValueFieldDatatField() throws ExecutionException, Interrup
719720
assertThat(fields.get("test_field").getValue(), equalTo("foobar"));
720721
}
721722

722-
public void testFieldsPulledFromFieldData() throws Exception {
723+
public void testDocValueFields() throws Exception {
723724
createIndex("test");
724725

725726
String mapping = Strings
@@ -763,6 +764,7 @@ public void testFieldsPulledFromFieldData() throws Exception {
763764
.endObject()
764765
.startObject("binary_field")
765766
.field("type", "binary")
767+
.field("doc_values", true) // off by default on binary fields
766768
.endObject()
767769
.startObject("ip_field")
768770
.field("type", "ip")
@@ -785,6 +787,7 @@ public void testFieldsPulledFromFieldData() throws Exception {
785787
.field("double_field", 6.0d)
786788
.field("date_field", Joda.forPattern("dateOptionalTime").printer().print(date))
787789
.field("boolean_field", true)
790+
.field("binary_field", new byte[] {42, 100})
788791
.field("ip_field", "::1")
789792
.endObject()).execute().actionGet();
790793

@@ -801,6 +804,7 @@ public void testFieldsPulledFromFieldData() throws Exception {
801804
.addDocValueField("double_field")
802805
.addDocValueField("date_field")
803806
.addDocValueField("boolean_field")
807+
.addDocValueField("binary_field")
804808
.addDocValueField("ip_field");
805809
SearchResponse searchResponse = builder.execute().actionGet();
806810

@@ -809,7 +813,7 @@ public void testFieldsPulledFromFieldData() throws Exception {
809813
Set<String> fields = new HashSet<>(searchResponse.getHits().getAt(0).getFields().keySet());
810814
assertThat(fields, equalTo(newHashSet("byte_field", "short_field", "integer_field", "long_field",
811815
"float_field", "double_field", "date_field", "boolean_field", "text_field", "keyword_field",
812-
"ip_field")));
816+
"binary_field", "ip_field")));
813817

814818
assertThat(searchResponse.getHits().getAt(0).getFields().get("byte_field").getValue().toString(), equalTo("1"));
815819
assertThat(searchResponse.getHits().getAt(0).getFields().get("short_field").getValue().toString(), equalTo("2"));
@@ -821,6 +825,8 @@ public void testFieldsPulledFromFieldData() throws Exception {
821825
assertThat(searchResponse.getHits().getAt(0).getFields().get("boolean_field").getValue(), equalTo((Object) true));
822826
assertThat(searchResponse.getHits().getAt(0).getFields().get("text_field").getValue(), equalTo("foo"));
823827
assertThat(searchResponse.getHits().getAt(0).getFields().get("keyword_field").getValue(), equalTo("foo"));
828+
assertThat(searchResponse.getHits().getAt(0).getFields().get("binary_field").getValue(),
829+
equalTo(new BytesRef(new byte[] {42, 100})));
824830
assertThat(searchResponse.getHits().getAt(0).getFields().get("ip_field").getValue(), equalTo("::1"));
825831

826832
builder = client().prepareSearch().setQuery(matchAllQuery())
@@ -834,6 +840,7 @@ public void testFieldsPulledFromFieldData() throws Exception {
834840
.addDocValueField("double_field", "use_field_mapping")
835841
.addDocValueField("date_field", "use_field_mapping")
836842
.addDocValueField("boolean_field", "use_field_mapping")
843+
.addDocValueField("binary_field", "use_field_mapping")
837844
.addDocValueField("ip_field", "use_field_mapping");
838845
searchResponse = builder.execute().actionGet();
839846

@@ -842,7 +849,7 @@ public void testFieldsPulledFromFieldData() throws Exception {
842849
fields = new HashSet<>(searchResponse.getHits().getAt(0).getFields().keySet());
843850
assertThat(fields, equalTo(newHashSet("byte_field", "short_field", "integer_field", "long_field",
844851
"float_field", "double_field", "date_field", "boolean_field", "text_field", "keyword_field",
845-
"ip_field")));
852+
"binary_field", "ip_field")));
846853

847854
assertThat(searchResponse.getHits().getAt(0).getFields().get("byte_field").getValue().toString(), equalTo("1"));
848855
assertThat(searchResponse.getHits().getAt(0).getFields().get("short_field").getValue().toString(), equalTo("2"));
@@ -855,6 +862,7 @@ public void testFieldsPulledFromFieldData() throws Exception {
855862
assertThat(searchResponse.getHits().getAt(0).getFields().get("boolean_field").getValue(), equalTo((Object) true));
856863
assertThat(searchResponse.getHits().getAt(0).getFields().get("text_field").getValue(), equalTo("foo"));
857864
assertThat(searchResponse.getHits().getAt(0).getFields().get("keyword_field").getValue(), equalTo("foo"));
865+
assertThat(searchResponse.getHits().getAt(0).getFields().get("binary_field").getValue(), equalTo("KmQ"));
858866
assertThat(searchResponse.getHits().getAt(0).getFields().get("ip_field").getValue(), equalTo("::1"));
859867

860868
builder = client().prepareSearch().setQuery(matchAllQuery())

0 commit comments

Comments
 (0)