diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java index 1838b60050e4c..e19bdb6708370 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java @@ -40,6 +40,8 @@ import org.elasticsearch.index.fielddata.plain.BytesBinaryDVIndexFieldData; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.query.QueryShardException; +import org.elasticsearch.search.DocValueFormat; +import org.joda.time.DateTimeZone; import java.io.IOException; import java.util.Base64; @@ -104,6 +106,10 @@ public String typeName() { return CONTENT_TYPE; } + @Override + public DocValueFormat docValueFormat(String format, DateTimeZone timeZone) { + return DocValueFormat.BINARY; + } @Override public BytesReference valueForDisplay(Object value) { diff --git a/server/src/main/java/org/elasticsearch/search/DocValueFormat.java b/server/src/main/java/org/elasticsearch/search/DocValueFormat.java index 8677370fc9927..242e088747341 100644 --- a/server/src/main/java/org/elasticsearch/search/DocValueFormat.java +++ b/server/src/main/java/org/elasticsearch/search/DocValueFormat.java @@ -39,6 +39,7 @@ import java.text.NumberFormat; import java.text.ParseException; import java.util.Arrays; +import java.util.Base64; import java.util.Locale; import java.util.Objects; import java.util.function.LongSupplier; @@ -121,6 +122,50 @@ public BytesRef parseBytesRef(String value) { } }; + DocValueFormat BINARY = new DocValueFormat() { + + @Override + public String getWriteableName() { + return "binary"; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + } + + @Override + public Object format(long value) { + throw new UnsupportedOperationException(); + } + + @Override + public Object format(double value) { + throw new UnsupportedOperationException(); + } + + @Override + public String format(BytesRef value) { + return Base64.getEncoder() + .withoutPadding() + .encodeToString(Arrays.copyOfRange(value.bytes, value.offset, value.offset + value.length)); + } + + @Override + public long parseLong(String value, boolean roundUp, LongSupplier now) { + throw new UnsupportedOperationException(); + } + + @Override + public double parseDouble(String value, boolean roundUp, LongSupplier now) { + throw new UnsupportedOperationException(); + } + + @Override + public BytesRef parseBytesRef(String value) { + return new BytesRef(Base64.getDecoder().decode(value)); + } + }; + final class DateTime implements DocValueFormat { public static final String NAME = "date_time"; diff --git a/server/src/main/java/org/elasticsearch/search/SearchModule.java b/server/src/main/java/org/elasticsearch/search/SearchModule.java index 66ea407f42afd..869dfe995ed4e 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchModule.java +++ b/server/src/main/java/org/elasticsearch/search/SearchModule.java @@ -645,6 +645,7 @@ private void registerValueFormats() { registerValueFormat(DocValueFormat.GEOHASH.getWriteableName(), in -> DocValueFormat.GEOHASH); registerValueFormat(DocValueFormat.IP.getWriteableName(), in -> DocValueFormat.IP); registerValueFormat(DocValueFormat.RAW.getWriteableName(), in -> DocValueFormat.RAW); + registerValueFormat(DocValueFormat.BINARY.getWriteableName(), in -> DocValueFormat.BINARY); } /** diff --git a/server/src/test/java/org/elasticsearch/search/DocValueFormatTests.java b/server/src/test/java/org/elasticsearch/search/DocValueFormatTests.java index e5cfbf98b3db9..0190627947448 100644 --- a/server/src/test/java/org/elasticsearch/search/DocValueFormatTests.java +++ b/server/src/test/java/org/elasticsearch/search/DocValueFormatTests.java @@ -44,6 +44,7 @@ public void testSerialization() throws Exception { entries.add(new Entry(DocValueFormat.class, DocValueFormat.GEOHASH.getWriteableName(), in -> DocValueFormat.GEOHASH)); entries.add(new Entry(DocValueFormat.class, DocValueFormat.IP.getWriteableName(), in -> DocValueFormat.IP)); entries.add(new Entry(DocValueFormat.class, DocValueFormat.RAW.getWriteableName(), in -> DocValueFormat.RAW)); + entries.add(new Entry(DocValueFormat.class, DocValueFormat.BINARY.getWriteableName(), in -> DocValueFormat.BINARY)); NamedWriteableRegistry registry = new NamedWriteableRegistry(entries); BytesStreamOutput out = new BytesStreamOutput(); @@ -82,6 +83,11 @@ public void testSerialization() throws Exception { out.writeNamedWriteable(DocValueFormat.RAW); in = new NamedWriteableAwareStreamInput(out.bytes().streamInput(), registry); assertSame(DocValueFormat.RAW, in.readNamedWriteable(DocValueFormat.class)); + + out = new BytesStreamOutput(); + out.writeNamedWriteable(DocValueFormat.BINARY); + in = new NamedWriteableAwareStreamInput(out.bytes().streamInput(), registry); + assertSame(DocValueFormat.BINARY, in.readNamedWriteable(DocValueFormat.class)); } public void testRawFormat() { @@ -96,6 +102,14 @@ public void testRawFormat() { assertEquals("abc", DocValueFormat.RAW.format(new BytesRef("abc"))); } + public void testBinaryFormat() { + assertEquals("", DocValueFormat.BINARY.format(new BytesRef())); + assertEquals("KmQ", DocValueFormat.BINARY.format(new BytesRef(new byte[] {42, 100}))); + + assertEquals(new BytesRef(), DocValueFormat.BINARY.parseBytesRef("")); + assertEquals(new BytesRef(new byte[] {42, 100}), DocValueFormat.BINARY.parseBytesRef("KmQ")); + } + public void testBooleanFormat() { assertEquals(false, DocValueFormat.BOOLEAN.format(0)); assertEquals(true, DocValueFormat.BOOLEAN.format(1)); diff --git a/server/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java b/server/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java index a8a2669ef9b4a..ab5387b6e3f48 100644 --- a/server/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java +++ b/server/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java @@ -19,6 +19,7 @@ package org.elasticsearch.search.fields; +import org.apache.lucene.util.BytesRef; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; @@ -700,7 +701,7 @@ public void testSingleValueFieldDatatField() throws ExecutionException, Interrup assertThat(fields.get("test_field").getValue(), equalTo("foobar")); } - public void testFieldsPulledFromFieldData() throws Exception { + public void testDocValueFields() throws Exception { createIndex("test"); String mapping = Strings @@ -744,6 +745,7 @@ public void testFieldsPulledFromFieldData() throws Exception { .endObject() .startObject("binary_field") .field("type", "binary") + .field("doc_values", true) // off by default on binary fields .endObject() .startObject("ip_field") .field("type", "ip") @@ -766,6 +768,7 @@ public void testFieldsPulledFromFieldData() throws Exception { .field("double_field", 6.0d) .field("date_field", Joda.forPattern("dateOptionalTime").printer().print(date)) .field("boolean_field", true) + .field("binary_field", new byte[] {42, 100}) .field("ip_field", "::1") .endObject()).execute().actionGet(); @@ -782,6 +785,7 @@ public void testFieldsPulledFromFieldData() throws Exception { .addDocValueField("double_field") .addDocValueField("date_field") .addDocValueField("boolean_field") + .addDocValueField("binary_field") .addDocValueField("ip_field"); SearchResponse searchResponse = builder.execute().actionGet(); @@ -790,7 +794,7 @@ public void testFieldsPulledFromFieldData() throws Exception { Set fields = new HashSet<>(searchResponse.getHits().getAt(0).getFields().keySet()); assertThat(fields, equalTo(newHashSet("byte_field", "short_field", "integer_field", "long_field", "float_field", "double_field", "date_field", "boolean_field", "text_field", "keyword_field", - "ip_field"))); + "binary_field", "ip_field"))); assertThat(searchResponse.getHits().getAt(0).getFields().get("byte_field").getValue().toString(), equalTo("1")); assertThat(searchResponse.getHits().getAt(0).getFields().get("short_field").getValue().toString(), equalTo("2")); @@ -802,6 +806,8 @@ public void testFieldsPulledFromFieldData() throws Exception { assertThat(searchResponse.getHits().getAt(0).getFields().get("boolean_field").getValue(), equalTo((Object) true)); assertThat(searchResponse.getHits().getAt(0).getFields().get("text_field").getValue(), equalTo("foo")); assertThat(searchResponse.getHits().getAt(0).getFields().get("keyword_field").getValue(), equalTo("foo")); + assertThat(searchResponse.getHits().getAt(0).getFields().get("binary_field").getValue(), + equalTo(new BytesRef(new byte[] {42, 100}))); assertThat(searchResponse.getHits().getAt(0).getFields().get("ip_field").getValue(), equalTo("::1")); builder = client().prepareSearch().setQuery(matchAllQuery()) @@ -815,6 +821,7 @@ public void testFieldsPulledFromFieldData() throws Exception { .addDocValueField("double_field", "use_field_mapping") .addDocValueField("date_field", "use_field_mapping") .addDocValueField("boolean_field", "use_field_mapping") + .addDocValueField("binary_field", "use_field_mapping") .addDocValueField("ip_field", "use_field_mapping"); searchResponse = builder.execute().actionGet(); @@ -823,7 +830,7 @@ public void testFieldsPulledFromFieldData() throws Exception { fields = new HashSet<>(searchResponse.getHits().getAt(0).getFields().keySet()); assertThat(fields, equalTo(newHashSet("byte_field", "short_field", "integer_field", "long_field", "float_field", "double_field", "date_field", "boolean_field", "text_field", "keyword_field", - "ip_field"))); + "binary_field", "ip_field"))); assertThat(searchResponse.getHits().getAt(0).getFields().get("byte_field").getValue().toString(), equalTo("1")); assertThat(searchResponse.getHits().getAt(0).getFields().get("short_field").getValue().toString(), equalTo("2")); @@ -836,6 +843,7 @@ public void testFieldsPulledFromFieldData() throws Exception { assertThat(searchResponse.getHits().getAt(0).getFields().get("boolean_field").getValue(), equalTo((Object) true)); assertThat(searchResponse.getHits().getAt(0).getFields().get("text_field").getValue(), equalTo("foo")); assertThat(searchResponse.getHits().getAt(0).getFields().get("keyword_field").getValue(), equalTo("foo")); + assertThat(searchResponse.getHits().getAt(0).getFields().get("binary_field").getValue(), equalTo("KmQ")); assertThat(searchResponse.getHits().getAt(0).getFields().get("ip_field").getValue(), equalTo("::1")); builder = client().prepareSearch().setQuery(matchAllQuery())