Skip to content

Commit b71de74

Browse files
jtibshiranijtibshirani
committed
Allow for aliases when fetching stored fields. (#31411)
* When loading stored fields, resolve wildcard patterns against the mappings instead of the index. * Minor simplifications to the logic in FetchPhase. * Pull out a shared method FetchPhase#getSearchFields. * Allow for aliases when fetching stored fields.
1 parent d5642a0 commit b71de74

File tree

4 files changed

+165
-76
lines changed

4 files changed

+165
-76
lines changed

‎server/src/main/java/org/elasticsearch/index/fieldvisitor/CustomFieldsVisitor.java‎

Lines changed: 1 addition & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -19,11 +19,8 @@
1919
package org.elasticsearch.index.fieldvisitor;
2020

2121
import org.apache.lucene.index.FieldInfo;
22-
import org.elasticsearch.common.regex.Regex;
2322

2423
import java.io.IOException;
25-
import java.util.Collections;
26-
import java.util.List;
2724
import java.util.Set;
2825

2926
/**
@@ -35,16 +32,10 @@
3532
public class CustomFieldsVisitor extends FieldsVisitor {
3633

3734
private final Set<String> fields;
38-
private final List<String> patterns;
3935

40-
public CustomFieldsVisitor(Set<String> fields, List<String> patterns, boolean loadSource) {
36+
public CustomFieldsVisitor(Set<String> fields, boolean loadSource) {
4137
super(loadSource);
4238
this.fields = fields;
43-
this.patterns = patterns;
44-
}
45-
46-
public CustomFieldsVisitor(Set<String> fields, boolean loadSource) {
47-
this(fields, Collections.emptyList(), loadSource);
4839
}
4940

5041
@Override
@@ -55,11 +46,6 @@ public Status needsField(FieldInfo fieldInfo) throws IOException {
5546
if (fields.contains(fieldInfo.name)) {
5647
return Status.YES;
5748
}
58-
for (String pattern : patterns) {
59-
if (Regex.simpleMatch(pattern, fieldInfo.name)) {
60-
return Status.YES;
61-
}
62-
}
6349
return Status.NO;
6450
}
6551
}

‎server/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java‎

Lines changed: 66 additions & 58 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,6 @@
3131
import org.elasticsearch.common.collect.Tuple;
3232
import org.elasticsearch.common.document.DocumentField;
3333
import org.elasticsearch.common.lucene.search.Queries;
34-
import org.elasticsearch.common.regex.Regex;
3534
import org.elasticsearch.common.text.Text;
3635
import org.elasticsearch.common.xcontent.XContentHelper;
3736
import org.elasticsearch.common.xcontent.XContentType;
@@ -55,7 +54,7 @@
5554
import org.elasticsearch.tasks.TaskCancelledException;
5655

5756
import java.io.IOException;
58-
import java.util.ArrayList;
57+
import java.util.Collection;
5958
import java.util.Collections;
6059
import java.util.HashMap;
6160
import java.util.HashSet;
@@ -84,8 +83,7 @@ public void preProcess(SearchContext context) {
8483
@Override
8584
public void execute(SearchContext context) {
8685
final FieldsVisitor fieldsVisitor;
87-
Set<String> fieldNames = null;
88-
List<String> fieldNamePatterns = null;
86+
Map<String, Set<String>> storedToRequestedFields = new HashMap<>();
8987
StoredFieldsContext storedFieldsContext = context.storedFieldsContext();
9088

9189
if (storedFieldsContext == null) {
@@ -98,39 +96,36 @@ public void execute(SearchContext context) {
9896
// disable stored fields entirely
9997
fieldsVisitor = null;
10098
} else {
101-
for (String fieldName : context.storedFieldsContext().fieldNames()) {
102-
if (fieldName.equals(SourceFieldMapper.NAME)) {
99+
for (String fieldNameOrPattern : context.storedFieldsContext().fieldNames()) {
100+
if (fieldNameOrPattern.equals(SourceFieldMapper.NAME)) {
103101
FetchSourceContext fetchSourceContext = context.hasFetchSourceContext() ? context.fetchSourceContext()
104-
: FetchSourceContext.FETCH_SOURCE;
102+
: FetchSourceContext.FETCH_SOURCE;
105103
context.fetchSourceContext(new FetchSourceContext(true, fetchSourceContext.includes(), fetchSourceContext.excludes()));
106104
continue;
107105
}
108-
if (Regex.isSimpleMatchPattern(fieldName)) {
109-
if (fieldNamePatterns == null) {
110-
fieldNamePatterns = new ArrayList<>();
111-
}
112-
fieldNamePatterns.add(fieldName);
113-
} else {
106+
107+
Collection<String> fieldNames = context.mapperService().simpleMatchToFullName(fieldNameOrPattern);
108+
for (String fieldName : fieldNames) {
114109
MappedFieldType fieldType = context.smartNameFieldType(fieldName);
115110
if (fieldType == null) {
116111
// Only fail if we know it is a object field, missing paths / fields shouldn't fail.
117112
if (context.getObjectMapper(fieldName) != null) {
118113
throw new IllegalArgumentException("field [" + fieldName + "] isn't a leaf field");
119114
}
115+
} else {
116+
String storedField = fieldType.name();
117+
Set<String> requestedFields = storedToRequestedFields.computeIfAbsent(
118+
storedField, key -> new HashSet<>());
119+
requestedFields.add(fieldName);
120120
}
121-
if (fieldNames == null) {
122-
fieldNames = new HashSet<>();
123-
}
124-
fieldNames.add(fieldName);
125121
}
126122
}
127123
boolean loadSource = context.sourceRequested();
128-
if (fieldNames == null && fieldNamePatterns == null) {
124+
if (storedToRequestedFields.isEmpty()) {
129125
// empty list specified, default to disable _source if no explicit indication
130126
fieldsVisitor = new FieldsVisitor(loadSource);
131127
} else {
132-
fieldsVisitor = new CustomFieldsVisitor(fieldNames == null ? Collections.emptySet() : fieldNames,
133-
fieldNamePatterns == null ? Collections.emptyList() : fieldNamePatterns, loadSource);
128+
fieldsVisitor = new CustomFieldsVisitor(storedToRequestedFields.keySet(), loadSource);
134129
}
135130
}
136131

@@ -149,10 +144,11 @@ public void execute(SearchContext context) {
149144
final SearchHit searchHit;
150145
int rootDocId = findRootDocumentIfNested(context, subReaderContext, subDocId);
151146
if (rootDocId != -1) {
152-
searchHit = createNestedSearchHit(context, docId, subDocId, rootDocId, fieldNames, fieldNamePatterns,
153-
subReaderContext);
147+
searchHit = createNestedSearchHit(context, docId, subDocId, rootDocId,
148+
storedToRequestedFields, subReaderContext);
154149
} else {
155-
searchHit = createSearchHit(context, fieldsVisitor, docId, subDocId, subReaderContext);
150+
searchHit = createSearchHit(context, fieldsVisitor, docId, subDocId,
151+
storedToRequestedFields, subReaderContext);
156152
}
157153

158154
hits[index] = searchHit;
@@ -190,21 +186,18 @@ private int findRootDocumentIfNested(SearchContext context, LeafReaderContext su
190186
return -1;
191187
}
192188

193-
private SearchHit createSearchHit(SearchContext context, FieldsVisitor fieldsVisitor, int docId, int subDocId,
189+
private SearchHit createSearchHit(SearchContext context,
190+
FieldsVisitor fieldsVisitor,
191+
int docId,
192+
int subDocId,
193+
Map<String, Set<String>> storedToRequestedFields,
194194
LeafReaderContext subReaderContext) {
195195
if (fieldsVisitor == null) {
196196
return new SearchHit(docId);
197197
}
198-
loadStoredFields(context, subReaderContext, fieldsVisitor, subDocId);
199-
fieldsVisitor.postProcess(context.mapperService());
200198

201-
Map<String, DocumentField> searchFields = null;
202-
if (!fieldsVisitor.fields().isEmpty()) {
203-
searchFields = new HashMap<>(fieldsVisitor.fields().size());
204-
for (Map.Entry<String, List<Object>> entry : fieldsVisitor.fields().entrySet()) {
205-
searchFields.put(entry.getKey(), new DocumentField(entry.getKey(), entry.getValue()));
206-
}
207-
}
199+
Map<String, DocumentField> searchFields = getSearchFields(context, fieldsVisitor, subDocId,
200+
storedToRequestedFields, subReaderContext);
208201

209202
DocumentMapper documentMapper = context.mapperService().documentMapper(fieldsVisitor.uid().type());
210203
Text typeText;
@@ -223,9 +216,40 @@ private SearchHit createSearchHit(SearchContext context, FieldsVisitor fieldsVis
223216
return searchHit;
224217
}
225218

226-
private SearchHit createNestedSearchHit(SearchContext context, int nestedTopDocId, int nestedSubDocId,
227-
int rootSubDocId, Set<String> fieldNames,
228-
List<String> fieldNamePatterns, LeafReaderContext subReaderContext) throws IOException {
219+
private Map<String, DocumentField> getSearchFields(SearchContext context,
220+
FieldsVisitor fieldsVisitor,
221+
int subDocId,
222+
Map<String, Set<String>> storedToRequestedFields,
223+
LeafReaderContext subReaderContext) {
224+
loadStoredFields(context, subReaderContext, fieldsVisitor, subDocId);
225+
fieldsVisitor.postProcess(context.mapperService());
226+
227+
if (fieldsVisitor.fields().isEmpty()) {
228+
return null;
229+
}
230+
231+
Map<String, DocumentField> searchFields = new HashMap<>(fieldsVisitor.fields().size());
232+
for (Map.Entry<String, List<Object>> entry : fieldsVisitor.fields().entrySet()) {
233+
String storedField = entry.getKey();
234+
List<Object> storedValues = entry.getValue();
235+
236+
if (storedToRequestedFields.containsKey(storedField)) {
237+
for (String requestedField : storedToRequestedFields.get(storedField)) {
238+
searchFields.put(requestedField, new DocumentField(requestedField, storedValues));
239+
}
240+
} else {
241+
searchFields.put(storedField, new DocumentField(storedField, storedValues));
242+
}
243+
}
244+
return searchFields;
245+
}
246+
247+
private SearchHit createNestedSearchHit(SearchContext context,
248+
int nestedTopDocId,
249+
int nestedSubDocId,
250+
int rootSubDocId,
251+
Map<String, Set<String>> storedToRequestedFields,
252+
LeafReaderContext subReaderContext) throws IOException {
229253
// Also if highlighting is requested on nested documents we need to fetch the _source from the root document,
230254
// otherwise highlighting will attempt to fetch the _source from the nested doc, which will fail,
231255
// because the entire _source is only stored with the root document.
@@ -244,9 +268,13 @@ private SearchHit createNestedSearchHit(SearchContext context, int nestedTopDocI
244268
source = null;
245269
}
246270

271+
Map<String, DocumentField> searchFields = null;
272+
if (context.hasStoredFields() && !context.storedFieldsContext().fieldNames().isEmpty()) {
273+
FieldsVisitor nestedFieldsVisitor = new CustomFieldsVisitor(storedToRequestedFields.keySet(), false);
274+
searchFields = getSearchFields(context, nestedFieldsVisitor, nestedSubDocId,
275+
storedToRequestedFields, subReaderContext);
276+
}
247277

248-
Map<String, DocumentField> searchFields =
249-
getSearchFields(context, nestedSubDocId, fieldNames, fieldNamePatterns, subReaderContext);
250278
DocumentMapper documentMapper = context.mapperService().documentMapper(uid.type());
251279
SourceLookup sourceLookup = context.lookup().source();
252280
sourceLookup.setSegmentAndDocument(subReaderContext, nestedSubDocId);
@@ -307,26 +335,6 @@ private SearchHit createNestedSearchHit(SearchContext context, int nestedTopDocI
307335
return new SearchHit(nestedTopDocId, uid.id(), documentMapper.typeText(), nestedIdentity, searchFields);
308336
}
309337

310-
private Map<String, DocumentField> getSearchFields(SearchContext context, int nestedSubDocId, Set<String> fieldNames,
311-
List<String> fieldNamePatterns, LeafReaderContext subReaderContext) {
312-
Map<String, DocumentField> searchFields = null;
313-
if (context.hasStoredFields() && !context.storedFieldsContext().fieldNames().isEmpty()) {
314-
FieldsVisitor nestedFieldsVisitor = new CustomFieldsVisitor(fieldNames == null ? Collections.emptySet() : fieldNames,
315-
fieldNamePatterns == null ? Collections.emptyList() : fieldNamePatterns, false);
316-
if (nestedFieldsVisitor != null) {
317-
loadStoredFields(context, subReaderContext, nestedFieldsVisitor, nestedSubDocId);
318-
nestedFieldsVisitor.postProcess(context.mapperService());
319-
if (!nestedFieldsVisitor.fields().isEmpty()) {
320-
searchFields = new HashMap<>(nestedFieldsVisitor.fields().size());
321-
for (Map.Entry<String, List<Object>> entry : nestedFieldsVisitor.fields().entrySet()) {
322-
searchFields.put(entry.getKey(), new DocumentField(entry.getKey(), entry.getValue()));
323-
}
324-
}
325-
}
326-
}
327-
return searchFields;
328-
}
329-
330338
private SearchHit.NestedIdentity getInternalNestedIdentity(SearchContext context, int nestedSubDocId,
331339
LeafReaderContext subReaderContext,
332340
MapperService mapperService,

‎server/src/test/java/org/elasticsearch/index/mapper/StoredNumericValuesTests.java‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -28,13 +28,14 @@
2828
import org.elasticsearch.common.bytes.BytesReference;
2929
import org.elasticsearch.common.compress.CompressedXContent;
3030
import org.elasticsearch.common.lucene.Lucene;
31+
import org.elasticsearch.common.util.set.Sets;
3132
import org.elasticsearch.common.xcontent.XContentFactory;
3233
import org.elasticsearch.common.xcontent.XContentType;
3334
import org.elasticsearch.index.fieldvisitor.CustomFieldsVisitor;
3435
import org.elasticsearch.index.mapper.MapperService.MergeReason;
3536
import org.elasticsearch.test.ESSingleNodeTestCase;
3637

37-
import java.util.Collections;
38+
import java.util.Set;
3839

3940
import static org.hamcrest.Matchers.equalTo;
4041

@@ -84,9 +85,11 @@ public void testBytesAndNumericRepresentation() throws Exception {
8485
DirectoryReader reader = DirectoryReader.open(writer);
8586
IndexSearcher searcher = new IndexSearcher(reader);
8687

87-
CustomFieldsVisitor fieldsVisitor = new CustomFieldsVisitor(
88-
Collections.emptySet(), Collections.singletonList("field*"), false);
88+
Set<String> fieldNames = Sets.newHashSet("field1", "field2", "field3", "field4", "field5",
89+
"field6", "field7", "field8", "field9", "field10");
90+
CustomFieldsVisitor fieldsVisitor = new CustomFieldsVisitor(fieldNames, false);
8991
searcher.doc(0, fieldsVisitor);
92+
9093
fieldsVisitor.postProcess(mapperService);
9194
assertThat(fieldsVisitor.fields().size(), equalTo(10));
9295
assertThat(fieldsVisitor.fields().get("field1").size(), equalTo(1));

‎server/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -980,6 +980,98 @@ public void testDocValueFieldsWithFieldAlias() throws Exception {
980980
assertThat(fetchedDate, equalTo(date));
981981
}
982982

983+
984+
public void testStoredFieldsWithFieldAlias() throws Exception {
985+
XContentBuilder mapping = XContentFactory.jsonBuilder()
986+
.startObject()
987+
.startObject("type")
988+
.startObject("properties")
989+
.startObject("field1")
990+
.field("type", "text")
991+
.field("store", true)
992+
.endObject()
993+
.startObject("field2")
994+
.field("type", "text")
995+
.field("store", false)
996+
.endObject()
997+
.startObject("field1-alias")
998+
.field("type", "alias")
999+
.field("path", "field1")
1000+
.endObject()
1001+
.startObject("field2-alias")
1002+
.field("type", "alias")
1003+
.field("path", "field2")
1004+
.endObject()
1005+
.endObject()
1006+
.endObject()
1007+
.endObject();
1008+
assertAcked(prepareCreate("test").addMapping("type", mapping));
1009+
1010+
index("test", "type", "1", "field1", "value1", "field2", "value2");
1011+
refresh("test");
1012+
1013+
SearchResponse searchResponse = client().prepareSearch()
1014+
.setQuery(matchAllQuery())
1015+
.addStoredField("field1-alias")
1016+
.addStoredField("field2-alias")
1017+
.get();
1018+
assertHitCount(searchResponse, 1L);
1019+
1020+
SearchHit hit = searchResponse.getHits().getAt(0);
1021+
assertEquals(1, hit.getFields().size());
1022+
assertTrue(hit.getFields().containsKey("field1-alias"));
1023+
1024+
DocumentField field = hit.getFields().get("field1-alias");
1025+
assertThat(field.getValue().toString(), equalTo("value1"));
1026+
}
1027+
1028+
public void testWildcardStoredFieldsWithFieldAlias() throws Exception {
1029+
XContentBuilder mapping = XContentFactory.jsonBuilder()
1030+
.startObject()
1031+
.startObject("type")
1032+
.startObject("properties")
1033+
.startObject("field1")
1034+
.field("type", "text")
1035+
.field("store", true)
1036+
.endObject()
1037+
.startObject("field2")
1038+
.field("type", "text")
1039+
.field("store", false)
1040+
.endObject()
1041+
.startObject("field1-alias")
1042+
.field("type", "alias")
1043+
.field("path", "field1")
1044+
.endObject()
1045+
.startObject("field2-alias")
1046+
.field("type", "alias")
1047+
.field("path", "field2")
1048+
.endObject()
1049+
.endObject()
1050+
.endObject()
1051+
.endObject();
1052+
assertAcked(prepareCreate("test").addMapping("type", mapping));
1053+
1054+
index("test", "type", "1", "field1", "value1", "field2", "value2");
1055+
refresh("test");
1056+
1057+
SearchResponse searchResponse = client().prepareSearch()
1058+
.setQuery(matchAllQuery())
1059+
.addStoredField("field*")
1060+
.get();
1061+
assertHitCount(searchResponse, 1L);
1062+
1063+
SearchHit hit = searchResponse.getHits().getAt(0);
1064+
assertEquals(2, hit.getFields().size());
1065+
assertTrue(hit.getFields().containsKey("field1"));
1066+
assertTrue(hit.getFields().containsKey("field1-alias"));
1067+
1068+
DocumentField field = hit.getFields().get("field1");
1069+
assertThat(field.getValue().toString(), equalTo("value1"));
1070+
1071+
DocumentField fieldAlias = hit.getFields().get("field1-alias");
1072+
assertThat(fieldAlias.getValue().toString(), equalTo("value1"));
1073+
}
1074+
9831075
public void testLoadMetadata() throws Exception {
9841076
assertAcked(prepareCreate("test"));
9851077

0 commit comments

Comments
 (0)