Skip to content

Commit e182072

Browse files
jimczijimczi
committed
Merge pull request #15017 from jimferenczi/fields_option
Refuse to load fields from _source when using the `fields` option and support wildcards.
2 parents 72be42d + 731833c commit e182072

File tree

9 files changed

+95
-149
lines changed

9 files changed

+95
-149
lines changed

‎core/src/main/java/org/elasticsearch/index/fieldvisitor/AllFieldsVisitor.java‎

Lines changed: 0 additions & 37 deletions
This file was deleted.

‎core/src/main/java/org/elasticsearch/index/fieldvisitor/CustomFieldsVisitor.java‎

Lines changed: 21 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -19,30 +19,47 @@
1919
package org.elasticsearch.index.fieldvisitor;
2020

2121
import org.apache.lucene.index.FieldInfo;
22+
import org.elasticsearch.common.regex.Regex;
2223

2324
import java.io.IOException;
25+
import java.util.Collections;
26+
import java.util.List;
2427
import java.util.Set;
2528

2629
/**
27-
* A field visitor that allows to load a selection of the stored fields.
30+
* A field visitor that allows to load a selection of the stored fields by exact name or by pattern.
31+
* Supported pattern styles: "xxx*", "*xxx", "*xxx*" and "xxx*yyy".
2832
* The Uid field is always loaded.
2933
* The class is optimized for source loading as it is a common use case.
3034
*/
3135
public class CustomFieldsVisitor extends FieldsVisitor {
3236

3337
private final Set<String> fields;
38+
private final List<String> patterns;
3439

35-
public CustomFieldsVisitor(Set<String> fields, boolean loadSource) {
40+
public CustomFieldsVisitor(Set<String> fields, List<String> patterns, boolean loadSource) {
3641
super(loadSource);
3742
this.fields = fields;
43+
this.patterns = patterns;
44+
}
45+
46+
public CustomFieldsVisitor(Set<String> fields, boolean loadSource) {
47+
this(fields, Collections.emptyList(), loadSource);
3848
}
3949

4050
@Override
4151
public Status needsField(FieldInfo fieldInfo) throws IOException {
4252
if (super.needsField(fieldInfo) == Status.YES) {
4353
return Status.YES;
4454
}
45-
46-
return fields.contains(fieldInfo.name) ? Status.YES : Status.NO;
55+
if (fields.contains(fieldInfo.name)) {
56+
return Status.YES;
57+
}
58+
for (String pattern : patterns) {
59+
if (Regex.simpleMatch(pattern, fieldInfo.name)) {
60+
return Status.YES;
61+
}
62+
}
63+
return Status.NO;
4764
}
4865
}

‎core/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java‎

Lines changed: 29 additions & 91 deletions
Original file line numberDiff line numberDiff line change
@@ -30,12 +30,12 @@
3030
import org.elasticsearch.common.collect.Tuple;
3131
import org.elasticsearch.common.inject.Inject;
3232
import org.elasticsearch.common.lucene.search.Queries;
33+
import org.elasticsearch.common.regex.Regex;
3334
import org.elasticsearch.common.text.StringAndBytesText;
3435
import org.elasticsearch.common.text.Text;
3536
import org.elasticsearch.common.xcontent.XContentHelper;
3637
import org.elasticsearch.common.xcontent.XContentType;
3738
import org.elasticsearch.common.xcontent.support.XContentMapValues;
38-
import org.elasticsearch.index.fieldvisitor.AllFieldsVisitor;
3939
import org.elasticsearch.index.fieldvisitor.CustomFieldsVisitor;
4040
import org.elasticsearch.index.fieldvisitor.FieldsVisitor;
4141
import org.elasticsearch.index.mapper.DocumentMapper;
@@ -55,13 +55,7 @@
5555
import org.elasticsearch.search.lookup.SourceLookup;
5656

5757
import java.io.IOException;
58-
import java.util.ArrayList;
59-
import java.util.Collections;
60-
import java.util.HashMap;
61-
import java.util.HashSet;
62-
import java.util.List;
63-
import java.util.Map;
64-
import java.util.Set;
58+
import java.util.*;
6559

6660
import static java.util.Collections.unmodifiableMap;
6761
import static org.elasticsearch.common.xcontent.XContentFactory.contentBuilder;
@@ -98,9 +92,7 @@ public void preProcess(SearchContext context) {
9892
public void execute(SearchContext context) {
9993
FieldsVisitor fieldsVisitor;
10094
Set<String> fieldNames = null;
101-
List<String> extractFieldNames = null;
102-
103-
boolean loadAllStored = false;
95+
List<String> fieldNamePatterns = null;
10496
if (!context.hasFieldNames()) {
10597
// no fields specified, default to return source if no explicit indication
10698
if (!context.hasScriptFields() && !context.hasFetchSourceContext()) {
@@ -111,10 +103,6 @@ public void execute(SearchContext context) {
111103
fieldsVisitor = new FieldsVisitor(context.sourceRequested());
112104
} else {
113105
for (String fieldName : context.fieldNames()) {
114-
if (fieldName.equals("*")) {
115-
loadAllStored = true;
116-
continue;
117-
}
118106
if (fieldName.equals(SourceFieldMapper.NAME)) {
119107
if (context.hasFetchSourceContext()) {
120108
context.fetchSourceContext().fetchSource(true);
@@ -123,32 +111,28 @@ public void execute(SearchContext context) {
123111
}
124112
continue;
125113
}
126-
MappedFieldType fieldType = context.smartNameFieldType(fieldName);
127-
if (fieldType == null) {
128-
// Only fail if we know it is a object field, missing paths / fields shouldn't fail.
129-
if (context.getObjectMapper(fieldName) != null) {
130-
throw new IllegalArgumentException("field [" + fieldName + "] isn't a leaf field");
114+
if (Regex.isSimpleMatchPattern(fieldName)) {
115+
if (fieldNamePatterns == null) {
116+
fieldNamePatterns = new ArrayList<>();
117+
}
118+
fieldNamePatterns.add(fieldName);
119+
} else {
120+
MappedFieldType fieldType = context.smartNameFieldType(fieldName);
121+
if (fieldType == null) {
122+
// Only fail if we know it is a object field, missing paths / fields shouldn't fail.
123+
if (context.getObjectMapper(fieldName) != null) {
124+
throw new IllegalArgumentException("field [" + fieldName + "] isn't a leaf field");
125+
}
131126
}
132-
} else if (fieldType.stored()) {
133127
if (fieldNames == null) {
134128
fieldNames = new HashSet<>();
135129
}
136-
fieldNames.add(fieldType.names().indexName());
137-
} else {
138-
if (extractFieldNames == null) {
139-
extractFieldNames = new ArrayList<>();
140-
}
141-
extractFieldNames.add(fieldName);
130+
fieldNames.add(fieldName);
142131
}
143132
}
144-
if (loadAllStored) {
145-
fieldsVisitor = new AllFieldsVisitor(); // load everything, including _source
146-
} else if (fieldNames != null) {
147-
boolean loadSource = extractFieldNames != null || context.sourceRequested();
148-
fieldsVisitor = new CustomFieldsVisitor(fieldNames, loadSource);
149-
} else {
150-
fieldsVisitor = new FieldsVisitor(extractFieldNames != null || context.sourceRequested());
151-
}
133+
boolean loadSource = context.sourceRequested();
134+
fieldsVisitor = new CustomFieldsVisitor(fieldNames == null ? Collections.emptySet() : fieldNames,
135+
fieldNamePatterns == null ? Collections.emptyList() : fieldNamePatterns, loadSource);
152136
}
153137

154138
InternalSearchHit[] hits = new InternalSearchHit[context.docIdsToLoadSize()];
@@ -163,9 +147,9 @@ public void execute(SearchContext context) {
163147
try {
164148
int rootDocId = findRootDocumentIfNested(context, subReaderContext, subDocId);
165149
if (rootDocId != -1) {
166-
searchHit = createNestedSearchHit(context, docId, subDocId, rootDocId, extractFieldNames, loadAllStored, fieldNames, subReaderContext);
150+
searchHit = createNestedSearchHit(context, docId, subDocId, rootDocId, fieldNames, fieldNamePatterns, subReaderContext);
167151
} else {
168-
searchHit = createSearchHit(context, fieldsVisitor, docId, subDocId, extractFieldNames, subReaderContext);
152+
searchHit = createSearchHit(context, fieldsVisitor, docId, subDocId, subReaderContext);
169153
}
170154
} catch (IOException e) {
171155
throw ExceptionsHelper.convertToElastic(e);
@@ -199,7 +183,7 @@ private int findRootDocumentIfNested(SearchContext context, LeafReaderContext su
199183
return -1;
200184
}
201185

202-
private InternalSearchHit createSearchHit(SearchContext context, FieldsVisitor fieldsVisitor, int docId, int subDocId, List<String> extractFieldNames, LeafReaderContext subReaderContext) {
186+
private InternalSearchHit createSearchHit(SearchContext context, FieldsVisitor fieldsVisitor, int docId, int subDocId, LeafReaderContext subReaderContext) {
203187
loadStoredFields(context, subReaderContext, fieldsVisitor, subDocId);
204188
fieldsVisitor.postProcess(context.mapperService());
205189

@@ -219,45 +203,24 @@ private InternalSearchHit createSearchHit(SearchContext context, FieldsVisitor f
219203
typeText = documentMapper.typeText();
220204
}
221205
InternalSearchHit searchHit = new InternalSearchHit(docId, fieldsVisitor.uid().id(), typeText, searchFields);
222-
223-
// go over and extract fields that are not mapped / stored
206+
// Set _source if requested.
224207
SourceLookup sourceLookup = context.lookup().source();
225208
sourceLookup.setSegmentAndDocument(subReaderContext, subDocId);
226209
if (fieldsVisitor.source() != null) {
227210
sourceLookup.setSource(fieldsVisitor.source());
228211
}
229-
if (extractFieldNames != null) {
230-
for (String extractFieldName : extractFieldNames) {
231-
List<Object> values = context.lookup().source().extractRawValues(extractFieldName);
232-
if (!values.isEmpty()) {
233-
if (searchHit.fieldsOrNull() == null) {
234-
searchHit.fields(new HashMap<String, SearchHitField>(2));
235-
}
236-
237-
SearchHitField hitField = searchHit.fields().get(extractFieldName);
238-
if (hitField == null) {
239-
hitField = new InternalSearchHitField(extractFieldName, new ArrayList<>(2));
240-
searchHit.fields().put(extractFieldName, hitField);
241-
}
242-
for (Object value : values) {
243-
hitField.values().add(value);
244-
}
245-
}
246-
}
247-
}
248-
249212
return searchHit;
250213
}
251214

252-
private InternalSearchHit createNestedSearchHit(SearchContext context, int nestedTopDocId, int nestedSubDocId, int rootSubDocId, List<String> extractFieldNames, boolean loadAllStored, Set<String> fieldNames, LeafReaderContext subReaderContext) throws IOException {
215+
private InternalSearchHit createNestedSearchHit(SearchContext context, int nestedTopDocId, int nestedSubDocId, int rootSubDocId, Set<String> fieldNames, List<String> fieldNamePatterns, LeafReaderContext subReaderContext) throws IOException {
253216
// Also if highlighting is requested on nested documents we need to fetch the _source from the root document,
254217
// otherwise highlighting will attempt to fetch the _source from the nested doc, which will fail,
255218
// because the entire _source is only stored with the root document.
256-
final FieldsVisitor rootFieldsVisitor = new FieldsVisitor(context.sourceRequested() || extractFieldNames != null || context.highlight() != null);
219+
final FieldsVisitor rootFieldsVisitor = new FieldsVisitor(context.sourceRequested() || context.highlight() != null);
257220
loadStoredFields(context, subReaderContext, rootFieldsVisitor, rootSubDocId);
258221
rootFieldsVisitor.postProcess(context.mapperService());
259222

260-
Map<String, SearchHitField> searchFields = getSearchFields(context, nestedSubDocId, loadAllStored, fieldNames, subReaderContext);
223+
Map<String, SearchHitField> searchFields = getSearchFields(context, nestedSubDocId, fieldNames, fieldNamePatterns, subReaderContext);
261224
DocumentMapper documentMapper = context.mapperService().documentMapper(rootFieldsVisitor.uid().type());
262225
SourceLookup sourceLookup = context.lookup().source();
263226
sourceLookup.setSegmentAndDocument(subReaderContext, nestedSubDocId);
@@ -299,39 +262,14 @@ private InternalSearchHit createNestedSearchHit(SearchContext context, int neste
299262
}
300263

301264
InternalSearchHit searchHit = new InternalSearchHit(nestedTopDocId, rootFieldsVisitor.uid().id(), documentMapper.typeText(), nestedIdentity, searchFields);
302-
if (extractFieldNames != null) {
303-
for (String extractFieldName : extractFieldNames) {
304-
List<Object> values = context.lookup().source().extractRawValues(extractFieldName);
305-
if (!values.isEmpty()) {
306-
if (searchHit.fieldsOrNull() == null) {
307-
searchHit.fields(new HashMap<String, SearchHitField>(2));
308-
}
309-
310-
SearchHitField hitField = searchHit.fields().get(extractFieldName);
311-
if (hitField == null) {
312-
hitField = new InternalSearchHitField(extractFieldName, new ArrayList<>(2));
313-
searchHit.fields().put(extractFieldName, hitField);
314-
}
315-
for (Object value : values) {
316-
hitField.values().add(value);
317-
}
318-
}
319-
}
320-
}
321-
322265
return searchHit;
323266
}
324267

325-
private Map<String, SearchHitField> getSearchFields(SearchContext context, int nestedSubDocId, boolean loadAllStored, Set<String> fieldNames, LeafReaderContext subReaderContext) {
268+
private Map<String, SearchHitField> getSearchFields(SearchContext context, int nestedSubDocId, Set<String> fieldNames, List<String> fieldNamePatterns, LeafReaderContext subReaderContext) {
326269
Map<String, SearchHitField> searchFields = null;
327270
if (context.hasFieldNames() && !context.fieldNames().isEmpty()) {
328-
FieldsVisitor nestedFieldsVisitor = null;
329-
if (loadAllStored) {
330-
nestedFieldsVisitor = new AllFieldsVisitor();
331-
} else if (fieldNames != null) {
332-
nestedFieldsVisitor = new CustomFieldsVisitor(fieldNames, false);
333-
}
334-
271+
FieldsVisitor nestedFieldsVisitor = new CustomFieldsVisitor(fieldNames == null ? Collections.emptySet() : fieldNames,
272+
fieldNamePatterns == null ? Collections.emptyList() : fieldNamePatterns, false);
335273
if (nestedFieldsVisitor != null) {
336274
loadStoredFields(context, subReaderContext, nestedFieldsVisitor, nestedSubDocId);
337275
nestedFieldsVisitor.postProcess(context.mapperService());

‎core/src/test/java/org/elasticsearch/indices/template/SimpleIndexTemplateIT.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -118,7 +118,8 @@ public void testSimpleIndexTemplateTests() throws Exception {
118118

119119
assertHitCount(searchResponse, 1);
120120
assertThat(searchResponse.getHits().getAt(0).field("field1").value().toString(), equalTo("value1"));
121-
assertThat(searchResponse.getHits().getAt(0).field("field2").value().toString(), equalTo("value 2")); // this will still be loaded because of the source feature
121+
// field2 is not stored.
122+
assertThat(searchResponse.getHits().getAt(0).field("field2"), nullValue());
122123

123124
client().prepareIndex("text_index", "type1", "1").setSource("field1", "value1", "field2", "value 2").setRefresh(true).execute().actionGet();
124125

‎core/src/test/java/org/elasticsearch/search/aggregations/metrics/AbstractGeoTestCase.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -154,7 +154,7 @@ public void setupSuiteScopeCluster() throws Exception {
154154
.endObject()));
155155
}
156156
assertAcked(prepareCreate(HIGH_CARD_IDX_NAME).setSettings(Settings.builder().put("number_of_shards", 2))
157-
.addMapping("type", SINGLE_VALUED_FIELD_NAME, "type=geo_point", MULTI_VALUED_FIELD_NAME, "type=geo_point", NUMBER_FIELD_NAME, "type=long", "tag", "type=string,index=not_analyzed"));
157+
.addMapping("type", SINGLE_VALUED_FIELD_NAME, "type=geo_point", MULTI_VALUED_FIELD_NAME, "type=geo_point", NUMBER_FIELD_NAME, "type=long,store=true", "tag", "type=string,index=not_analyzed"));
158158

159159
for (int i = 0; i < 2000; i++) {
160160
singleVal = singleValues[i % numUniqueGeoPoints];
@@ -196,8 +196,8 @@ public void setupSuiteScopeCluster() throws Exception {
196196
SearchHitField hitField = searchHit.field(NUMBER_FIELD_NAME);
197197

198198
assertThat("Hit " + i + " has wrong number of values", hitField.getValues().size(), equalTo(1));
199-
Integer value = hitField.getValue();
200-
assertThat("Hit " + i + " has wrong value", value, equalTo(i));
199+
Long value = hitField.getValue();
200+
assertThat("Hit " + i + " has wrong value", value.intValue(), equalTo(i));
201201
}
202202
assertThat(totalHits, equalTo(2000l));
203203
}

‎core/src/test/java/org/elasticsearch/search/aggregations/metrics/TopHitsIT.java‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -532,8 +532,8 @@ public void testFetchFeatures() {
532532
topHits("hits").setSize(1)
533533
.highlighter(new HighlightBuilder().field("text"))
534534
.setExplain(true)
535-
.addFieldDataField("field1")
536535
.addField("text")
536+
.addFieldDataField("field1")
537537
.addScriptField("script", new Script("5", ScriptService.ScriptType.INLINE, MockScriptEngine.NAME, Collections.emptyMap()))
538538
.setFetchSource("text", null)
539539
.setVersion(true)
@@ -569,8 +569,7 @@ public void testFetchFeatures() {
569569
SearchHitField field = hit.field("field1");
570570
assertThat(field.getValue().toString(), equalTo("5"));
571571

572-
field = hit.field("text");
573-
assertThat(field.getValue().toString(), equalTo("some text to entertain"));
572+
assertThat(hit.getSource().get("text").toString(), equalTo("some text to entertain"));
574573

575574
field = hit.field("script");
576575
assertThat(field.getValue().toString(), equalTo("5"));

‎docs/reference/migration/migrate_3_0.asciidoc‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -468,3 +468,7 @@ response is output by default.
468468
Finally, the API for org.elasticsearch.monitor.os.OsStats has changed. The `getLoadAverage` method has been removed. The
469469
value for this can now be obtained from `OsStats.Cpu#getLoadAverage`. Additionally, the recent CPU usage can be obtained
470470
from `OsStats.Cpu#getPercent`.
471+
472+
=== Fields option
473+
Only stored fields are retrievable with this option.
474+
The fields option won't be able to load non stored fields from _source anymore.

0 commit comments

Comments
 (0)