Skip to content

Commit a6b0f0a

Browse files
rjernstrjernst
committed
Merge pull request #11171 from rjernst/fix/11116
Mappings: Add back support for enabled/includes/excludes in _source
2 parents 7efc43d + 0e14c6d commit a6b0f0a

File tree

4 files changed

+135
-144
lines changed

4 files changed

+135
-144
lines changed

‎src/main/java/org/elasticsearch/index/mapper/internal/SourceFieldMapper.java‎

Lines changed: 30 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@
5454
import org.elasticsearch.index.mapper.core.AbstractFieldMapper;
5555

5656
import java.io.IOException;
57+
import java.util.Arrays;
5758
import java.util.Iterator;
5859
import java.util.List;
5960
import java.util.Map;
@@ -150,7 +151,7 @@ public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext
150151
Map.Entry<String, Object> entry = iterator.next();
151152
String fieldName = Strings.toUnderscoreCase(entry.getKey());
152153
Object fieldNode = entry.getValue();
153-
if (fieldName.equals("enabled") && parserContext.indexVersionCreated().before(Version.V_2_0_0)) {
154+
if (fieldName.equals("enabled")) {
154155
builder.enabled(nodeBooleanValue(fieldNode));
155156
iterator.remove();
156157
} else if (fieldName.equals("compress") && parserContext.indexVersionCreated().before(Version.V_2_0_0)) {
@@ -172,15 +173,15 @@ public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext
172173
} else if ("format".equals(fieldName)) {
173174
builder.format(nodeStringValue(fieldNode, null));
174175
iterator.remove();
175-
} else if (fieldName.equals("includes") && parserContext.indexVersionCreated().before(Version.V_2_0_0)) {
176+
} else if (fieldName.equals("includes")) {
176177
List<Object> values = (List<Object>) fieldNode;
177178
String[] includes = new String[values.size()];
178179
for (int i = 0; i < includes.length; i++) {
179180
includes[i] = values.get(i).toString();
180181
}
181182
builder.includes(includes);
182183
iterator.remove();
183-
} else if (fieldName.equals("excludes") && parserContext.indexVersionCreated().before(Version.V_2_0_0)) {
184+
} else if (fieldName.equals("excludes")) {
184185
List<Object> values = (List<Object>) fieldNode;
185186
String[] excludes = new String[values.size()];
186187
for (int i = 0; i < excludes.length; i++) {
@@ -197,11 +198,14 @@ public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext
197198

198199
private final boolean enabled;
199200

201+
/** indicates whether the source will always exist and be complete, for use by features like the update API */
202+
private final boolean complete;
203+
200204
private Boolean compress;
201205
private long compressThreshold;
202206

203-
private String[] includes;
204-
private String[] excludes;
207+
private final String[] includes;
208+
private final String[] excludes;
205209

206210
private String format;
207211

@@ -218,23 +222,28 @@ protected SourceFieldMapper(String name, boolean enabled, String format, Boolean
218222
this.enabled = enabled;
219223
this.compress = compress;
220224
this.compressThreshold = compressThreshold;
221-
this.includes = includes;
222-
this.excludes = excludes;
225+
this.includes = includes == null ? Strings.EMPTY_ARRAY : includes;
226+
this.excludes = excludes == null ? Strings.EMPTY_ARRAY : excludes;
223227
this.format = format;
224228
this.formatContentType = format == null ? null : XContentType.fromRestContentType(format);
229+
this.complete = enabled && includes == null && excludes == null;
225230
}
226231

227232
public boolean enabled() {
228-
return this.enabled;
233+
return enabled;
229234
}
230235

231236
public String[] excludes() {
232-
return this.excludes != null ? this.excludes : Strings.EMPTY_ARRAY;
237+
return excludes;
233238

234239
}
235240

236241
public String[] includes() {
237-
return this.includes != null ? this.includes : Strings.EMPTY_ARRAY;
242+
return includes;
243+
}
244+
245+
public boolean isComplete() {
246+
return complete;
238247
}
239248

240249
@Override
@@ -420,19 +429,23 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
420429
@Override
421430
public void merge(Mapper mergeWith, MergeResult mergeResult) throws MergeMappingException {
422431
SourceFieldMapper sourceMergeWith = (SourceFieldMapper) mergeWith;
423-
if (!mergeResult.simulate()) {
432+
if (mergeResult.simulate()) {
433+
if (this.enabled != sourceMergeWith.enabled) {
434+
mergeResult.addConflict("Cannot update enabled setting for [_source]");
435+
}
436+
if (Arrays.equals(includes, sourceMergeWith.includes) == false) {
437+
mergeResult.addConflict("Cannot update includes setting for [_source]");
438+
}
439+
if (Arrays.equals(excludes, sourceMergeWith.excludes) == false) {
440+
mergeResult.addConflict("Cannot update excludes setting for [_source]");
441+
}
442+
} else {
424443
if (sourceMergeWith.compress != null) {
425444
this.compress = sourceMergeWith.compress;
426445
}
427446
if (sourceMergeWith.compressThreshold != -1) {
428447
this.compressThreshold = sourceMergeWith.compressThreshold;
429448
}
430-
if (sourceMergeWith.includes != null) {
431-
this.includes = sourceMergeWith.includes;
432-
}
433-
if (sourceMergeWith.excludes != null) {
434-
this.excludes = sourceMergeWith.excludes;
435-
}
436449
}
437450
}
438451
}

‎src/main/java/org/elasticsearch/search/highlight/HighlightPhase.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -86,8 +86,8 @@ public void hitExecute(SearchContext context, HitContext hitContext) {
8686

8787
if (context.highlight().forceSource(field)) {
8888
SourceFieldMapper sourceFieldMapper = context.mapperService().documentMapper(hitContext.hit().type()).sourceMapper();
89-
if (!sourceFieldMapper.enabled()) {
90-
throw new IllegalArgumentException("source is forced for fields " + fieldNamesToHighlight + " but type [" + hitContext.hit().type() + "] has disabled _source");
89+
if (!sourceFieldMapper.isComplete()) {
90+
throw new IllegalArgumentException("source is forced for fields " + fieldNamesToHighlight + " but type [" + hitContext.hit().type() + "] has incomplete _source");
9191
}
9292
}
9393

‎src/test/java/org/elasticsearch/index/mapper/source/DefaultSourceMappingTests.java‎

Lines changed: 103 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -35,14 +35,15 @@
3535
import org.elasticsearch.test.ElasticsearchSingleNodeTest;
3636
import org.junit.Test;
3737

38+
import java.io.IOException;
39+
import java.util.ArrayList;
3840
import java.util.Arrays;
3941
import java.util.List;
4042
import java.util.Map;
4143

4244
import static org.hamcrest.Matchers.*;
4345

4446
public class DefaultSourceMappingTests extends ElasticsearchSingleNodeTest {
45-
Settings backcompatSettings = ImmutableSettings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_1_4_2.id).build();
4647

4748
public void testNoFormat() throws Exception {
4849
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
@@ -80,8 +81,8 @@ public void testJsonFormat() throws Exception {
8081

8182
documentMapper = parser.parse(mapping);
8283
doc = documentMapper.parse("type", "1", XContentFactory.smileBuilder().startObject()
83-
.field("field", "value")
84-
.endObject().bytes());
84+
.field("field", "value")
85+
.endObject().bytes());
8586

8687
assertThat(XContentFactory.xContentType(doc.source()), equalTo(XContentType.JSON));
8788
}
@@ -91,6 +92,7 @@ public void testJsonFormatCompressedBackcompat() throws Exception {
9192
.startObject("_source").field("format", "json").field("compress", true).endObject()
9293
.endObject().endObject().string();
9394

95+
Settings backcompatSettings = ImmutableSettings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_1_4_2.id).build();
9496
DocumentMapperParser parser = createIndex("test", backcompatSettings).mapperService().documentMapperParser();
9597
DocumentMapper documentMapper = parser.parse(mapping);
9698
ParsedDocument doc = documentMapper.parse("type", "1", XContentFactory.jsonBuilder().startObject()
@@ -111,19 +113,12 @@ public void testJsonFormatCompressedBackcompat() throws Exception {
111113
assertThat(XContentFactory.xContentType(uncompressed), equalTo(XContentType.JSON));
112114
}
113115

114-
public void testIncludesBackcompat() throws Exception {
116+
public void testIncludes() throws Exception {
115117
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
116118
.startObject("_source").field("includes", new String[]{"path1*"}).endObject()
117119
.endObject().endObject().string();
118120

119-
try {
120-
createIndex("testbad").mapperService().documentMapperParser().parse(mapping);
121-
fail("includes should not be allowed");
122-
} catch (MapperParsingException e) {
123-
assertTrue(e.getMessage().contains("unsupported parameters"));
124-
}
125-
126-
DocumentMapper documentMapper = createIndex("test", backcompatSettings).mapperService().documentMapperParser().parse(mapping);
121+
DocumentMapper documentMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping);
127122

128123
ParsedDocument doc = documentMapper.parse("type", "1", XContentFactory.jsonBuilder().startObject()
129124
.startObject("path1").field("field1", "value1").endObject()
@@ -136,19 +131,12 @@ public void testIncludesBackcompat() throws Exception {
136131
assertThat(sourceAsMap.containsKey("path2"), equalTo(false));
137132
}
138133

139-
public void testExcludesBackcompat() throws Exception {
134+
public void testExcludes() throws Exception {
140135
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
141136
.startObject("_source").field("excludes", new String[]{"path1*"}).endObject()
142137
.endObject().endObject().string();
143138

144-
try {
145-
createIndex("testbad").mapperService().documentMapperParser().parse(mapping);
146-
fail("excludes should not be allowed");
147-
} catch (MapperParsingException e) {
148-
assertTrue(e.getMessage().contains("unsupported parameters"));
149-
}
150-
151-
DocumentMapper documentMapper = createIndex("test", backcompatSettings).mapperService().documentMapperParser().parse(mapping);
139+
DocumentMapper documentMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping);
152140

153141
ParsedDocument doc = documentMapper.parse("type", "1", XContentFactory.jsonBuilder().startObject()
154142
.startObject("path1").field("field1", "value1").endObject()
@@ -161,12 +149,12 @@ public void testExcludesBackcompat() throws Exception {
161149
assertThat(sourceAsMap.containsKey("path2"), equalTo(true));
162150
}
163151

164-
public void testDefaultMappingAndNoMappingBackcompat() throws Exception {
152+
public void testDefaultMappingAndNoMapping() throws Exception {
165153
String defaultMapping = XContentFactory.jsonBuilder().startObject().startObject(MapperService.DEFAULT_MAPPING)
166154
.startObject("_source").field("enabled", false).endObject()
167155
.endObject().endObject().string();
168156

169-
DocumentMapperParser parser = createIndex("test", backcompatSettings).mapperService().documentMapperParser();
157+
DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser();
170158
DocumentMapper mapper = parser.parse("my_type", null, defaultMapping);
171159
assertThat(mapper.type(), equalTo("my_type"));
172160
assertThat(mapper.sourceMapper().enabled(), equalTo(false));
@@ -189,7 +177,7 @@ public void testDefaultMappingAndNoMappingBackcompat() throws Exception {
189177
}
190178
}
191179

192-
public void testDefaultMappingAndWithMappingOverrideBackcompat() throws Exception {
180+
public void testDefaultMappingAndWithMappingOverride() throws Exception {
193181
String defaultMapping = XContentFactory.jsonBuilder().startObject().startObject(MapperService.DEFAULT_MAPPING)
194182
.startObject("_source").field("enabled", false).endObject()
195183
.endObject().endObject().string();
@@ -198,30 +186,30 @@ public void testDefaultMappingAndWithMappingOverrideBackcompat() throws Exceptio
198186
.startObject("_source").field("enabled", true).endObject()
199187
.endObject().endObject().string();
200188

201-
DocumentMapper mapper = createIndex("test", backcompatSettings).mapperService().documentMapperParser().parse("my_type", mapping, defaultMapping);
189+
DocumentMapper mapper = createIndex("test").mapperService().documentMapperParser().parse("my_type", mapping, defaultMapping);
202190
assertThat(mapper.type(), equalTo("my_type"));
203191
assertThat(mapper.sourceMapper().enabled(), equalTo(true));
204192
}
205193

206-
public void testDefaultMappingAndNoMappingWithMapperServiceBackcompat() throws Exception {
194+
public void testDefaultMappingAndNoMappingWithMapperService() throws Exception {
207195
String defaultMapping = XContentFactory.jsonBuilder().startObject().startObject(MapperService.DEFAULT_MAPPING)
208196
.startObject("_source").field("enabled", false).endObject()
209197
.endObject().endObject().string();
210198

211-
MapperService mapperService = createIndex("test", backcompatSettings).mapperService();
199+
MapperService mapperService = createIndex("test").mapperService();
212200
mapperService.merge(MapperService.DEFAULT_MAPPING, new CompressedString(defaultMapping), true);
213201

214202
DocumentMapper mapper = mapperService.documentMapperWithAutoCreate("my_type").v1();
215203
assertThat(mapper.type(), equalTo("my_type"));
216204
assertThat(mapper.sourceMapper().enabled(), equalTo(false));
217205
}
218206

219-
public void testDefaultMappingAndWithMappingOverrideWithMapperServiceBackcompat() throws Exception {
207+
public void testDefaultMappingAndWithMappingOverrideWithMapperService() throws Exception {
220208
String defaultMapping = XContentFactory.jsonBuilder().startObject().startObject(MapperService.DEFAULT_MAPPING)
221209
.startObject("_source").field("enabled", false).endObject()
222210
.endObject().endObject().string();
223211

224-
MapperService mapperService = createIndex("test", backcompatSettings).mapperService();
212+
MapperService mapperService = createIndex("test").mapperService();
225213
mapperService.merge(MapperService.DEFAULT_MAPPING, new CompressedString(defaultMapping), true);
226214

227215
String mapping = XContentFactory.jsonBuilder().startObject().startObject("my_type")
@@ -233,4 +221,90 @@ public void testDefaultMappingAndWithMappingOverrideWithMapperServiceBackcompat(
233221
assertThat(mapper.type(), equalTo("my_type"));
234222
assertThat(mapper.sourceMapper().enabled(), equalTo(true));
235223
}
224+
225+
void assertConflicts(String mapping1, String mapping2, DocumentMapperParser parser, String... conflicts) throws IOException {
226+
DocumentMapper docMapper = parser.parse(mapping1);
227+
docMapper.refreshSource();
228+
docMapper = parser.parse(docMapper.mappingSource().string());
229+
MergeResult mergeResult = docMapper.merge(parser.parse(mapping2).mapping(), true);
230+
231+
List<String> expectedConflicts = new ArrayList<>(Arrays.asList(conflicts));
232+
for (String conflict : mergeResult.buildConflicts()) {
233+
assertTrue("found unexpected conflict [" + conflict + "]", expectedConflicts.remove(conflict));
234+
}
235+
assertTrue("missing conflicts: " + Arrays.toString(expectedConflicts.toArray()), expectedConflicts.isEmpty());
236+
}
237+
238+
public void testEnabledNotUpdateable() throws Exception {
239+
DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser();
240+
// using default of true
241+
String mapping1 = XContentFactory.jsonBuilder().startObject().startObject("type").endObject().endObject().string();
242+
String mapping2 = XContentFactory.jsonBuilder().startObject().startObject("type")
243+
.startObject("_source").field("enabled", false).endObject()
244+
.endObject().endObject().string();
245+
assertConflicts(mapping1, mapping2, parser, "Cannot update enabled setting for [_source]");
246+
247+
// not changing is ok
248+
String mapping3 = XContentFactory.jsonBuilder().startObject().startObject("type")
249+
.startObject("_source").field("enabled", true).endObject()
250+
.endObject().endObject().string();
251+
assertConflicts(mapping1, mapping3, parser);
252+
}
253+
254+
public void testIncludesNotUpdateable() throws Exception {
255+
DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser();
256+
String defaultMapping = XContentFactory.jsonBuilder().startObject().startObject("type").endObject().endObject().string();
257+
String mapping1 = XContentFactory.jsonBuilder().startObject().startObject("type")
258+
.startObject("_source").array("includes", "foo.*").endObject()
259+
.endObject().endObject().string();
260+
assertConflicts(defaultMapping, mapping1, parser, "Cannot update includes setting for [_source]");
261+
assertConflicts(mapping1, defaultMapping, parser, "Cannot update includes setting for [_source]");
262+
263+
String mapping2 = XContentFactory.jsonBuilder().startObject().startObject("type")
264+
.startObject("_source").array("includes", "foo.*", "bar.*").endObject()
265+
.endObject().endObject().string();
266+
assertConflicts(mapping1, mapping2, parser, "Cannot update includes setting for [_source]");
267+
268+
// not changing is ok
269+
assertConflicts(mapping1, mapping1, parser);
270+
}
271+
272+
public void testExcludesNotUpdateable() throws Exception {
273+
DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser();
274+
String defaultMapping = XContentFactory.jsonBuilder().startObject().startObject("type").endObject().endObject().string();
275+
String mapping1 = XContentFactory.jsonBuilder().startObject().startObject("type")
276+
.startObject("_source").array("excludes", "foo.*").endObject()
277+
.endObject().endObject().string();
278+
assertConflicts(defaultMapping, mapping1, parser, "Cannot update excludes setting for [_source]");
279+
assertConflicts(mapping1, defaultMapping, parser, "Cannot update excludes setting for [_source]");
280+
281+
String mapping2 = XContentFactory.jsonBuilder().startObject().startObject("type")
282+
.startObject("_source").array("excludes", "foo.*", "bar.*").endObject()
283+
.endObject().endObject().string();
284+
assertConflicts(mapping1, mapping2, parser, "Cannot update excludes setting for [_source]");
285+
286+
// not changing is ok
287+
assertConflicts(mapping1, mapping1, parser);
288+
}
289+
290+
public void testComplete() throws Exception {
291+
DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser();
292+
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").endObject().endObject().string();
293+
assertTrue(parser.parse(mapping).sourceMapper().isComplete());
294+
295+
mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
296+
.startObject("_source").field("enabled", false).endObject()
297+
.endObject().endObject().string();
298+
assertFalse(parser.parse(mapping).sourceMapper().isComplete());
299+
300+
mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
301+
.startObject("_source").array("includes", "foo.*").endObject()
302+
.endObject().endObject().string();
303+
assertFalse(parser.parse(mapping).sourceMapper().isComplete());
304+
305+
mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
306+
.startObject("_source").array("excludes", "foo.*").endObject()
307+
.endObject().endObject().string();
308+
assertFalse(parser.parse(mapping).sourceMapper().isComplete());
309+
}
236310
}

0 commit comments

Comments
 (0)