Skip to content

Commit f35f01a

Browse files
rjernstrjernst
committed
Mappings: Remove includes and excludes from _source
Regardless of the outcome of #8142, we should at least enforce that when _source is enabled, it is sufficient to reindex. This change removes the excludes and includes settings, since these modify the source, causing us to lose the ability to reindex some fields.
1 parent 91e2bb1 commit f35f01a

File tree

6 files changed

+66
-142
lines changed

6 files changed

+66
-142
lines changed

‎src/main/java/org/elasticsearch/index/mapper/internal/SourceFieldMapper.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
import org.apache.lucene.index.IndexOptions;
2727
import org.apache.lucene.util.BytesRef;
2828
import org.elasticsearch.ElasticsearchParseException;
29+
import org.elasticsearch.Version;
2930
import org.elasticsearch.common.Strings;
3031
import org.elasticsearch.common.bytes.BytesArray;
3132
import org.elasticsearch.common.bytes.BytesReference;
@@ -166,15 +167,15 @@ public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext
166167
} else if ("format".equals(fieldName)) {
167168
builder.format(nodeStringValue(fieldNode, null));
168169
iterator.remove();
169-
} else if (fieldName.equals("includes")) {
170+
} else if (fieldName.equals("includes") && parserContext.indexVersionCreated().before(Version.V_2_0_0)) {
170171
List<Object> values = (List<Object>) fieldNode;
171172
String[] includes = new String[values.size()];
172173
for (int i = 0; i < includes.length; i++) {
173174
includes[i] = values.get(i).toString();
174175
}
175176
builder.includes(includes);
176177
iterator.remove();
177-
} else if (fieldName.equals("excludes")) {
178+
} else if (fieldName.equals("excludes") && parserContext.indexVersionCreated().before(Version.V_2_0_0)) {
178179
List<Object> values = (List<Object>) fieldNode;
179180
String[] excludes = new String[values.size()];
180181
for (int i = 0; i < excludes.length; i++) {

‎src/test/java/org/elasticsearch/get/GetActionTests.java‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,11 +21,13 @@
2121

2222
import org.elasticsearch.ElasticsearchException;
2323
import org.elasticsearch.ElasticsearchIllegalArgumentException;
24+
import org.elasticsearch.Version;
2425
import org.elasticsearch.action.ShardOperationFailedException;
2526
import org.elasticsearch.action.admin.indices.alias.Alias;
2627
import org.elasticsearch.action.admin.indices.flush.FlushResponse;
2728
import org.elasticsearch.action.delete.DeleteResponse;
2829
import org.elasticsearch.action.get.*;
30+
import org.elasticsearch.cluster.metadata.IndexMetaData;
2931
import org.elasticsearch.common.Base64;
3032
import org.elasticsearch.common.Nullable;
3133
import org.elasticsearch.common.Strings;
@@ -412,7 +414,7 @@ public void testThatGetFromTranslogShouldWorkWithExclude() throws Exception {
412414

413415
assertAcked(prepareCreate(index)
414416
.addMapping(type, mapping)
415-
.setSettings(ImmutableSettings.settingsBuilder().put("index.refresh_interval", -1)));
417+
.setSettings("index.refresh_interval", -1, IndexMetaData.SETTING_VERSION_CREATED, Version.V_1_4_2.id));
416418

417419
client().prepareIndex(index, type, "1")
418420
.setSource(jsonBuilder().startObject().field("field", "1", "2").field("excluded", "should not be seen").endObject())
@@ -446,7 +448,7 @@ public void testThatGetFromTranslogShouldWorkWithInclude() throws Exception {
446448

447449
assertAcked(prepareCreate(index)
448450
.addMapping(type, mapping)
449-
.setSettings(ImmutableSettings.settingsBuilder().put("index.refresh_interval", -1)));
451+
.setSettings("index.refresh_interval", -1, IndexMetaData.SETTING_VERSION_CREATED, Version.V_1_4_2.id));
450452

451453
client().prepareIndex(index, type, "1")
452454
.setSource(jsonBuilder().startObject().field("field", "1", "2").field("included", "should be seen").endObject())
@@ -482,7 +484,7 @@ public void testThatGetFromTranslogShouldWorkWithIncludeExcludeAndFields() throw
482484

483485
assertAcked(prepareCreate(index)
484486
.addMapping(type, mapping)
485-
.setSettings(ImmutableSettings.settingsBuilder().put("index.refresh_interval", -1)));
487+
.setSettings("index.refresh_interval", -1, IndexMetaData.SETTING_VERSION_CREATED, Version.V_1_4_2.id));
486488

487489
client().prepareIndex(index, type, "1")
488490
.setSource(jsonBuilder().startObject()

‎src/test/java/org/elasticsearch/index/mapper/source/DefaultSourceMappingTests.java‎

Lines changed: 44 additions & 80 deletions
Original file line numberDiff line numberDiff line change
@@ -20,10 +20,13 @@
2020
package org.elasticsearch.index.mapper.source;
2121

2222
import org.apache.lucene.index.IndexableField;
23+
import org.elasticsearch.Version;
24+
import org.elasticsearch.cluster.metadata.IndexMetaData;
2325
import org.elasticsearch.common.bytes.BytesArray;
2426
import org.elasticsearch.common.compress.CompressedString;
2527
import org.elasticsearch.common.compress.CompressorFactory;
2628
import org.elasticsearch.common.settings.ImmutableSettings;
29+
import org.elasticsearch.common.settings.Settings;
2730
import org.elasticsearch.common.xcontent.XContentBuilder;
2831
import org.elasticsearch.common.xcontent.XContentFactory;
2932
import org.elasticsearch.common.xcontent.XContentType;
@@ -38,12 +41,8 @@
3841

3942
import static org.hamcrest.Matchers.*;
4043

41-
/**
42-
*
43-
*/
4444
public class DefaultSourceMappingTests extends ElasticsearchSingleNodeTest {
4545

46-
@Test
4746
public void testNoFormat() throws Exception {
4847
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
4948
.startObject("_source").endObject()
@@ -65,7 +64,6 @@ public void testNoFormat() throws Exception {
6564
assertThat(XContentFactory.xContentType(doc.source()), equalTo(XContentType.SMILE));
6665
}
6766

68-
@Test
6967
public void testJsonFormat() throws Exception {
7068
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
7169
.startObject("_source").field("format", "json").endObject()
@@ -87,7 +85,6 @@ public void testJsonFormat() throws Exception {
8785
assertThat(XContentFactory.xContentType(doc.source()), equalTo(XContentType.JSON));
8886
}
8987

90-
@Test
9188
public void testJsonFormatCompressed() throws Exception {
9289
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
9390
.startObject("_source").field("format", "json").field("compress", true).endObject()
@@ -113,26 +110,58 @@ public void testJsonFormatCompressed() throws Exception {
113110
assertThat(XContentFactory.xContentType(uncompressed), equalTo(XContentType.JSON));
114111
}
115112

116-
@Test
117-
public void testIncludeExclude() throws Exception {
113+
public void testIncludesBackcompat() throws Exception {
118114
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
119-
.startObject("_source").field("includes", new String[]{"path1*"}).endObject()
120-
.endObject().endObject().string();
115+
.startObject("_source").field("includes", new String[]{"path1*"}).endObject()
116+
.endObject().endObject().string();
117+
118+
try {
119+
createIndex("testbad").mapperService().documentMapperParser().parse(mapping);
120+
fail("includes should not be allowed");
121+
} catch (MapperParsingException e) {
122+
assertTrue(e.getMessage().contains("unsupported parameters"));
123+
}
121124

122-
DocumentMapper documentMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping);
125+
Settings settings = ImmutableSettings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_1_4_2.id).build();
126+
DocumentMapper documentMapper = createIndex("test", settings).mapperService().documentMapperParser().parse(mapping);
123127

124128
ParsedDocument doc = documentMapper.parse("type", "1", XContentFactory.jsonBuilder().startObject()
125-
.startObject("path1").field("field1", "value1").endObject()
126-
.startObject("path2").field("field2", "value2").endObject()
127-
.endObject().bytes());
129+
.startObject("path1").field("field1", "value1").endObject()
130+
.startObject("path2").field("field2", "value2").endObject()
131+
.endObject().bytes());
128132

129133
IndexableField sourceField = doc.rootDoc().getField("_source");
130134
Map<String, Object> sourceAsMap = XContentFactory.xContent(XContentType.JSON).createParser(new BytesArray(sourceField.binaryValue())).mapAndClose();
131135
assertThat(sourceAsMap.containsKey("path1"), equalTo(true));
132136
assertThat(sourceAsMap.containsKey("path2"), equalTo(false));
133137
}
134138

135-
@Test
139+
public void testExcludesBackcompat() throws Exception {
140+
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
141+
.startObject("_source").field("excludes", new String[]{"path1*"}).endObject()
142+
.endObject().endObject().string();
143+
144+
try {
145+
createIndex("testbad").mapperService().documentMapperParser().parse(mapping);
146+
fail("excludes should not be allowed");
147+
} catch (MapperParsingException e) {
148+
assertTrue(e.getMessage().contains("unsupported parameters"));
149+
}
150+
151+
Settings settings = ImmutableSettings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_1_4_2.id).build();
152+
DocumentMapper documentMapper = createIndex("test", settings).mapperService().documentMapperParser().parse(mapping);
153+
154+
ParsedDocument doc = documentMapper.parse("type", "1", XContentFactory.jsonBuilder().startObject()
155+
.startObject("path1").field("field1", "value1").endObject()
156+
.startObject("path2").field("field2", "value2").endObject()
157+
.endObject().bytes());
158+
159+
IndexableField sourceField = doc.rootDoc().getField("_source");
160+
Map<String, Object> sourceAsMap = XContentFactory.xContent(XContentType.JSON).createParser(new BytesArray(sourceField.binaryValue())).mapAndClose();
161+
assertThat(sourceAsMap.containsKey("path1"), equalTo(false));
162+
assertThat(sourceAsMap.containsKey("path2"), equalTo(true));
163+
}
164+
136165
public void testDefaultMappingAndNoMapping() throws Exception {
137166
String defaultMapping = XContentFactory.jsonBuilder().startObject().startObject(MapperService.DEFAULT_MAPPING)
138167
.startObject("_source").field("enabled", false).endObject()
@@ -161,7 +190,6 @@ public void testDefaultMappingAndNoMapping() throws Exception {
161190
}
162191
}
163192

164-
@Test
165193
public void testDefaultMappingAndWithMappingOverride() throws Exception {
166194
String defaultMapping = XContentFactory.jsonBuilder().startObject().startObject(MapperService.DEFAULT_MAPPING)
167195
.startObject("_source").field("enabled", false).endObject()
@@ -176,7 +204,6 @@ public void testDefaultMappingAndWithMappingOverride() throws Exception {
176204
assertThat(mapper.sourceMapper().enabled(), equalTo(true));
177205
}
178206

179-
@Test
180207
public void testDefaultMappingAndNoMappingWithMapperService() throws Exception {
181208
String defaultMapping = XContentFactory.jsonBuilder().startObject().startObject(MapperService.DEFAULT_MAPPING)
182209
.startObject("_source").field("enabled", false).endObject()
@@ -190,7 +217,6 @@ public void testDefaultMappingAndNoMappingWithMapperService() throws Exception {
190217
assertThat(mapper.sourceMapper().enabled(), equalTo(false));
191218
}
192219

193-
@Test
194220
public void testDefaultMappingAndWithMappingOverrideWithMapperService() throws Exception {
195221
String defaultMapping = XContentFactory.jsonBuilder().startObject().startObject(MapperService.DEFAULT_MAPPING)
196222
.startObject("_source").field("enabled", false).endObject()
@@ -208,66 +234,4 @@ public void testDefaultMappingAndWithMappingOverrideWithMapperService() throws E
208234
assertThat(mapper.type(), equalTo("my_type"));
209235
assertThat(mapper.sourceMapper().enabled(), equalTo(true));
210236
}
211-
212-
@Test
213-
public void testParsingWithDefaultAppliedAndNotApplied() throws Exception {
214-
String defaultMapping = XContentFactory.jsonBuilder().startObject().startObject(MapperService.DEFAULT_MAPPING)
215-
.startObject("_source").array("includes", "default_field_path.").endObject()
216-
.endObject().endObject().string();
217-
218-
MapperService mapperService = createIndex("test").mapperService();
219-
mapperService.merge(MapperService.DEFAULT_MAPPING, new CompressedString(defaultMapping), true);
220-
221-
String mapping = XContentFactory.jsonBuilder().startObject().startObject("my_type")
222-
.startObject("_source").array("includes", "custom_field_path.").endObject()
223-
.endObject().endObject().string();
224-
mapperService.merge("my_type", new CompressedString(mapping), true);
225-
DocumentMapper mapper = mapperService.documentMapper("my_type");
226-
assertThat(mapper.type(), equalTo("my_type"));
227-
assertThat(mapper.sourceMapper().includes().length, equalTo(2));
228-
assertThat(mapper.sourceMapper().includes(), hasItemInArray("default_field_path."));
229-
assertThat(mapper.sourceMapper().includes(), hasItemInArray("custom_field_path."));
230-
231-
mapping = XContentFactory.jsonBuilder().startObject().startObject("my_type")
232-
.startObject("properties").startObject("text").field("type", "string").endObject().endObject()
233-
.endObject().endObject().string();
234-
mapperService.merge("my_type", new CompressedString(mapping), false);
235-
mapper = mapperService.documentMapper("my_type");
236-
assertThat(mapper.type(), equalTo("my_type"));
237-
assertThat(mapper.sourceMapper().includes(), hasItemInArray("default_field_path."));
238-
assertThat(mapper.sourceMapper().includes(), hasItemInArray("custom_field_path."));
239-
assertThat(mapper.sourceMapper().includes().length, equalTo(2));
240-
}
241-
242-
public void testDefaultNotAppliedOnUpdate() throws Exception {
243-
XContentBuilder defaultMapping = XContentFactory.jsonBuilder().startObject().startObject(MapperService.DEFAULT_MAPPING)
244-
.startObject("_source").array("includes", "default_field_path.").endObject()
245-
.endObject().endObject();
246-
247-
IndexService indexService = createIndex("test", ImmutableSettings.EMPTY, MapperService.DEFAULT_MAPPING, defaultMapping);
248-
249-
String mapping = XContentFactory.jsonBuilder().startObject().startObject("my_type")
250-
.startObject("_source").array("includes", "custom_field_path.").endObject()
251-
.endObject().endObject().string();
252-
client().admin().indices().preparePutMapping("test").setType("my_type").setSource(mapping).get();
253-
254-
DocumentMapper mapper = indexService.mapperService().documentMapper("my_type");
255-
assertThat(mapper.type(), equalTo("my_type"));
256-
assertThat(mapper.sourceMapper().includes().length, equalTo(2));
257-
List<String> includes = Arrays.asList(mapper.sourceMapper().includes());
258-
assertThat("default_field_path.", isIn(includes));
259-
assertThat("custom_field_path.", isIn(includes));
260-
261-
mapping = XContentFactory.jsonBuilder().startObject().startObject("my_type")
262-
.startObject("properties").startObject("text").field("type", "string").endObject().endObject()
263-
.endObject().endObject().string();
264-
client().admin().indices().preparePutMapping("test").setType("my_type").setSource(mapping).get();
265-
266-
mapper = indexService.mapperService().documentMapper("my_type");
267-
assertThat(mapper.type(), equalTo("my_type"));
268-
includes = Arrays.asList(mapper.sourceMapper().includes());
269-
assertThat("default_field_path.", isIn(includes));
270-
assertThat("custom_field_path.", isIn(includes));
271-
assertThat(mapper.sourceMapper().includes().length, equalTo(2));
272-
}
273237
}

‎src/test/java/org/elasticsearch/indices/mapping/UpdateMappingTests.java‎ renamed to ‎src/test/java/org/elasticsearch/indices/mapping/UpdateMappingIntegrationTests.java‎

Lines changed: 10 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -21,13 +21,15 @@
2121

2222
import com.google.common.collect.Lists;
2323

24+
import org.elasticsearch.Version;
2425
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
2526
import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse;
2627
import org.elasticsearch.action.admin.indices.refresh.RefreshResponse;
2728
import org.elasticsearch.action.count.CountResponse;
2829
import org.elasticsearch.action.get.GetResponse;
2930
import org.elasticsearch.action.index.IndexRequestBuilder;
3031
import org.elasticsearch.client.Client;
32+
import org.elasticsearch.cluster.metadata.IndexMetaData;
3133
import org.elasticsearch.cluster.metadata.MappingMetaData;
3234
import org.elasticsearch.common.Priority;
3335
import org.elasticsearch.common.collect.ImmutableOpenMap;
@@ -54,7 +56,7 @@
5456
import static org.hamcrest.Matchers.*;
5557

5658
@ClusterScope(randomDynamicTemplates = false)
57-
public class UpdateMappingTests extends ElasticsearchIntegrationTest {
59+
public class UpdateMappingIntegrationTests extends ElasticsearchIntegrationTest {
5860

5961
@Test
6062
public void dynamicUpdates() throws Exception {
@@ -213,13 +215,13 @@ public void updateMappingNoChanges() throws Exception {
213215

214216
@SuppressWarnings("unchecked")
215217
@Test
216-
public void updateIncludeExclude() throws Exception {
217-
assertAcked(prepareCreate("test").addMapping("type",
218-
jsonBuilder().startObject().startObject("type").startObject("properties")
219-
.startObject("normal").field("type", "long").endObject()
220-
.startObject("exclude").field("type", "long").endObject()
221-
.startObject("include").field("type", "long").endObject()
222-
.endObject().endObject().endObject()));
218+
public void updateIncludeExcludeBackcompat() throws Exception {
219+
assertAcked(prepareCreate("test").setSettings(IndexMetaData.SETTING_VERSION_CREATED, Version.V_1_4_2.id)
220+
.addMapping("type", jsonBuilder().startObject().startObject("type").startObject("properties")
221+
.startObject("normal").field("type", "long").endObject()
222+
.startObject("exclude").field("type", "long").endObject()
223+
.startObject("include").field("type", "long").endObject()
224+
.endObject().endObject().endObject()));
223225
ensureGreen(); // make sure that replicas are initialized so the refresh command will work them too
224226

225227
logger.info("Index doc");
@@ -229,7 +231,6 @@ public void updateIncludeExclude() throws Exception {
229231
);
230232
refresh(); // commit it for later testing.
231233

232-
233234
logger.info("Adding exclude settings");
234235
PutMappingResponse putResponse = client().admin().indices().preparePutMapping("test").setType("type").setSource(
235236
JsonXContent.contentBuilder().startObject().startObject("type")
@@ -259,7 +260,6 @@ public void updateIncludeExclude() throws Exception {
259260
assertThat(getResponse.getSource(), not(hasKey("exclude")));
260261
assertThat(getResponse.getSource(), hasKey("include"));
261262

262-
263263
logger.info("Changing mapping to includes");
264264
putResponse = client().admin().indices().preparePutMapping("test").setType("type").setSource(
265265
JsonXContent.contentBuilder().startObject().startObject("type")
@@ -278,7 +278,6 @@ public void updateIncludeExclude() throws Exception {
278278
assertThat((Map<String, Object>) typeMapping.getSourceAsMap().get("_source"), hasKey("excludes"));
279279
assertThat((ArrayList<String>) ((Map<String, Object>) typeMapping.getSourceAsMap().get("_source")).get("excludes"), emptyIterable());
280280

281-
282281
logger.info("Indexing doc yet again");
283282
index("test", "type", "1", JsonXContent.contentBuilder().startObject()
284283
.field("normal", 3).field("exclude", 3).field("include", 3)
@@ -290,7 +289,6 @@ public void updateIncludeExclude() throws Exception {
290289
assertThat(getResponse.getSource(), not(hasKey("exclude")));
291290
assertThat(getResponse.getSource(), hasKey("include"));
292291

293-
294292
logger.info("Adding excludes, but keep includes");
295293
putResponse = client().admin().indices().preparePutMapping("test").setType("type").setSource(
296294
JsonXContent.contentBuilder().startObject().startObject("type")
@@ -308,8 +306,6 @@ public void updateIncludeExclude() throws Exception {
308306
assertThat((Map<String, Object>) typeMapping.getSourceAsMap().get("_source"), hasKey("excludes"));
309307
ArrayList<String> excludes = (ArrayList<String>) ((Map<String, Object>) typeMapping.getSourceAsMap().get("_source")).get("excludes");
310308
assertThat(excludes, contains("*.excludes"));
311-
312-
313309
}
314310

315311
@SuppressWarnings("unchecked")

0 commit comments

Comments
 (0)