Skip to content

Commit 9492dce

Browse files
martijnvghub-cap
martijnvg
and
hub-cap
authored
Update IndexTemplateMetaData to allow unknown fields (#38448)
Updated IndexTemplateMetaData to use ObjectParser. The IndexTemplateMetaData class used old parsing logic and was not resiliant to new fields. This commit updates it to use the ConstructingObjectParser and allow unknown fields. Relates #36938 Co-authored-by: Michael Basnight <[email protected]> Co-authored-by: Martijn van Groningen <[email protected]>
1 parent e1d464b commit 9492dce

File tree

2 files changed

+184
-95
lines changed

2 files changed

+184
-95
lines changed

‎client/rest-high-level/src/main/java/org/elasticsearch/client/indices/IndexTemplateMetaData.java‎

Lines changed: 53 additions & 89 deletions
Original file line numberDiff line numberDiff line change
@@ -18,27 +18,66 @@
1818
*/
1919
package org.elasticsearch.client.indices;
2020

21-
import org.elasticsearch.ElasticsearchParseException;
2221
import org.elasticsearch.cluster.metadata.AliasMetaData;
2322
import org.elasticsearch.cluster.metadata.IndexMetaData;
2423
import org.elasticsearch.cluster.metadata.MappingMetaData;
2524
import org.elasticsearch.common.Nullable;
25+
import org.elasticsearch.common.ParseField;
2626
import org.elasticsearch.common.collect.ImmutableOpenMap;
2727
import org.elasticsearch.common.settings.Settings;
28-
import org.elasticsearch.common.util.set.Sets;
28+
import org.elasticsearch.common.xcontent.ConstructingObjectParser;
2929
import org.elasticsearch.common.xcontent.XContentParser;
3030
import org.elasticsearch.index.mapper.MapperService;
3131

3232
import java.io.IOException;
33-
import java.util.ArrayList;
34-
import java.util.Collections;
33+
import java.util.AbstractMap;
3534
import java.util.List;
3635
import java.util.Map;
3736
import java.util.Objects;
38-
import java.util.Set;
37+
import java.util.stream.Collectors;
38+
39+
import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg;
3940

4041
public class IndexTemplateMetaData {
4142

43+
@SuppressWarnings("unchecked")
44+
private static final ConstructingObjectParser<IndexTemplateMetaData, String> PARSER = new ConstructingObjectParser<>(
45+
"IndexTemplateMetaData", true, (a, name) -> {
46+
List<Map.Entry<String, AliasMetaData>> alias = (List<Map.Entry<String, AliasMetaData>>) a[5];
47+
ImmutableOpenMap<String, AliasMetaData> aliasMap =
48+
new ImmutableOpenMap.Builder<String, AliasMetaData>()
49+
.putAll(alias.stream().collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)))
50+
.build();
51+
return new IndexTemplateMetaData(
52+
name,
53+
(Integer) a[0],
54+
(Integer) a[1],
55+
(List<String>) a[2],
56+
(Settings) a[3],
57+
(MappingMetaData) a[4],
58+
aliasMap);
59+
});
60+
61+
static {
62+
PARSER.declareInt(optionalConstructorArg(), new ParseField("order"));
63+
PARSER.declareInt(optionalConstructorArg(), new ParseField("version"));
64+
PARSER.declareStringArray(optionalConstructorArg(), new ParseField("index_patterns"));
65+
PARSER.declareObject(optionalConstructorArg(), (p, c) -> {
66+
Settings.Builder templateSettingsBuilder = Settings.builder();
67+
templateSettingsBuilder.put(Settings.fromXContent(p));
68+
templateSettingsBuilder.normalizePrefix(IndexMetaData.INDEX_SETTING_PREFIX);
69+
return templateSettingsBuilder.build();
70+
}, new ParseField("settings"));
71+
PARSER.declareObject(optionalConstructorArg(), (p, c) -> {
72+
Map<String, Object> mapping = p.map();
73+
if (mapping.isEmpty()) {
74+
return null;
75+
}
76+
return new MappingMetaData(MapperService.SINGLE_MAPPING_NAME, mapping);
77+
}, new ParseField("mappings"));
78+
PARSER.declareNamedObjects(optionalConstructorArg(),
79+
(p, c, name) -> new AbstractMap.SimpleEntry<>(name, AliasMetaData.Builder.fromXContent(p)), new ParseField("aliases"));
80+
}
4281

4382
private final String name;
4483

@@ -125,28 +164,23 @@ public static Builder builder(String name) {
125164
public boolean equals(Object o) {
126165
if (this == o) return true;
127166
if (o == null || getClass() != o.getClass()) return false;
128-
129167
IndexTemplateMetaData that = (IndexTemplateMetaData) o;
130-
131-
if (order != that.order) return false;
132-
if (!Objects.equals(mappings, that.mappings)) return false;
133-
if (!name.equals(that.name)) return false;
134-
if (!settings.equals(that.settings)) return false;
135-
if (!patterns.equals(that.patterns)) return false;
136-
137-
return Objects.equals(version, that.version);
168+
return order == that.order &&
169+
Objects.equals(name, that.name) &&
170+
Objects.equals(version, that.version) &&
171+
Objects.equals(patterns, that.patterns) &&
172+
Objects.equals(settings, that.settings) &&
173+
Objects.equals(mappings, that.mappings) &&
174+
Objects.equals(aliases, that.aliases);
138175
}
139176

140177
@Override
141178
public int hashCode() {
142-
return Objects.hash(name, order, version, patterns, settings, mappings);
179+
return Objects.hash(name, order, version, patterns, settings, mappings, aliases);
143180
}
144181

145182
public static class Builder {
146183

147-
private static final Set<String> VALID_FIELDS = Sets.newHashSet(
148-
"template", "order", "mappings", "settings", "index_patterns", "aliases", "version");
149-
150184
private String name;
151185

152186
private int order;
@@ -193,7 +227,6 @@ public Builder patterns(List<String> indexPatterns) {
193227
return this;
194228
}
195229

196-
197230
public Builder settings(Settings.Builder settings) {
198231
this.settings = settings.build();
199232
return this;
@@ -225,76 +258,7 @@ public IndexTemplateMetaData build() {
225258

226259

227260
public static IndexTemplateMetaData fromXContent(XContentParser parser, String templateName) throws IOException {
228-
Builder builder = new Builder(templateName);
229-
230-
String currentFieldName = skipTemplateName(parser);
231-
XContentParser.Token token;
232-
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
233-
if (token == XContentParser.Token.FIELD_NAME) {
234-
currentFieldName = parser.currentName();
235-
} else if (token == XContentParser.Token.START_OBJECT) {
236-
if ("settings".equals(currentFieldName)) {
237-
Settings.Builder templateSettingsBuilder = Settings.builder();
238-
templateSettingsBuilder.put(Settings.fromXContent(parser));
239-
templateSettingsBuilder.normalizePrefix(IndexMetaData.INDEX_SETTING_PREFIX);
240-
builder.settings(templateSettingsBuilder.build());
241-
} else if ("mappings".equals(currentFieldName)) {
242-
Map<String, Object> mapping = parser.map();
243-
if (mapping.isEmpty() == false) {
244-
MappingMetaData md = new MappingMetaData(MapperService.SINGLE_MAPPING_NAME, mapping);
245-
builder.mapping(md);
246-
}
247-
} else if ("aliases".equals(currentFieldName)) {
248-
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
249-
builder.putAlias(AliasMetaData.Builder.fromXContent(parser));
250-
}
251-
} else {
252-
throw new ElasticsearchParseException("unknown key [{}] for index template", currentFieldName);
253-
}
254-
} else if (token == XContentParser.Token.START_ARRAY) {
255-
if ("mappings".equals(currentFieldName)) {
256-
// The server-side IndexTemplateMetaData has toXContent impl that can return mappings
257-
// in an array but also a comment saying this never happens with typeless APIs.
258-
throw new ElasticsearchParseException("Invalid response format - "
259-
+ "mappings are not expected to be returned in an array", currentFieldName);
260-
} else if ("index_patterns".equals(currentFieldName)) {
261-
List<String> index_patterns = new ArrayList<>();
262-
while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
263-
index_patterns.add(parser.text());
264-
}
265-
builder.patterns(index_patterns);
266-
}
267-
} else if (token.isValue()) {
268-
// Prior to 5.1.0, elasticsearch only supported a single index pattern called `template` (#21009)
269-
if("template".equals(currentFieldName)) {
270-
builder.patterns(Collections.singletonList(parser.text()));
271-
} else if ("order".equals(currentFieldName)) {
272-
builder.order(parser.intValue());
273-
} else if ("version".equals(currentFieldName)) {
274-
builder.version(parser.intValue());
275-
}
276-
}
277-
}
278-
return builder.build();
279-
}
280-
281-
private static String skipTemplateName(XContentParser parser) throws IOException {
282-
XContentParser.Token token = parser.nextToken();
283-
if (token == XContentParser.Token.START_OBJECT) {
284-
token = parser.nextToken();
285-
if (token == XContentParser.Token.FIELD_NAME) {
286-
String currentFieldName = parser.currentName();
287-
if (VALID_FIELDS.contains(currentFieldName)) {
288-
return currentFieldName;
289-
} else {
290-
// we just hit the template name, which should be ignored and we move on
291-
parser.nextToken();
292-
}
293-
}
294-
}
295-
296-
return null;
261+
return PARSER.parse(parser, templateName);
297262
}
298263
}
299-
300264
}

‎client/rest-high-level/src/test/java/org/elasticsearch/client/indices/GetIndexTemplatesResponseTests.java‎

Lines changed: 131 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -22,24 +22,37 @@
2222
import org.elasticsearch.cluster.metadata.AliasMetaData;
2323
import org.elasticsearch.cluster.metadata.MappingMetaData;
2424
import org.elasticsearch.common.bytes.BytesArray;
25+
import org.elasticsearch.common.bytes.BytesReference;
26+
import org.elasticsearch.common.compress.CompressedXContent;
2527
import org.elasticsearch.common.settings.Settings;
28+
import org.elasticsearch.common.xcontent.DeprecationHandler;
29+
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
2630
import org.elasticsearch.common.xcontent.ToXContent;
2731
import org.elasticsearch.common.xcontent.XContentBuilder;
32+
import org.elasticsearch.common.xcontent.XContentFactory;
2833
import org.elasticsearch.common.xcontent.XContentHelper;
34+
import org.elasticsearch.common.xcontent.XContentParser;
2935
import org.elasticsearch.common.xcontent.XContentType;
3036
import org.elasticsearch.index.mapper.MapperService;
3137
import org.elasticsearch.test.ESTestCase;
3238

3339
import java.io.IOException;
3440
import java.io.UncheckedIOException;
3541
import java.util.ArrayList;
42+
import java.util.Arrays;
43+
import java.util.Comparator;
3644
import java.util.Iterator;
3745
import java.util.List;
46+
import java.util.Locale;
3847
import java.util.Map;
48+
import java.util.function.Predicate;
3949
import java.util.stream.Collectors;
4050
import java.util.stream.IntStream;
4151

52+
import static org.elasticsearch.index.RandomCreateIndexGenerator.randomIndexSettings;
53+
import static org.elasticsearch.index.RandomCreateIndexGenerator.randomMappingFields;
4254
import static org.elasticsearch.test.AbstractXContentTestCase.xContentTester;
55+
import static org.hamcrest.Matchers.equalTo;
4356

4457
public class GetIndexTemplatesResponseTests extends ESTestCase {
4558

@@ -50,11 +63,88 @@ public class GetIndexTemplatesResponseTests extends ESTestCase {
5063

5164

5265
public void testFromXContent() throws IOException {
53-
xContentTester(this::createParser, GetIndexTemplatesResponseTests::createTestInstance, GetIndexTemplatesResponseTests::toXContent,
54-
GetIndexTemplatesResponse::fromXContent).supportsUnknownFields(false)
55-
.assertEqualsConsumer(GetIndexTemplatesResponseTests::assertEqualInstances)
56-
.shuffleFieldsExceptions(new String[] {"aliases", "mappings", "patterns", "settings"})
57-
.test();
66+
xContentTester(this::createParser,
67+
GetIndexTemplatesResponseTests::createTestInstance,
68+
GetIndexTemplatesResponseTests::toXContent,
69+
GetIndexTemplatesResponse::fromXContent)
70+
.assertEqualsConsumer(GetIndexTemplatesResponseTests::assertEqualInstances)
71+
.supportsUnknownFields(true)
72+
.randomFieldsExcludeFilter(randomFieldsExcludeFilter())
73+
.shuffleFieldsExceptions(new String[] {"aliases", "mappings", "patterns", "settings"})
74+
.test();
75+
}
76+
77+
public void testParsingFromEsResponse() throws IOException {
78+
for (int runs = 0; runs < 20; runs++) {
79+
org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse esResponse =
80+
new org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse(new ArrayList<>());
81+
82+
XContentType xContentType = randomFrom(XContentType.values());
83+
int numTemplates = randomIntBetween(0, 32);
84+
for (int i = 0; i < numTemplates; i++) {
85+
org.elasticsearch.cluster.metadata.IndexTemplateMetaData.Builder esIMD =
86+
new org.elasticsearch.cluster.metadata.IndexTemplateMetaData.Builder(String.format(Locale.ROOT, "%02d ", i) +
87+
randomAlphaOfLength(4));
88+
esIMD.patterns(Arrays.asList(generateRandomStringArray(32, 4, false, false)));
89+
esIMD.settings(randomIndexSettings());
90+
esIMD.putMapping("_doc", new CompressedXContent(BytesReference.bytes(randomMapping("_doc", xContentType))));
91+
int numAliases = randomIntBetween(0, 8);
92+
for (int j = 0; j < numAliases; j++) {
93+
esIMD.putAlias(randomAliasMetaData(String.format(Locale.ROOT, "%02d ", j) + randomAlphaOfLength(4)));
94+
}
95+
esIMD.order(randomIntBetween(0, Integer.MAX_VALUE));
96+
esIMD.version(randomIntBetween(0, Integer.MAX_VALUE));
97+
esResponse.getIndexTemplates().add(esIMD.build());
98+
}
99+
100+
XContentBuilder xContentBuilder = XContentBuilder.builder(xContentType.xContent());
101+
esResponse.toXContent(xContentBuilder, ToXContent.EMPTY_PARAMS);
102+
103+
try (XContentParser parser = XContentHelper.createParser(NamedXContentRegistry.EMPTY,
104+
DeprecationHandler.THROW_UNSUPPORTED_OPERATION, BytesReference.bytes(xContentBuilder), xContentType)) {
105+
GetIndexTemplatesResponse response = GetIndexTemplatesResponse.fromXContent(parser);
106+
assertThat(response.getIndexTemplates().size(), equalTo(numTemplates));
107+
108+
response.getIndexTemplates().sort(Comparator.comparing(IndexTemplateMetaData::name));
109+
for (int i = 0; i < numTemplates; i++) {
110+
org.elasticsearch.cluster.metadata.IndexTemplateMetaData esIMD = esResponse.getIndexTemplates().get(i);
111+
IndexTemplateMetaData result = response.getIndexTemplates().get(i);
112+
113+
assertThat(result.patterns(), equalTo(esIMD.patterns()));
114+
assertThat(result.settings(), equalTo(esIMD.settings()));
115+
assertThat(result.order(), equalTo(esIMD.order()));
116+
assertThat(result.version(), equalTo(esIMD.version()));
117+
118+
assertThat(esIMD.mappings().size(), equalTo(1));
119+
BytesArray mappingSource = new BytesArray(esIMD.mappings().valuesIt().next().uncompressed());
120+
Map<String, Object> expectedMapping =
121+
XContentHelper.convertToMap(mappingSource, true, xContentBuilder.contentType()).v2();
122+
assertThat(result.mappings().sourceAsMap(), equalTo(expectedMapping.get("_doc")));
123+
124+
assertThat(result.aliases().size(), equalTo(esIMD.aliases().size()));
125+
List<AliasMetaData> expectedAliases = Arrays.stream(esIMD.aliases().values().toArray(AliasMetaData.class))
126+
.sorted(Comparator.comparing(AliasMetaData::alias))
127+
.collect(Collectors.toList());
128+
List<AliasMetaData> actualAliases = Arrays.stream(result.aliases().values().toArray(AliasMetaData.class))
129+
.sorted(Comparator.comparing(AliasMetaData::alias))
130+
.collect(Collectors.toList());
131+
for (int j = 0; j < result.aliases().size(); j++) {
132+
assertThat(actualAliases.get(j), equalTo(expectedAliases.get(j)));
133+
}
134+
}
135+
}
136+
}
137+
}
138+
139+
private Predicate<String> randomFieldsExcludeFilter() {
140+
return (field) ->
141+
field.isEmpty()
142+
|| field.endsWith("aliases")
143+
|| field.endsWith("settings")
144+
|| field.endsWith("settings.index")
145+
|| field.endsWith("mappings") // uses parser.map()
146+
|| field.contains("mappings.properties") // cannot have extra properties
147+
;
58148
}
59149

60150
private static void assertEqualInstances(GetIndexTemplatesResponse expectedInstance, GetIndexTemplatesResponse newInstance) {
@@ -64,7 +154,7 @@ private static void assertEqualInstances(GetIndexTemplatesResponse expectedInsta
64154
new BytesArray(mappingString), true, XContentType.JSON).v2();
65155
for (IndexTemplateMetaData template : newInstance.getIndexTemplates()) {
66156
MappingMetaData mappingMD = template.mappings();
67-
if(mappingMD!=null) {
157+
if(mappingMD != null) {
68158
Map<String, Object> mappingAsMap = mappingMD.sourceAsMap();
69159
assertEquals(expectedMap, mappingAsMap);
70160
}
@@ -133,4 +223,39 @@ static void toXContent(GetIndexTemplatesResponse response, XContentBuilder build
133223
org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse(serverIndexTemplates);
134224
serverResponse.toXContent(builder, ToXContent.EMPTY_PARAMS);
135225
}
226+
227+
private static AliasMetaData randomAliasMetaData(String name) {
228+
AliasMetaData.Builder alias = AliasMetaData.builder(name);
229+
if (randomBoolean()) {
230+
if (randomBoolean()) {
231+
alias.routing(randomAlphaOfLength(5));
232+
} else {
233+
if (randomBoolean()) {
234+
alias.indexRouting(randomAlphaOfLength(5));
235+
}
236+
if (randomBoolean()) {
237+
alias.searchRouting(randomAlphaOfLength(5));
238+
}
239+
}
240+
}
241+
242+
if (randomBoolean()) {
243+
alias.filter("{\"term\":{\"year\":2016}}");
244+
}
245+
246+
if (randomBoolean()) {
247+
alias.writeIndex(randomBoolean());
248+
}
249+
return alias.build();
250+
}
251+
252+
static XContentBuilder randomMapping(String type, XContentType xContentType) throws IOException {
253+
XContentBuilder builder = XContentFactory.contentBuilder(xContentType);
254+
builder.startObject().startObject(type);
255+
256+
randomMappingFields(builder, true);
257+
258+
builder.endObject().endObject();
259+
return builder;
260+
}
136261
}

0 commit comments

Comments
 (0)