Skip to content

Clean up IndexTemplateMetaData xcontent serialization #54172

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 3 commits into from
Closed

Clean up IndexTemplateMetaData xcontent serialization #54172

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
9742739
Clean up IndexTemplateMetaData xcontent serialization
romseygeek Mar 23, 2020
d610d39
httpexporter test
romseygeek Mar 24, 2020
cde0ff2
Add BWC test for v7-style templates with arrays of mappings
romseygeek Mar 25, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 1 addition & 6 deletions ...n/java/org/elasticsearch/action/admin/indices/template/get/GetIndexTemplatesResponse.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@
import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.ToXContentObject;
import org.elasticsearch.common.xcontent.XContentBuilder;

Expand All @@ -31,8 +30,6 @@
import java.util.List;
import java.util.Objects;

import static java.util.Collections.singletonMap;

public class GetIndexTemplatesResponse extends ActionResponse implements ToXContentObject {

private final List<IndexTemplateMetaData> indexTemplates;
Expand Down Expand Up @@ -77,11 +74,9 @@ public int hashCode() {

@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
params = new ToXContent.DelegatingMapParams(singletonMap("reduce_mappings", "true"), params);

builder.startObject();
for (IndexTemplateMetaData indexTemplateMetaData : getIndexTemplates()) {
IndexTemplateMetaData.Builder.toXContent(indexTemplateMetaData, builder, params);
indexTemplateMetaData.toXContent(builder, params);
}
builder.endObject();
return builder;
Expand Down
5 changes: 3 additions & 2 deletions server/src/main/java/org/elasticsearch/cluster/ClusterState.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.io.stream.VersionedNamedWriteable;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.ToXContentFragment;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentHelper;
Expand Down Expand Up @@ -426,10 +427,10 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
coordinationMetaData().toXContent(builder, params);
builder.endObject();

ToXContent.Params templateParams = new DelegatingMapParams(Map.of(IndexTemplateMetaData.INCLUDE_TYPE_NAME, "true"), params);
builder.startObject("templates");
for (ObjectCursor<IndexTemplateMetaData> cursor : metaData().templates().values()) {
IndexTemplateMetaData templateMetaData = cursor.value;
IndexTemplateMetaData.Builder.toXContentWithTypes(templateMetaData, builder, params);
cursor.value.toXContent(builder, templateParams);
}
builder.endObject();

Expand Down
265 changes: 106 additions & 159 deletions server/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java
View file
Open in desktop

Large diffs are not rendered by default.

6 changes: 4 additions & 2 deletions server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1372,9 +1372,11 @@ public static void toXContent(MetaData metaData, XContentBuilder builder, ToXCon
builder.endObject();
}

ToXContent.Params typedParams
= new ToXContent.DelegatingMapParams(Map.of(IndexTemplateMetaData.INCLUDE_TYPE_NAME, "true"), params);
builder.startObject("templates");
for (ObjectCursor<IndexTemplateMetaData> cursor : metaData.templates().values()) {
IndexTemplateMetaData.Builder.toXContentWithTypes(cursor.value, builder, params);
cursor.value.toXContent(builder, typedParams);
}
builder.endObject();

Expand Down Expand Up @@ -1439,7 +1441,7 @@ public static MetaData fromXContent(XContentParser parser, boolean preserveUnkno
builder.hashesOfConsistentSettings(parser.mapStrings());
} else if ("templates".equals(currentFieldName)) {
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
builder.put(IndexTemplateMetaData.Builder.fromXContent(parser, parser.currentName()));
builder.put(IndexTemplateMetaData.fromXContent(parser, parser.currentName()));
}
} else {
try {
Expand Down
10 changes: 3 additions & 7 deletions server/src/main/java/org/elasticsearch/cluster/metadata/TemplateUpgradeService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,8 +57,6 @@
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.UnaryOperator;

import static java.util.Collections.singletonMap;

/**
* Upgrades Templates on behalf of installed {@link Plugin}s when a node joins the cluster
*/
Expand Down Expand Up @@ -248,14 +246,12 @@ Optional<Tuple<Map<String, BytesReference>, Set<String>>> calculateTemplateChang
return Optional.empty();
}

private static final ToXContent.Params PARAMS = new ToXContent.MapParams(singletonMap("reduce_mappings", "true"));
private static final ToXContent.Params PARAMS
= new ToXContent.MapParams(Map.of(IndexTemplateMetaData.INCLUDE_TEMPLATE_NAME, "false"));

private BytesReference toBytesReference(IndexTemplateMetaData templateMetaData) {
try {
return XContentHelper.toXContent((builder, params) -> {
IndexTemplateMetaData.Builder.toInnerXContentWithTypes(templateMetaData, builder, params);
return builder;
}, XContentType.JSON, PARAMS, false);
return XContentHelper.toXContent(templateMetaData, XContentType.JSON, PARAMS, false);
} catch (IOException ex) {
throw new IllegalStateException("Cannot serialize template [" + templateMetaData.getName() + "]", ex);
}
Expand Down
30 changes: 22 additions & 8 deletions server/src/test/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaDataTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@

import java.util.Arrays;
import java.util.Collections;
import java.util.Map;

import static org.hamcrest.CoreMatchers.equalTo;
import static org.hamcrest.Matchers.contains;
Expand All @@ -54,21 +55,23 @@ public void testIndexTemplateMetaDataXContentRoundTrip() throws Exception {
final IndexTemplateMetaData indexTemplateMetaData;
try (XContentParser parser = XContentHelper.createParser(NamedXContentRegistry.EMPTY,
DeprecationHandler.THROW_UNSUPPORTED_OPERATION, templateBytes, XContentType.JSON)) {
indexTemplateMetaData = IndexTemplateMetaData.Builder.fromXContent(parser, "test");
indexTemplateMetaData = IndexTemplateMetaData.fromXContent(parser, "test");
}

final BytesReference templateBytesRoundTrip;
final ToXContent.Params params = new ToXContent.MapParams(
Map.of(IndexTemplateMetaData.INCLUDE_TYPE_NAME, "true", IndexTemplateMetaData.INCLUDE_TEMPLATE_NAME, "false"));
try (XContentBuilder builder = XContentBuilder.builder(JsonXContent.jsonXContent)) {
builder.startObject();
IndexTemplateMetaData.Builder.toXContentWithTypes(indexTemplateMetaData, builder, ToXContent.EMPTY_PARAMS);
indexTemplateMetaData.toXContent(builder, params);
builder.endObject();
templateBytesRoundTrip = BytesReference.bytes(builder);
}

final IndexTemplateMetaData indexTemplateMetaDataRoundTrip;
try (XContentParser parser = XContentHelper.createParser(NamedXContentRegistry.EMPTY,
DeprecationHandler.THROW_UNSUPPORTED_OPERATION, templateBytesRoundTrip, XContentType.JSON)) {
indexTemplateMetaDataRoundTrip = IndexTemplateMetaData.Builder.fromXContent(parser, "test");
indexTemplateMetaDataRoundTrip = IndexTemplateMetaData.fromXContent(parser, "test");
}
assertThat(indexTemplateMetaData, equalTo(indexTemplateMetaDataRoundTrip));
}
Expand Down Expand Up @@ -97,7 +100,7 @@ public void testValidateInvalidIndexPatterns() throws Exception {
XContentHelper.createParser(NamedXContentRegistry.EMPTY,
DeprecationHandler.THROW_UNSUPPORTED_OPERATION, new BytesArray(templateWithEmptyPattern), XContentType.JSON)) {
final IllegalArgumentException ex = expectThrows(IllegalArgumentException.class,
() -> IndexTemplateMetaData.Builder.fromXContent(parser, randomAlphaOfLengthBetween(1, 100)));
() -> IndexTemplateMetaData.fromXContent(parser, randomAlphaOfLengthBetween(1, 100)));
assertThat(ex.getMessage(), equalTo("Index patterns must not be null or empty; got []"));
}

Expand All @@ -112,7 +115,7 @@ DeprecationHandler.THROW_UNSUPPORTED_OPERATION, new BytesArray(templateWithEmpty
XContentHelper.createParser(NamedXContentRegistry.EMPTY,
DeprecationHandler.THROW_UNSUPPORTED_OPERATION, new BytesArray(templateWithoutPattern), XContentType.JSON)) {
final IllegalArgumentException ex = expectThrows(IllegalArgumentException.class,
() -> IndexTemplateMetaData.Builder.fromXContent(parser, randomAlphaOfLengthBetween(1, 100)));
() -> IndexTemplateMetaData.fromXContent(parser, randomAlphaOfLengthBetween(1, 100)));
assertThat(ex.getMessage(), equalTo("Index patterns must not be null or empty; got null"));
}
}
Expand All @@ -122,12 +125,21 @@ public void testParseTemplateWithAliases() throws Exception {
try (XContentParser parser =
XContentHelper.createParser(NamedXContentRegistry.EMPTY,
DeprecationHandler.THROW_UNSUPPORTED_OPERATION, new BytesArray(templateInJSON), XContentType.JSON)) {
IndexTemplateMetaData template = IndexTemplateMetaData.Builder.fromXContent(parser, randomAlphaOfLengthBetween(1, 100));
IndexTemplateMetaData template = IndexTemplateMetaData.fromXContent(parser, randomAlphaOfLengthBetween(1, 100));
assertThat(template.aliases().containsKey("log"), equalTo(true));
assertThat(template.patterns(), contains("pattern-1"));
}
}

public void testXContentFromV7Nodes() throws Exception {
String json = "{\"mappings\":[{\"_doc\":{}}], \"index_patterns\": [\"pattern-1\"]}";
try (XContentParser parser = XContentHelper.createParser(NamedXContentRegistry.EMPTY,
DeprecationHandler.THROW_UNSUPPORTED_OPERATION, new BytesArray(json), XContentType.JSON)) {
IndexTemplateMetaData template = IndexTemplateMetaData.fromXContent(parser, "");
assertNotNull(template.mappings());
}
}

public void testFromToXContent() throws Exception {
String templateName = randomUnicodeOfCodepointLengthBetween(1, 10);
IndexTemplateMetaData.Builder templateBuilder = IndexTemplateMetaData.builder(templateName);
Expand Down Expand Up @@ -157,12 +169,14 @@ public void testFromToXContent() throws Exception {
templateBuilder.putMapping("doc", "{\"doc\":{\"properties\":{\"type\":\"text\"}}}");
}
IndexTemplateMetaData template = templateBuilder.build();
final ToXContent.Params params = new ToXContent.MapParams(
Map.of(IndexTemplateMetaData.INCLUDE_TYPE_NAME, "true", IndexTemplateMetaData.INCLUDE_TEMPLATE_NAME, "false"));
XContentBuilder builder = XContentBuilder.builder(randomFrom(XContentType.JSON.xContent()));
builder.startObject();
IndexTemplateMetaData.Builder.toXContentWithTypes(template, builder, ToXContent.EMPTY_PARAMS);
template.toXContent(builder, params);
builder.endObject();
try (XContentParser parser = createParser(shuffleXContent(builder))) {
IndexTemplateMetaData parsed = IndexTemplateMetaData.Builder.fromXContent(parser, templateName);
IndexTemplateMetaData parsed = IndexTemplateMetaData.fromXContent(parser, templateName);
assertThat(parsed, equalTo(template));
}
}
Expand Down
2 changes: 1 addition & 1 deletion x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/TemplateUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ public static void loadTemplateIntoMap(String resource, Map<String, IndexTemplat
final String template = loadTemplate(resource, version, versionProperty);
try (XContentParser parser = XContentFactory.xContent(XContentType.JSON)
.createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, template)) {
map.put(templateName, IndexTemplateMetaData.Builder.fromXContent(parser, templateName));
map.put(templateName, IndexTemplateMetaData.fromXContent(parser, templateName));
} catch (IOException e) {
// TODO: should we handle this with a thrown exception?
logger.error("Error loading template [{}] as part of metadata upgrading", templateName);
Expand Down
6 changes: 5 additions & 1 deletion .../src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/TemplateHttpResource.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentParser;
Expand Down Expand Up @@ -107,10 +108,13 @@ protected void doPublish(final RestClient client, final ActionListener<Boolean>
HttpEntity templateToHttpEntity() {
// the internal representation of a template has type nested under mappings.
// this uses xContent to help remove the type before sending to the remote cluster
ToXContent.Params params = new ToXContent.MapParams(Map.of(IndexTemplateMetaData.INCLUDE_TEMPLATE_NAME, "false"));
try (XContentParser parser = XContentFactory.xContent(XContentType.JSON)
.createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, template.get())) {
XContentBuilder builder = JsonXContent.contentBuilder();
IndexTemplateMetaData.Builder.removeType(IndexTemplateMetaData.Builder.fromXContent(parser, templateName), builder);
builder.startObject();
IndexTemplateMetaData.fromXContent(parser, templateName).toXContent(builder, params);
builder.endObject();
return new StringEntity(BytesReference.bytes(builder).utf8ToString(), ContentType.APPLICATION_JSON);
} catch (IOException ex) {
throw new IllegalStateException("Cannot serialize template [" + templateName + "] for monitoring export", ex);
Expand Down
6 changes: 5 additions & 1 deletion ...toring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporterIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
import org.elasticsearch.common.util.concurrent.ThreadContext;
import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentHelper;
Expand Down Expand Up @@ -971,7 +972,10 @@ private String getExternalTemplateRepresentation(String internalRepresentation)
try (XContentParser parser = XContentFactory.xContent(XContentType.JSON)
.createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, internalRepresentation)) {
XContentBuilder builder = JsonXContent.contentBuilder();
IndexTemplateMetaData.Builder.removeType(IndexTemplateMetaData.Builder.fromXContent(parser, ""), builder);
ToXContent.Params params = new ToXContent.MapParams(Map.of(IndexTemplateMetaData.INCLUDE_TEMPLATE_NAME, "false"));
builder.startObject();
IndexTemplateMetaData.fromXContent(parser, "").toXContent(builder, params);
builder.endObject();
return BytesReference.bytes(builder).utf8ToString();
}
}
Expand Down