From d435f17e329f67a5205efc7ebc973625a771e3c4 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Mon, 18 Sep 2017 13:48:50 -0400 Subject: [PATCH] Upgrade API: fix excessive logging and unnecessary template updates TemplateUpgradeService might get stuck in repeatedly upgrading templates after upgrade to 5.6.0. This is caused by shuffling mappings definition in the template during template serialization. This commit makes the template serialization consistent. Closes #26673 --- .../metadata/IndexTemplateMetaData.java | 3 +- .../metadata/TemplateUpgradeService.java | 8 ++-- .../metadata/IndexTemplateMetaDataTests.java | 42 +++++++++++++++++++ 3 files changed, 47 insertions(+), 6 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java index 67c8fe44b4bee..891e407e82a45 100644 --- a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java +++ b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java @@ -26,7 +26,6 @@ import org.elasticsearch.cluster.Diff; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.bytes.BytesArray; -import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.collect.MapBuilder; import org.elasticsearch.common.compress.CompressedXContent; @@ -405,7 +404,7 @@ public static void toInnerXContent(IndexTemplateMetaData indexTemplateMetaData, builder.startObject("mappings"); for (ObjectObjectCursor cursor : indexTemplateMetaData.mappings()) { byte[] mappingSource = cursor.value.uncompressed(); - Map mapping = XContentHelper.convertToMap(new BytesArray(mappingSource), false).v2(); + Map mapping = XContentHelper.convertToMap(new BytesArray(mappingSource), true).v2(); if (mapping.size() == 1 && mapping.containsKey(cursor.key)) { // the type name is the root value, reduce it mapping = (Map) mapping.get(cursor.key); diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/TemplateUpgradeService.java b/core/src/main/java/org/elasticsearch/cluster/metadata/TemplateUpgradeService.java index 5f3b9cdf2da3d..8e8f0c594bc71 100644 --- a/core/src/main/java/org/elasticsearch/cluster/metadata/TemplateUpgradeService.java +++ b/core/src/main/java/org/elasticsearch/cluster/metadata/TemplateUpgradeService.java @@ -123,11 +123,11 @@ public void clusterChanged(ClusterChangedEvent event) { lastTemplateMetaData = templates; Optional, Set>> changes = calculateTemplateChanges(templates); if (changes.isPresent()) { - logger.info("Starting template upgrade to version {}, {} templates will be updated and {} will be removed", - Version.CURRENT, - changes.get().v1().size(), - changes.get().v2().size()); if (updatesInProgress.compareAndSet(0, changes.get().v1().size() + changes.get().v2().size())) { + logger.info("Starting template upgrade to version {}, {} templates will be updated and {} will be removed", + Version.CURRENT, + changes.get().v1().size(), + changes.get().v2().size()); threadPool.generic().execute(() -> updateTemplates(changes.get().v1(), changes.get().v2())); } } diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaDataTests.java b/core/src/test/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaDataTests.java index bfc6f5d78d24b..8f247abcf3387 100644 --- a/core/src/test/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaDataTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaDataTests.java @@ -20,9 +20,17 @@ import org.elasticsearch.Version; import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.test.ESTestCase; import java.io.IOException; @@ -30,7 +38,9 @@ import java.util.Base64; import java.util.Collections; +import static java.util.Collections.singletonMap; import static org.elasticsearch.cluster.metadata.AliasMetaData.newAliasMetaDataBuilder; +import static org.hamcrest.CoreMatchers.equalTo; public class IndexTemplateMetaDataTests extends ESTestCase { @@ -78,4 +88,36 @@ public void testIndexTemplateMetaData510() throws IOException { } } + public void testIndexTemplateMetaDataXContentRoundTrip() throws Exception { + ToXContent.Params params = new ToXContent.MapParams(singletonMap("reduce_mappings", "true")); + + String template = "{\"index_patterns\" : [ \".test-*\" ],\"order\" : 1000," + + "\"settings\" : {\"number_of_shards\" : 1,\"number_of_replicas\" : 0}," + + "\"mappings\" : {\"doc\" :" + + "{\"properties\":{\"" + + randomAlphaOfLength(10) + "\":{\"type\":\"text\"},\"" + + randomAlphaOfLength(10) + "\":{\"type\":\"keyword\"}}" + + "}}}"; + + BytesReference templateBytes = new BytesArray(template); + final IndexTemplateMetaData indexTemplateMetaData; + try (XContentParser parser = XContentHelper.createParser(NamedXContentRegistry.EMPTY, templateBytes, XContentType.JSON)) { + indexTemplateMetaData = IndexTemplateMetaData.Builder.fromXContent(parser, "test"); + } + + final BytesReference templateBytesRoundTrip; + try (XContentBuilder builder = XContentBuilder.builder(JsonXContent.jsonXContent)) { + builder.startObject(); + IndexTemplateMetaData.Builder.toXContent(indexTemplateMetaData, builder, params); + builder.endObject(); + templateBytesRoundTrip = builder.bytes(); + } + + final IndexTemplateMetaData indexTemplateMetaDataRoundTrip; + try (XContentParser parser = XContentHelper.createParser(NamedXContentRegistry.EMPTY, templateBytesRoundTrip, XContentType.JSON)) { + indexTemplateMetaDataRoundTrip = IndexTemplateMetaData.Builder.fromXContent(parser, "test"); + } + assertThat(indexTemplateMetaData, equalTo(indexTemplateMetaDataRoundTrip)); + } + }