Skip to content

Adding deprecation checks for geo_shape parameters #76627

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 8 commits into from
Sep 1, 2021
Merged

Adding deprecation checks for geo_shape parameters #76627

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
5f0f7c7
Adding deprecation checks for geo_shape parameters
masseyke Aug 17, 2021
6b62492
Merge branch '7.x' into feature/geo_shape_parameter_deprecation
elasticmachine Aug 26, 2021
8d23a22
fixing precommit check problems
masseyke Aug 26, 2021
f65d661
fixing unit tests
masseyke Aug 26, 2021
d8a2fc4
fixing unit tests
masseyke Aug 26, 2021
0b4d931
fixing precommit check problems
masseyke Aug 26, 2021
37ecfaf
code review feedback
masseyke Aug 30, 2021
e974c64
checking component templates
masseyke Aug 30, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
89 changes: 89 additions & 0 deletions ...precation/src/main/java/org/elasticsearch/xpack/deprecation/ClusterDeprecationChecks.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,25 +10,32 @@
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.metadata.ComponentTemplate;
import org.elasticsearch.cluster.metadata.IndexTemplateMetadata;
import org.elasticsearch.cluster.metadata.MappingMetadata;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.collect.ImmutableOpenMap;
import org.elasticsearch.common.compress.CompressedXContent;
import org.elasticsearch.common.xcontent.XContentHelper;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.core.TimeValue;
import org.elasticsearch.core.Tuple;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.mapper.FieldNamesFieldMapper;
import org.elasticsearch.index.mapper.LegacyGeoShapeFieldMapper;
import org.elasticsearch.ingest.IngestService;
import org.elasticsearch.ingest.PipelineConfiguration;

import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
import java.util.stream.StreamSupport;

import static org.elasticsearch.cluster.routing.allocation.DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_INCLUDE_RELOCATIONS_SETTING;
import static org.elasticsearch.search.SearchModule.INDICES_MAX_CLAUSE_COUNT_SETTING;
Expand Down Expand Up @@ -201,4 +208,86 @@ static DeprecationIssue checkClusterRoutingAllocationIncludeRelocationsSetting(f
DeprecationIssue.Level.WARNING
);
}

@SuppressWarnings("unchecked")
private static String getDetailsMessageForComponentTemplates(Map<String, ComponentTemplate> componentTemplates) {
String detailsForComponentTemplates =
componentTemplates.entrySet().stream().map((templateCursor) -> {
String templateName = templateCursor.getKey();
ComponentTemplate componentTemplate = templateCursor.getValue();
CompressedXContent mappings = componentTemplate.template().mappings();
if (mappings != null) {
Tuple<XContentType, Map<String, Object>> tuple = XContentHelper.convertToMap(mappings.uncompressed(), true,
XContentType.JSON);
Map<String, Object> mappingAsMap = tuple.v2();
List<String> messages = mappingAsMap == null ? Collections.emptyList() :
IndexDeprecationChecks.findInPropertiesRecursively(LegacyGeoShapeFieldMapper.CONTENT_TYPE,
mappingAsMap,
IndexDeprecationChecks::isGeoShapeFieldWithDeprecatedParam,
IndexDeprecationChecks::formatDeprecatedGeoShapeParamMessage);
if (messages.isEmpty() == false) {
String messageForMapping =
"mappings in component template " + templateName + " contains deprecated geo_shape properties. " +
messages.stream().collect(Collectors.joining("; "));
return messageForMapping;
}
}
return null;
}).filter(messageForTemplate -> Strings.isEmpty(messageForTemplate) == false).collect(Collectors.joining("; "));
return detailsForComponentTemplates;
}

@SuppressWarnings("unchecked")
private static String getDetailsMessageForIndexTemplates(ImmutableOpenMap<String, IndexTemplateMetadata> indexTemplates) {
String detailsForIndexTemplates =
StreamSupport.stream(indexTemplates.spliterator(), false).map((templateCursor) -> {
String templateName = templateCursor.key;
IndexTemplateMetadata indexTemplateMetadata = templateCursor.value;
String messageForTemplate =
StreamSupport.stream(indexTemplateMetadata.getMappings().spliterator(), false).map((mappingCursor) -> {
CompressedXContent mapping = mappingCursor.value;
Tuple<XContentType, Map<String, Object>> tuple = XContentHelper.convertToMap(mapping.uncompressed(), true,
XContentType.JSON);
Map<String, Object> mappingAsMap = (Map<String, Object>) tuple.v2().get("_doc");
List<String> messages = mappingAsMap == null ? Collections.emptyList() :
IndexDeprecationChecks.findInPropertiesRecursively(LegacyGeoShapeFieldMapper.CONTENT_TYPE,
mappingAsMap,
IndexDeprecationChecks::isGeoShapeFieldWithDeprecatedParam,
IndexDeprecationChecks::formatDeprecatedGeoShapeParamMessage);
return messages;
}).filter(messages -> messages.isEmpty() == false).map(messages -> {
String messageForMapping =
"mappings in index template " + templateName + " contains deprecated geo_shape properties. " +
messages.stream().collect(Collectors.joining("; "));
return messageForMapping;
}).collect(Collectors.joining("; "));
return messageForTemplate;
}).filter(messageForTemplate -> Strings.isEmpty(messageForTemplate) == false).collect(Collectors.joining("; "));
return detailsForIndexTemplates;
}

@SuppressWarnings("unchecked")
static DeprecationIssue checkGeoShapeTemplates(final ClusterState clusterState) {
String detailsForComponentTemplates = getDetailsMessageForComponentTemplates(clusterState.getMetadata().componentTemplates());
String detailsForIndexTemplates = getDetailsMessageForIndexTemplates(clusterState.getMetadata().getTemplates());
boolean deprecationInComponentTemplates = Strings.isEmpty(detailsForComponentTemplates) == false;
boolean deprecationInIndexTemplates = Strings.isEmpty(detailsForIndexTemplates) == false;
String url = "https://www.elastic.co/guide/en/elasticsearch/reference/master/migrating-8.0.html#breaking_80_mappings_changes";
if (deprecationInComponentTemplates && deprecationInIndexTemplates) {
String message = "component templates and index templates contain deprecated geo_shape properties that must be removed";
String details = detailsForComponentTemplates + "; " + detailsForIndexTemplates;
return new DeprecationIssue(DeprecationIssue.Level.CRITICAL, message, url, details, false,
null);
} if (deprecationInComponentTemplates == false && deprecationInIndexTemplates) {
String message = "index templates contain deprecated geo_shape properties that must be removed";
return new DeprecationIssue(DeprecationIssue.Level.CRITICAL, message, url, detailsForIndexTemplates, false,
null);
} else if (deprecationInIndexTemplates == false && deprecationInComponentTemplates) {
String message = "component templates contain deprecated geo_shape properties that must be removed";
return new DeprecationIssue(DeprecationIssue.Level.CRITICAL, message, url, detailsForComponentTemplates, false,
null);
} else {
return null;
}
}
}
6 changes: 4 additions & 2 deletions ...ugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,8 @@ private DeprecationChecks() {
ClusterDeprecationChecks::checkPollIntervalTooLow,
ClusterDeprecationChecks::checkTemplatesWithFieldNamesDisabled,
ClusterDeprecationChecks::checkTemplatesWithMultipleTypes,
ClusterDeprecationChecks::checkClusterRoutingAllocationIncludeRelocationsSetting
ClusterDeprecationChecks::checkClusterRoutingAllocationIncludeRelocationsSetting,
ClusterDeprecationChecks::checkGeoShapeTemplates
));

static final List<NodeDeprecationCheck<Settings, PluginsAndModules, ClusterState, XPackLicenseState, DeprecationIssue>>
Expand Down Expand Up @@ -111,7 +112,8 @@ private DeprecationChecks() {
IndexDeprecationChecks::checkIndexDataPath,
IndexDeprecationChecks::indexingSlowLogLevelSettingCheck,
IndexDeprecationChecks::searchSlowLogLevelSettingCheck,
IndexDeprecationChecks::storeTypeSettingCheck
IndexDeprecationChecks::storeTypeSettingCheck,
IndexDeprecationChecks::checkGeoShapeMappings
));

/**
Expand Down
40 changes: 40 additions & 0 deletions ...deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecks.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
import org.elasticsearch.index.IndexingSlowLog;
import org.elasticsearch.index.SearchSlowLog;
import org.elasticsearch.index.SlowLogLevel;
import org.elasticsearch.index.mapper.LegacyGeoShapeFieldMapper;

import java.util.ArrayList;
import java.util.Collections;
Expand All @@ -31,6 +32,7 @@
import java.util.function.BiConsumer;
import java.util.function.BiFunction;
import java.util.function.Function;
import java.util.stream.Collectors;


/**
Expand Down Expand Up @@ -317,4 +319,42 @@ static DeprecationIssue storeTypeSettingCheck(IndexMetadata indexMetadata) {
}
return null;
}

protected static boolean isGeoShapeFieldWithDeprecatedParam(Map<?, ?> property) {
return LegacyGeoShapeFieldMapper.CONTENT_TYPE.equals(property.get("type")) &&
LegacyGeoShapeFieldMapper.DEPRECATED_PARAMETERS.stream().anyMatch(deprecatedParameter ->
property.containsKey(deprecatedParameter)
);
}

protected static String formatDeprecatedGeoShapeParamMessage(String type, Map.Entry<?, ?> entry) {
String fieldName = entry.getKey().toString();
Map<?, ?> value = (Map<?, ?>) entry.getValue();
return LegacyGeoShapeFieldMapper.DEPRECATED_PARAMETERS.stream()
.filter(deprecatedParameter -> value.containsKey(deprecatedParameter))
.map(deprecatedParameter -> String.format(Locale.ROOT, "parameter [%s] in field [%s]", type, deprecatedParameter, fieldName))
.collect(Collectors.joining("; "));
}

@SuppressWarnings("unchecked")
static DeprecationIssue checkGeoShapeMappings(IndexMetadata indexMetadata) {
if (indexMetadata == null || indexMetadata.mapping() == null) {
return null;
}
Map<String, Object> sourceAsMap = indexMetadata.mapping().getSourceAsMap();
List<String> messages = findInPropertiesRecursively(LegacyGeoShapeFieldMapper.CONTENT_TYPE, sourceAsMap,
IndexDeprecationChecks::isGeoShapeFieldWithDeprecatedParam,
IndexDeprecationChecks::formatDeprecatedGeoShapeParamMessage);
if (messages.isEmpty()) {
return null;
} else {
String message = String.format(Locale.ROOT,"mappings for index %s contains deprecated geo_shape properties that must be " +
"removed", indexMetadata.getIndex().getName());
String details = String.format(Locale.ROOT,
"The following geo_shape parameters must be removed from %s: [%s]", indexMetadata.getIndex().getName(),
messages.stream().collect(Collectors.joining("; ")));
String url = "https://www.elastic.co/guide/en/elasticsearch/reference/master/migrating-8.0.html#breaking_80_mappings_changes";
return new DeprecationIssue(DeprecationIssue.Level.CRITICAL, message, url, details, false, null);
}
}
}
84 changes: 84 additions & 0 deletions ...tion/src/test/java/org/elasticsearch/xpack/deprecation/ClusterDeprecationChecksTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,15 @@
import org.elasticsearch.action.ingest.PutPipelineRequest;
import org.elasticsearch.cluster.ClusterName;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.metadata.AliasMetadata;
import org.elasticsearch.cluster.metadata.ComponentTemplate;
import org.elasticsearch.cluster.metadata.IndexTemplateMetadata;
import org.elasticsearch.cluster.metadata.Metadata;
import org.elasticsearch.cluster.metadata.Template;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.collect.ImmutableOpenMap;
import org.elasticsearch.common.compress.CompressedXContent;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentType;
Expand All @@ -24,6 +28,7 @@

import java.io.IOException;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;

Expand Down Expand Up @@ -355,4 +360,83 @@ public void testClusterRoutingAllocationIncludeRelocationsSetting() {

assertWarnings(expectedWarning);
}

public void testCheckGeoShapeMappings() throws Exception {
// First, testing only an index template:
IndexTemplateMetadata indexTemplateMetadata = IndexTemplateMetadata.builder("single-type")
.patterns(Collections.singletonList("foo"))
.putMapping("_doc", "{\n" +
" \"_doc\":{\n" +
" \"properties\":{\n" +
" \"nested_field\":{\n" +
" \"type\":\"nested\",\n" +
" \"properties\":{\n" +
" \"location\":{\n" +
" \"type\":\"geo_shape\",\n" +
" \"strategy\":\"recursive\",\n" +
" \"points_only\":true\n" +
" }\n" +
" }\n" +
" }\n" +
" }\n" +
" }\n" +
"}")
.build();
ImmutableOpenMap<String, IndexTemplateMetadata> templates = ImmutableOpenMap.<String, IndexTemplateMetadata>builder()
.fPut("single-type", indexTemplateMetadata)
.build();
Metadata badMetadata = Metadata.builder()
.templates(templates)
.build();
ClusterState badState = ClusterState.builder(new ClusterName("test")).metadata(badMetadata).build();
DeprecationIssue issue = ClusterDeprecationChecks.checkGeoShapeTemplates(badState);

assertThat(issue, equalTo(
new DeprecationIssue(DeprecationIssue.Level.CRITICAL,
"index templates contain deprecated geo_shape properties that must be removed",
"https://www.elastic.co/guide/en/elasticsearch/reference/master/migrating-8.0.html#breaking_80_mappings_changes",
"mappings in index template single-type contains deprecated geo_shape properties. [parameter [geo_shape] in field " +
"[points_only]; parameter [geo_shape] in field [strategy]]", false, null)
));

// Second, testing only a component template:
String templateName = "my-template";
Settings settings = Settings.builder().put("index.number_of_shards", 1).build();
CompressedXContent mappings = new CompressedXContent("{\"properties\":{\"location\":{\"type\":\"geo_shape\", " +
"\"strategy\":\"recursive\", \"points_only\":true}}}");
AliasMetadata alias = AliasMetadata.builder("alias").writeIndex(true).build();
Template template = new Template(settings, mappings, Collections.singletonMap("alias", alias));
ComponentTemplate componentTemplate = new ComponentTemplate(template, 1L, new HashMap<>());
badMetadata = Metadata.builder()
.componentTemplates(Collections.singletonMap(templateName, componentTemplate))
.build();
badState = ClusterState.builder(new ClusterName("test")).metadata(badMetadata).build();
issue = ClusterDeprecationChecks.checkGeoShapeTemplates(badState);

assertThat(issue, equalTo(
new DeprecationIssue(DeprecationIssue.Level.CRITICAL,
"component templates contain deprecated geo_shape properties that must be removed",
"https://www.elastic.co/guide/en/elasticsearch/reference/master/migrating-8.0.html#breaking_80_mappings_changes",
"mappings in component template my-template contains deprecated geo_shape properties. [parameter [geo_shape] in field " +
"[points_only]; parameter [geo_shape] in field [strategy]]", false, null)
));

// Third, trying a component template and an index template:
badMetadata = Metadata.builder()
.componentTemplates(Collections.singletonMap(templateName, componentTemplate))
.templates(templates)
.build();
badState = ClusterState.builder(new ClusterName("test")).metadata(badMetadata).build();
issue = ClusterDeprecationChecks.checkGeoShapeTemplates(badState);

assertThat(issue, equalTo(
new DeprecationIssue(DeprecationIssue.Level.CRITICAL,
"component templates and index templates contain deprecated geo_shape properties that must be removed",
"https://www.elastic.co/guide/en/elasticsearch/reference/master/migrating-8.0.html#breaking_80_mappings_changes",
"mappings in component template my-template contains deprecated geo_shape properties. [parameter [geo_shape] in field " +
"[points_only]; parameter [geo_shape] in field [strategy]]; mappings in index template single-type contains " +
"deprecated geo_shape properties. [parameter [geo_shape] in field [points_only]; parameter [geo_shape] in field " +
"[strategy]]", false, null)
));
}
}
Loading