Implement MlConfigIndexMappingsFullClusterRestartIT test which verifies that .ml-config index mappings are properly updated during cluster upgrade (#44341)

przemekwitek · web-flow · commit 747ec94b43fa · 2019-07-15T20:35:22.000+02:00
diff --git a/server/src/main/java/org/elasticsearch/common/xcontent/support/XContentMapValues.java b/server/src/main/java/org/elasticsearch/common/xcontent/support/XContentMapValues.java
@@ -98,7 +98,10 @@ private static void extractRawValues(List values, List<Object> part, String[] pa
     }
 
     public static Object extractValue(String path, Map<?, ?> map) {
-        String[] pathElements = path.split("\\.");
+        return extractValue(map, path.split("\\."));
+    }
+
+    public static Object extractValue(Map<?, ?> map, String... pathElements) {
         if (pathElements.length == 0) {
             return null;
         }
diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/AnalysisConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/AnalysisConfig.java
@@ -398,10 +398,10 @@ public Builder(AnalysisConfig analysisConfig) {
             this.multivariateByFields = analysisConfig.multivariateByFields;
         }
 
-        public void setDetectors(List<Detector> detectors) {
+        public Builder setDetectors(List<Detector> detectors) {
             if (detectors == null) {
                 this.detectors = null;
-                return;
+                return this;
             }
             // We always assign sequential IDs to the detectors that are correct for this analysis config
             int detectorIndex = 0;
@@ -412,42 +412,52 @@ public void setDetectors(List<Detector> detectors) {
                 sequentialIndexDetectors.add(builder.build());
             }
             this.detectors = sequentialIndexDetectors;
+            return this;
         }
 
-        public void setDetector(int detectorIndex, Detector detector) {
+        public Builder setDetector(int detectorIndex, Detector detector) {
             detectors.set(detectorIndex, detector);
+            return this;
         }
 
-        public void setBucketSpan(TimeValue bucketSpan) {
+        public Builder setBucketSpan(TimeValue bucketSpan) {
             this.bucketSpan = bucketSpan;
+            return this;
         }
 
-        public void setLatency(TimeValue latency) {
+        public Builder setLatency(TimeValue latency) {
             this.latency = latency;
+            return this;
         }
 
-        public void setCategorizationFieldName(String categorizationFieldName) {
+        public Builder setCategorizationFieldName(String categorizationFieldName) {
             this.categorizationFieldName = categorizationFieldName;
+            return this;
         }
 
-        public void setCategorizationFilters(List<String> categorizationFilters) {
+        public Builder setCategorizationFilters(List<String> categorizationFilters) {
             this.categorizationFilters = categorizationFilters;
+            return this;
         }
 
-        public void setCategorizationAnalyzerConfig(CategorizationAnalyzerConfig categorizationAnalyzerConfig) {
+        public Builder setCategorizationAnalyzerConfig(CategorizationAnalyzerConfig categorizationAnalyzerConfig) {
             this.categorizationAnalyzerConfig = categorizationAnalyzerConfig;
+            return this;
         }
 
-        public void setSummaryCountFieldName(String summaryCountFieldName) {
+        public Builder setSummaryCountFieldName(String summaryCountFieldName) {
             this.summaryCountFieldName = summaryCountFieldName;
+            return this;
         }
 
-        public void setInfluencers(List<String> influencers) {
+        public Builder setInfluencers(List<String> influencers) {
             this.influencers = ExceptionsHelper.requireNonNull(influencers, INFLUENCERS.getPreferredName());
+            return this;
         }
 
-        public void setMultivariateByFields(Boolean multivariateByFields) {
+        public Builder setMultivariateByFields(Boolean multivariateByFields) {
             this.multivariateByFields = multivariateByFields;
+            return this;
         }
 
         /**
diff --git a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlConfigIndexMappingsFullClusterRestartIT.java b/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/MlConfigIndexMappingsFullClusterRestartIT.java
@@ -0,0 +1,135 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.restart;
+
+import org.elasticsearch.Version;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.ResponseException;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.util.concurrent.ThreadContext;
+import org.elasticsearch.common.xcontent.support.XContentMapValues;
+import org.elasticsearch.upgrades.AbstractFullClusterRestartTestCase;
+import org.elasticsearch.xpack.core.ml.job.config.AnalysisConfig;
+import org.elasticsearch.xpack.core.ml.job.config.DataDescription;
+import org.elasticsearch.xpack.core.ml.job.config.Detector;
+import org.elasticsearch.xpack.core.ml.job.config.Job;
+import org.elasticsearch.xpack.test.rest.XPackRestTestConstants;
+import org.elasticsearch.xpack.test.rest.XPackRestTestHelper;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.Base64;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.nullValue;
+
+public class MlConfigIndexMappingsFullClusterRestartIT extends AbstractFullClusterRestartTestCase {
+
+    private static final String OLD_CLUSTER_JOB_ID = "ml-config-mappings-old-cluster-job";
+    private static final String NEW_CLUSTER_JOB_ID = "ml-config-mappings-new-cluster-job";
+
+    private static final Map<String, Object> EXPECTED_DATA_FRAME_ANALYSIS_MAPPINGS =
+        mapOf(
+            "properties", mapOf(
+                "outlier_detection", mapOf(
+                    "properties", mapOf(
+                        "method", mapOf("type", "keyword"),
+                        "n_neighbors", mapOf("type", "integer"),
+                        "feature_influence_threshold", mapOf("type", "double")))));
+
+    @Override
+    protected Settings restClientSettings() {
+        String token = "Basic " + Base64.getEncoder().encodeToString("test_user:x-pack-test-password".getBytes(StandardCharsets.UTF_8));
+        return Settings.builder()
+            .put(ThreadContext.PREFIX + ".Authorization", token)
+            .build();
+    }
+
+    @Before
+    public void waitForMlTemplates() throws Exception {
+        List<String> templatesToWaitFor = XPackRestTestConstants.ML_POST_V660_TEMPLATES;
+        XPackRestTestHelper.waitForTemplates(client(), templatesToWaitFor);
+    }
+
+    public void testMlConfigIndexMappingsAfterMigration() throws Exception {
+        if (isRunningAgainstOldCluster()) {
+            assertThatMlConfigIndexDoesNotExist();
+            // trigger .ml-config index creation
+            createAnomalyDetectorJob(OLD_CLUSTER_JOB_ID);
+            if (getOldClusterVersion().onOrAfter(Version.V_7_3_0)) {
+                // .ml-config has correct mappings from the start
+                assertThat(mappingsForDataFrameAnalysis(), is(equalTo(EXPECTED_DATA_FRAME_ANALYSIS_MAPPINGS)));
+            } else {
+                // .ml-config does not yet have correct mappings, it will need an update after cluster is upgraded
+                assertThat(mappingsForDataFrameAnalysis(), is(nullValue()));
+            }
+        } else {
+            // trigger .ml-config index mappings update
+            createAnomalyDetectorJob(NEW_CLUSTER_JOB_ID);
+            // assert that the mappings are updated
+            assertThat(mappingsForDataFrameAnalysis(), is(equalTo(EXPECTED_DATA_FRAME_ANALYSIS_MAPPINGS)));
+        }
+    }
+
+    private void assertThatMlConfigIndexDoesNotExist() {
+        Request getIndexRequest = new Request("GET", ".ml-config");
+        ResponseException e = expectThrows(ResponseException.class, () -> client().performRequest(getIndexRequest));
+        assertThat(e.getResponse().getStatusLine().getStatusCode(), equalTo(404));
+    }
+
+    private void createAnomalyDetectorJob(String jobId) throws IOException {
+        Detector.Builder detector = new Detector.Builder("metric", "responsetime")
+            .setByFieldName("airline");
+        AnalysisConfig.Builder analysisConfig = new AnalysisConfig.Builder(Collections.singletonList(detector.build()))
+            .setBucketSpan(TimeValue.timeValueMinutes(10));
+        Job.Builder job = new Job.Builder(jobId)
+            .setAnalysisConfig(analysisConfig)
+            .setDataDescription(new DataDescription.Builder());
+
+        Request putJobRequest = new Request("PUT", "/_ml/anomaly_detectors/" + jobId);
+        putJobRequest.setJsonEntity(Strings.toString(job));
+        Response putJobResponse = client().performRequest(putJobRequest);
+        assertThat(putJobResponse.getStatusLine().getStatusCode(), equalTo(200));
+    }
+
+    @SuppressWarnings("unchecked")
+    private Map<String, Object> mappingsForDataFrameAnalysis() throws Exception {
+        Request getIndexMappingsRequest = new Request("GET", ".ml-config/_mappings");
+        Response getIndexMappingsResponse = client().performRequest(getIndexMappingsRequest);
+        assertThat(getIndexMappingsResponse.getStatusLine().getStatusCode(), equalTo(200));
+
+        Map<String, Object> mappings = entityAsMap(getIndexMappingsResponse);
+        mappings = (Map<String, Object>) XContentMapValues.extractValue(mappings, ".ml-config", "mappings");
+        if (mappings.containsKey("doc")) {
+            mappings = (Map<String, Object>) XContentMapValues.extractValue(mappings, "doc");
+        }
+        mappings = (Map<String, Object>) XContentMapValues.extractValue(mappings, "properties", "analysis");
+        return mappings;
+    }
+
+    private static <K, V> Map<K, V> mapOf(K k1, V v1) {
+        Map<K, V> map = new HashMap<>();
+        map.put(k1, v1);
+        return map;
+    }
+
+    private static <K, V> Map<K, V> mapOf(K k1, V v1, K k2, V v2, K k3, V v3) {
+        Map<K, V> map = new HashMap<>();
+        map.put(k1, v1);
+        map.put(k2, v2);
+        map.put(k3, v3);
+        return map;
+    }
+}

Original file line number	Diff line number	Diff line change
`@@ -98,7 +98,10 @@ private static void extractRawValues(List values, List<Object> part, String[] pa`
`98`	`98`	`}`
`99`	`99`
`100`	`100`	`public static Object extractValue(String path, Map<?, ?> map) {`
`101`		`- String[] pathElements = path.split("\\.");`
	`101`	`+ return extractValue(map, path.split("\\."));`
	`102`	`+ }`
	`103`	`+`
	`104`	`+ public static Object extractValue(Map<?, ?> map, String... pathElements) {`
`102`	`105`	`if (pathElements.length == 0) {`
`103`	`106`	`return null;`
`104`	`107`	`}`
Original file line number	Diff line number	Diff line change
`@@ -398,10 +398,10 @@ public Builder(AnalysisConfig analysisConfig) {`
`398`	`398`	`this.multivariateByFields = analysisConfig.multivariateByFields;`
`399`	`399`	`}`
`400`	`400`
`401`		`- public void setDetectors(List<Detector> detectors) {`
	`401`	`+ public Builder setDetectors(List<Detector> detectors) {`
`402`	`402`	`if (detectors == null) {`
`403`	`403`	`this.detectors = null;`
`404`		`- return;`
	`404`	`+ return this;`
`405`	`405`	`}`
`406`	`406`	`// We always assign sequential IDs to the detectors that are correct for this analysis config`
`407`	`407`	`int detectorIndex = 0;`
`@@ -412,42 +412,52 @@ public void setDetectors(List<Detector> detectors) {`
`412`	`412`	`sequentialIndexDetectors.add(builder.build());`
`413`	`413`	`}`
`414`	`414`	`this.detectors = sequentialIndexDetectors;`
	`415`	`+ return this;`
`415`	`416`	`}`
`416`	`417`
`417`		`- public void setDetector(int detectorIndex, Detector detector) {`
	`418`	`+ public Builder setDetector(int detectorIndex, Detector detector) {`
`418`	`419`	`detectors.set(detectorIndex, detector);`
	`420`	`+ return this;`
`419`	`421`	`}`
`420`	`422`
`421`		`- public void setBucketSpan(TimeValue bucketSpan) {`
	`423`	`+ public Builder setBucketSpan(TimeValue bucketSpan) {`
`422`	`424`	`this.bucketSpan = bucketSpan;`
	`425`	`+ return this;`
`423`	`426`	`}`
`424`	`427`
`425`		`- public void setLatency(TimeValue latency) {`
	`428`	`+ public Builder setLatency(TimeValue latency) {`
`426`	`429`	`this.latency = latency;`
	`430`	`+ return this;`
`427`	`431`	`}`
`428`	`432`
`429`		`- public void setCategorizationFieldName(String categorizationFieldName) {`
	`433`	`+ public Builder setCategorizationFieldName(String categorizationFieldName) {`
`430`	`434`	`this.categorizationFieldName = categorizationFieldName;`
	`435`	`+ return this;`
`431`	`436`	`}`
`432`	`437`
`433`		`- public void setCategorizationFilters(List<String> categorizationFilters) {`
	`438`	`+ public Builder setCategorizationFilters(List<String> categorizationFilters) {`
`434`	`439`	`this.categorizationFilters = categorizationFilters;`
	`440`	`+ return this;`
`435`	`441`	`}`
`436`	`442`
`437`		`- public void setCategorizationAnalyzerConfig(CategorizationAnalyzerConfig categorizationAnalyzerConfig) {`
	`443`	`+ public Builder setCategorizationAnalyzerConfig(CategorizationAnalyzerConfig categorizationAnalyzerConfig) {`
`438`	`444`	`this.categorizationAnalyzerConfig = categorizationAnalyzerConfig;`
	`445`	`+ return this;`
`439`	`446`	`}`
`440`	`447`
`441`		`- public void setSummaryCountFieldName(String summaryCountFieldName) {`
	`448`	`+ public Builder setSummaryCountFieldName(String summaryCountFieldName) {`
`442`	`449`	`this.summaryCountFieldName = summaryCountFieldName;`
	`450`	`+ return this;`
`443`	`451`	`}`
`444`	`452`
`445`		`- public void setInfluencers(List<String> influencers) {`
	`453`	`+ public Builder setInfluencers(List<String> influencers) {`
`446`	`454`	`this.influencers = ExceptionsHelper.requireNonNull(influencers, INFLUENCERS.getPreferredName());`
	`455`	`+ return this;`
`447`	`456`	`}`
`448`	`457`
`449`		`- public void setMultivariateByFields(Boolean multivariateByFields) {`
	`458`	`+ public Builder setMultivariateByFields(Boolean multivariateByFields) {`
`450`	`459`	`this.multivariateByFields = multivariateByFields;`
	`460`	`+ return this;`
`451`	`461`	`}`
`452`	`462`
`453`	`463`	`/**`