From 09ff0af4f1144e5146bd293de530646f200afc8a Mon Sep 17 00:00:00 2001 From: David Roberts Date: Mon, 14 Oct 2019 13:08:05 +0100 Subject: [PATCH 1/5] [ML] Adjust BWC version following backport The BWC version for await_lazy_open/await_lazy_start should be 7.5.0 now #47726 is backported. --- .../ml/action/StartDataFrameAnalyticsAction.java | 6 ++---- .../core/ml/dataframe/DataFrameAnalyticsConfig.java | 3 +-- .../core/ml/dataframe/DataFrameAnalyticsState.java | 3 +-- .../elasticsearch/xpack/core/ml/job/config/Job.java | 12 ++++-------- .../xpack/core/ml/job/config/JobUpdate.java | 6 ++---- .../ml/dataframe/DataFrameAnalyticsStateTests.java | 3 +-- 6 files changed, 11 insertions(+), 22 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartDataFrameAnalyticsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartDataFrameAnalyticsAction.java index 15baa03079a39..c9e486fb86cc0 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartDataFrameAnalyticsAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartDataFrameAnalyticsAction.java @@ -197,8 +197,7 @@ public TaskParams(StreamInput in) throws IOException { } else { progressOnStart = Collections.emptyList(); } - // TODO: change version in backport - if (in.getVersion().onOrAfter(Version.V_8_0_0)) { + if (in.getVersion().onOrAfter(Version.V_7_5_0)) { allowLazyStart = in.readBoolean(); } else { allowLazyStart = false; @@ -234,8 +233,7 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getVersion().onOrAfter(Version.V_7_5_0)) { out.writeList(progressOnStart); } - // TODO: change version in backport - if (out.getVersion().onOrAfter(Version.V_8_0_0)) { + if (out.getVersion().onOrAfter(Version.V_7_5_0)) { out.writeBoolean(allowLazyStart); } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfig.java index 7307fa38ed4af..af749ef27f8cd 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfig.java @@ -163,8 +163,7 @@ public DataFrameAnalyticsConfig(StreamInput in) throws IOException { createTime = null; version = null; } - // TODO: change version in backport - if (in.getVersion().onOrAfter(Version.V_8_0_0)) { + if (in.getVersion().onOrAfter(Version.V_7_5_0)) { allowLazyStart = in.readBoolean(); } else { allowLazyStart = false; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsState.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsState.java index 61279b192bacc..392c0538e2454 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsState.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsState.java @@ -29,8 +29,7 @@ public static DataFrameAnalyticsState fromStream(StreamInput in) throws IOExcept @Override public void writeTo(StreamOutput out) throws IOException { DataFrameAnalyticsState toWrite = this; - // TODO: change version in backport - if (out.getVersion().before(Version.V_8_0_0) && toWrite == STARTING) { + if (out.getVersion().before(Version.V_7_5_0) && toWrite == STARTING) { // Before 7.5.0 there was no STARTING state and jobs for which // tasks existed but were unassigned were considered STOPPED toWrite = STOPPED; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java index 28ab4778d1483..36a3ed2d1d7c4 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java @@ -225,8 +225,7 @@ public Job(StreamInput in) throws IOException { } resultsIndexName = in.readString(); deleting = in.readBoolean(); - // TODO: change version in backport - if (in.getVersion().onOrAfter(Version.V_8_0_0)) { + if (in.getVersion().onOrAfter(Version.V_7_5_0)) { allowLazyOpen = in.readBoolean(); } else { allowLazyOpen = false; @@ -495,8 +494,7 @@ public void writeTo(StreamOutput out) throws IOException { } out.writeString(resultsIndexName); out.writeBoolean(deleting); - // TODO: change version in backport - if (out.getVersion().onOrAfter(Version.V_8_0_0)) { + if (out.getVersion().onOrAfter(Version.V_7_5_0)) { out.writeBoolean(allowLazyOpen); } } @@ -718,8 +716,7 @@ public Builder(StreamInput in) throws IOException { } resultsIndexName = in.readOptionalString(); deleting = in.readBoolean(); - // TODO: change version in backport - if (in.getVersion().onOrAfter(Version.V_8_0_0)) { + if (in.getVersion().onOrAfter(Version.V_7_5_0)) { allowLazyOpen = in.readBoolean(); } } @@ -915,8 +912,7 @@ public void writeTo(StreamOutput out) throws IOException { } out.writeOptionalString(resultsIndexName); out.writeBoolean(deleting); - // TODO: change version in backport - if (out.getVersion().onOrAfter(Version.V_8_0_0)) { + if (out.getVersion().onOrAfter(Version.V_7_5_0)) { out.writeBoolean(allowLazyOpen); } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdate.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdate.java index 2b95d4571dd61..8d01a62921994 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdate.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdate.java @@ -142,8 +142,7 @@ public JobUpdate(StreamInput in) throws IOException { } else { modelSnapshotMinVersion = null; } - // TODO: change version in backport - if (in.getVersion().onOrAfter(Version.V_8_0_0)) { + if (in.getVersion().onOrAfter(Version.V_7_5_0)) { allowLazyOpen = in.readOptionalBoolean(); } else { allowLazyOpen = null; @@ -187,8 +186,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeBoolean(false); } } - // TODO: change version in backport - if (out.getVersion().onOrAfter(Version.V_8_0_0)) { + if (out.getVersion().onOrAfter(Version.V_7_5_0)) { out.writeOptionalBoolean(allowLazyOpen); } } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsStateTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsStateTests.java index a6258718b4439..def59a26462cd 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsStateTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsStateTests.java @@ -49,8 +49,7 @@ public void testWriteStartingStateToPre75() throws IOException { public void testWriteStartingStateToPost75() throws IOException { StreamOutput streamOutput = mock(StreamOutput.class); - // TODO: change version in backport - when(streamOutput.getVersion()).thenReturn(Version.V_8_0_0); + when(streamOutput.getVersion()).thenReturn(Version.V_7_5_0); DataFrameAnalyticsState.STARTING.writeTo(streamOutput); verify(streamOutput, times(1)).writeEnum(DataFrameAnalyticsState.STARTING); } From 2025ef3d315f5f1b2459d396fc52cfd92de79349 Mon Sep 17 00:00:00 2001 From: David Roberts Date: Mon, 14 Oct 2019 13:22:06 +0100 Subject: [PATCH 2/5] Fix one more BWC version --- .../xpack/core/ml/dataframe/DataFrameAnalyticsConfig.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfig.java index af749ef27f8cd..6788fdaf7c8a0 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/DataFrameAnalyticsConfig.java @@ -270,8 +270,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeBoolean(false); } } - // TODO: change version in backport - if (out.getVersion().onOrAfter(Version.V_8_0_0)) { + if (out.getVersion().onOrAfter(Version.V_7_5_0)) { out.writeBoolean(allowLazyStart); } } From df09515964564cc608f35cad363d0d8812307db6 Mon Sep 17 00:00:00 2001 From: David Roberts Date: Mon, 14 Oct 2019 14:37:21 +0100 Subject: [PATCH 3/5] Mute backport-sensitive BWC tests --- .../90_ml_data_frame_analytics_crud.yml | 21 +++++++++++++++++++ .../90_ml_data_frame_analytics_crud.yml | 18 ++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/90_ml_data_frame_analytics_crud.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/90_ml_data_frame_analytics_crud.yml index ae63640543651..4110d7f1664ac 100644 --- a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/90_ml_data_frame_analytics_crud.yml +++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/90_ml_data_frame_analytics_crud.yml @@ -1,6 +1,9 @@ --- "Get old outlier_detection job": + - skip: + version: "7.5.0 - " + reason: waiting merge of https://github.com/elastic/elasticsearch/pull/47993 - do: ml.get_data_frame_analytics: id: "old_cluster_outlier_detection_job" @@ -20,6 +23,9 @@ --- "Get old outlier_detection job stats": + - skip: + version: "7.5.0 - " + reason: waiting merge of https://github.com/elastic/elasticsearch/pull/47993 - do: ml.get_data_frame_analytics_stats: id: "old_cluster_outlier_detection_job" @@ -30,6 +36,9 @@ --- "Start and stop old outlier_detection job": + - skip: + version: "7.5.0 - " + reason: waiting merge of https://github.com/elastic/elasticsearch/pull/47993 - do: ml.start_data_frame_analytics: id: "old_cluster_outlier_detection_job" @@ -50,6 +59,9 @@ --- "Get old regression job": + - skip: + version: "7.5.0 - " + reason: waiting merge of https://github.com/elastic/elasticsearch/pull/47993 - do: ml.get_data_frame_analytics: id: "old_cluster_regression_job" @@ -63,6 +75,9 @@ --- "Get old regression job stats": + - skip: + version: "7.5.0 - " + reason: waiting merge of https://github.com/elastic/elasticsearch/pull/47993 - do: ml.get_data_frame_analytics_stats: id: "old_cluster_regression_job" @@ -73,6 +88,9 @@ --- "Start and stop old regression job": + - skip: + version: "7.5.0 - " + reason: waiting merge of https://github.com/elastic/elasticsearch/pull/47993 - do: ml.start_data_frame_analytics: id: "old_cluster_regression_job" @@ -93,6 +111,9 @@ --- "Put an outlier_detection job on the mixed cluster": + - skip: + version: "7.5.0 - " + reason: waiting merge of https://github.com/elastic/elasticsearch/pull/47993 - do: ml.put_data_frame_analytics: id: "mixed_cluster_outlier_detection_job" diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/90_ml_data_frame_analytics_crud.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/90_ml_data_frame_analytics_crud.yml index d4c2d63ec500b..47ff23c3b60cd 100644 --- a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/90_ml_data_frame_analytics_crud.yml +++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/90_ml_data_frame_analytics_crud.yml @@ -1,6 +1,9 @@ --- "Get old cluster outlier_detection job": + - skip: + version: "7.5.0 - " + reason: waiting merge of https://github.com/elastic/elasticsearch/pull/47993 - do: ml.get_data_frame_analytics: id: "old_cluster_outlier_detection_job" @@ -20,6 +23,9 @@ --- "Get old cluster outlier_detection job stats": + - skip: + version: "7.5.0 - " + reason: waiting merge of https://github.com/elastic/elasticsearch/pull/47993 - do: ml.get_data_frame_analytics_stats: id: "old_cluster_outlier_detection_job" @@ -30,6 +36,9 @@ --- "Get old cluster regression job": + - skip: + version: "7.5.0 - " + reason: waiting merge of https://github.com/elastic/elasticsearch/pull/47993 - do: ml.get_data_frame_analytics: id: "old_cluster_regression_job" @@ -43,6 +52,9 @@ --- "Get old cluster regression job stats": + - skip: + version: "7.5.0 - " + reason: waiting merge of https://github.com/elastic/elasticsearch/pull/47993 - do: ml.get_data_frame_analytics_stats: id: "old_cluster_regression_job" @@ -53,6 +65,9 @@ --- "Get mixed cluster outlier_detection job": + - skip: + version: "7.5.0 - " + reason: waiting merge of https://github.com/elastic/elasticsearch/pull/47993 - do: ml.get_data_frame_analytics: id: "mixed_cluster_outlier_detection_job" @@ -72,6 +87,9 @@ --- "Get mixed cluster outlier_detection job stats": + - skip: + version: "7.5.0 - " + reason: waiting merge of https://github.com/elastic/elasticsearch/pull/47993 - do: ml.get_data_frame_analytics_stats: id: "mixed_cluster_outlier_detection_job" From bea5de20bfb3a2d25d930b4bd96062b99c05c8eb Mon Sep 17 00:00:00 2001 From: David Roberts Date: Mon, 14 Oct 2019 14:49:04 +0100 Subject: [PATCH 4/5] Adjust integration test --- .../xpack/ml/integration/MlDistributedFailureIT.java | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/MlDistributedFailureIT.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/MlDistributedFailureIT.java index aedbda0917706..3509d54999db9 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/MlDistributedFailureIT.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/MlDistributedFailureIT.java @@ -5,7 +5,6 @@ */ package org.elasticsearch.xpack.ml.integration; -import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.MetaData; @@ -168,16 +167,9 @@ public void testCloseUnassignedJobAndDatafeed() throws Exception { StopDatafeedAction.Response stopDatafeedResponse = client().execute(StopDatafeedAction.INSTANCE, stopDatafeedRequest).actionGet(); assertTrue(stopDatafeedResponse.isStopped()); - // Can't normal stop an unassigned job + // Since 7.5 we can also stop an unassigned job either normally or by force CloseJobAction.Request closeJobRequest = new CloseJobAction.Request(jobId); - ElasticsearchStatusException statusException = expectThrows(ElasticsearchStatusException.class, - () -> client().execute(CloseJobAction.INSTANCE, closeJobRequest).actionGet()); - assertEquals("Cannot close job [" + jobId + - "] because the job does not have an assigned node. Use force close to close the job", - statusException.getMessage()); - - // Can only force close an unassigned job - closeJobRequest.setForce(true); + closeJobRequest.setForce(randomBoolean()); CloseJobAction.Response closeJobResponse = client().execute(CloseJobAction.INSTANCE, closeJobRequest).actionGet(); assertTrue(closeJobResponse.isClosed()); } From ddbce106a653d861b4d0bc0ed9202d375fe3f44b Mon Sep 17 00:00:00 2001 From: David Roberts Date: Mon, 14 Oct 2019 16:48:34 +0100 Subject: [PATCH 5/5] Mute more backport-sensitive BWC tests --- .../test/mixed_cluster/30_ml_jobs_crud.yml | 15 +++++++++++++++ .../test/upgraded_cluster/30_ml_jobs_crud.yml | 12 ++++++++++++ 2 files changed, 27 insertions(+) diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/30_ml_jobs_crud.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/30_ml_jobs_crud.yml index 812f4d2683170..22751b620467e 100644 --- a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/30_ml_jobs_crud.yml +++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/30_ml_jobs_crud.yml @@ -1,5 +1,8 @@ --- "Test get old cluster job": + - skip: + version: "7.5.0 - " + reason: waiting merge of https://github.com/elastic/elasticsearch/pull/47993 - do: ml.get_jobs: job_id: old-cluster-job @@ -37,6 +40,9 @@ --- "Test get old cluster job's timing stats": + - skip: + version: "7.5.0 - " + reason: waiting merge of https://github.com/elastic/elasticsearch/pull/47993 - do: ml.get_job_stats: job_id: old-cluster-job-with-ts @@ -51,6 +57,9 @@ --- "Test get old cluster categorization job": + - skip: + version: "7.5.0 - " + reason: waiting merge of https://github.com/elastic/elasticsearch/pull/47993 - do: ml.get_jobs: job_id: old-cluster-categorization-job @@ -88,6 +97,9 @@ --- "Create a job in the mixed cluster and write some data": + - skip: + version: "7.5.0 - " + reason: waiting merge of https://github.com/elastic/elasticsearch/pull/47993 - do: ml.put_job: job_id: mixed-cluster-job @@ -141,6 +153,9 @@ --- "Test job with pre 6.4 rules": + - skip: + version: "7.5.0 - " + reason: waiting merge of https://github.com/elastic/elasticsearch/pull/47993 - do: ml.get_jobs: job_id: job-with-old-rules diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/30_ml_jobs_crud.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/30_ml_jobs_crud.yml index b558583bd066e..2c831d91cc802 100644 --- a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/30_ml_jobs_crud.yml +++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/30_ml_jobs_crud.yml @@ -8,6 +8,9 @@ setup: --- "Test open old jobs": + - skip: + version: "7.5.0 - " + reason: waiting merge of https://github.com/elastic/elasticsearch/pull/47993 - do: ml.open_job: job_id: old-cluster-job @@ -111,6 +114,9 @@ setup: --- "Test get old cluster job's timing stats": + - skip: + version: "7.5.0 - " + reason: waiting merge of https://github.com/elastic/elasticsearch/pull/47993 - do: ml.get_job_stats: job_id: old-cluster-job-with-ts @@ -136,6 +142,9 @@ setup: --- "Test job with pre 6.4 rules": + - skip: + version: "7.5.0 - " + reason: waiting merge of https://github.com/elastic/elasticsearch/pull/47993 - do: ml.get_jobs: job_id: job-with-old-rules @@ -146,6 +155,9 @@ setup: --- "Test get job with function shortcut should expand": + - skip: + version: "7.5.0 - " + reason: waiting merge of https://github.com/elastic/elasticsearch/pull/47993 - do: ml.get_jobs: job_id: old-cluster-function-shortcut-expansion