From 348a45cc7dc7ef561e0852b817aecf5fc2469f0c Mon Sep 17 00:00:00 2001 From: David Kyle Date: Fri, 15 Aug 2025 16:52:34 +0100 Subject: [PATCH 1/8] Compare int to int --- muted-tests.yml | 3 --- .../elasticsearch/upgrades/TransformSurvivesUpgradeIT.java | 6 +++--- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/muted-tests.yml b/muted-tests.yml index 9b66edc80eb1e..8b00ce91d27dd 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -510,9 +510,6 @@ tests: - class: org.elasticsearch.packaging.test.ArchiveGenerateInitialCredentialsTests method: test50CredentialAutogenerationOnlyOnce issue: https://github.com/elastic/elasticsearch/issues/132878 -- class: org.elasticsearch.upgrades.TransformSurvivesUpgradeIT - method: testTransformRollingUpgrade - issue: https://github.com/elastic/elasticsearch/issues/132892 - class: org.elasticsearch.index.mapper.LongFieldMapperTests method: testFetchCoerced issue: https://github.com/elastic/elasticsearch/issues/132893 diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TransformSurvivesUpgradeIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TransformSurvivesUpgradeIT.java index ee9954429df1f..f9876ccd5d614 100644 --- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TransformSurvivesUpgradeIT.java +++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TransformSurvivesUpgradeIT.java @@ -130,11 +130,11 @@ private void createAndStartContinuousTransform() throws Exception { assertBusy(() -> { var stateAndStats = getTransformStats(CONTINUOUS_TRANSFORM_ID); + assertThat((Integer) XContentMapValues.extractValue("stats.documents_indexed", stateAndStats), equalTo(ENTITIES.size())); assertThat( - ((Integer) XContentMapValues.extractValue("stats.documents_indexed", stateAndStats)).longValue(), - equalTo(ENTITIES.size()) + ((Integer) XContentMapValues.extractValue("stats.documents_processed", stateAndStats)).longValue(), + equalTo(totalDocsWritten) ); - assertThat((Integer) XContentMapValues.extractValue("stats.documents_processed", stateAndStats), equalTo(totalDocsWritten)); // Even if we get back to started, we may periodically get set back to `indexing` when triggered. // Though short lived due to no changes on the source indices, it could result in flaky test behavior assertThat(stateAndStats.get("state"), oneOf("started", "indexing")); From 53545b9a826d6c2d569bdcd632af44648a1bff52 Mon Sep 17 00:00:00 2001 From: David Kyle Date: Tue, 19 Aug 2025 10:50:05 +0100 Subject: [PATCH 2/8] use proper mixed version check --- .../transform/action/TransportUpgradeTransformsAction.java | 3 +-- .../org/elasticsearch/upgrades/TransformSurvivesUpgradeIT.java | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportUpgradeTransformsAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportUpgradeTransformsAction.java index b91cf6d3bb5b6..f8d6c399f9fda 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportUpgradeTransformsAction.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportUpgradeTransformsAction.java @@ -29,7 +29,6 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.XPackSettings; -import org.elasticsearch.xpack.core.ml.utils.TransportVersionUtils; import org.elasticsearch.xpack.core.security.SecurityContext; import org.elasticsearch.xpack.core.transform.TransformMetadata; import org.elasticsearch.xpack.core.transform.action.UpgradeTransformsAction; @@ -107,7 +106,7 @@ protected void masterOperation(Task ignoredTask, Request request, ClusterState s } // do not allow in mixed clusters - if (TransportVersionUtils.isMinTransportVersionSameAsCurrent(state) == false) { + if (state.nodes().isMixedVersionCluster()) { listener.onFailure( new ElasticsearchStatusException("Cannot upgrade transforms while cluster upgrade is in progress.", RestStatus.CONFLICT) ); diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TransformSurvivesUpgradeIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TransformSurvivesUpgradeIT.java index f9876ccd5d614..50f36d0a04e5e 100644 --- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TransformSurvivesUpgradeIT.java +++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TransformSurvivesUpgradeIT.java @@ -239,7 +239,7 @@ private void verifyUpgradeFailsIfMixedCluster() { final Request upgradeTransformRequest = new Request("POST", getTransformEndpoint() + "_upgrade"); Exception ex = expectThrows(Exception.class, () -> client().performRequest(upgradeTransformRequest)); - assertThat(ex.getMessage(), containsString("All nodes must be the same version")); + assertThat(ex.getMessage(), containsString("Cannot upgrade transforms while cluster upgrade is in progress")); } private void verifyUpgrade() throws IOException { From 56c99e1be8920b41123c70f77bc8e89b2d923873 Mon Sep 17 00:00:00 2001 From: David Kyle Date: Tue, 19 Aug 2025 15:21:00 +0100 Subject: [PATCH 3/8] WIP need to check version --- .../org/elasticsearch/upgrades/TransformSurvivesUpgradeIT.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TransformSurvivesUpgradeIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TransformSurvivesUpgradeIT.java index 50f36d0a04e5e..4a1f546166d9d 100644 --- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TransformSurvivesUpgradeIT.java +++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TransformSurvivesUpgradeIT.java @@ -9,6 +9,7 @@ import org.apache.http.HttpHost; import org.apache.http.entity.ContentType; import org.apache.http.entity.StringEntity; +import org.elasticsearch.Build; import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; import org.elasticsearch.client.RestClient; @@ -234,6 +235,7 @@ private void verifyContinuousTransformHandlesData(long expectedLastCheckpoint) t private void verifyUpgradeFailsIfMixedCluster() { // upgrade tests by design are also executed with the same version, this check must be skipped in this case, see gh#39102. if (isOriginalClusterCurrent()) { + Build.current().v return; } final Request upgradeTransformRequest = new Request("POST", getTransformEndpoint() + "_upgrade"); From 69cae2cf2cbcc2b5d37f79d57a725ed5cb97984e Mon Sep 17 00:00:00 2001 From: David Kyle Date: Tue, 19 Aug 2025 16:19:30 +0100 Subject: [PATCH 4/8] skip mixed cluster in older versions --- .../upgrades/TransformSurvivesUpgradeIT.java | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TransformSurvivesUpgradeIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TransformSurvivesUpgradeIT.java index 4a1f546166d9d..17cd9068869af 100644 --- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TransformSurvivesUpgradeIT.java +++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TransformSurvivesUpgradeIT.java @@ -10,6 +10,7 @@ import org.apache.http.entity.ContentType; import org.apache.http.entity.StringEntity; import org.elasticsearch.Build; +import org.elasticsearch.Version; import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; import org.elasticsearch.client.RestClient; @@ -235,13 +236,14 @@ private void verifyContinuousTransformHandlesData(long expectedLastCheckpoint) t private void verifyUpgradeFailsIfMixedCluster() { // upgrade tests by design are also executed with the same version, this check must be skipped in this case, see gh#39102. if (isOriginalClusterCurrent()) { - Build.current().v return; } - final Request upgradeTransformRequest = new Request("POST", getTransformEndpoint() + "_upgrade"); - - Exception ex = expectThrows(Exception.class, () -> client().performRequest(upgradeTransformRequest)); - assertThat(ex.getMessage(), containsString("Cannot upgrade transforms while cluster upgrade is in progress")); + var oldestVersion = Version.fromString(UPGRADE_FROM_VERSION); + if (oldestVersion.onOrAfter(Version.V_9_2_0)) { + final Request upgradeTransformRequest = new Request("POST", getTransformEndpoint() + "_upgrade"); + Exception ex = expectThrows(Exception.class, () -> client().performRequest(upgradeTransformRequest)); + assertThat(ex.getMessage(), containsString("Cannot upgrade transforms while cluster upgrade is in progress")); + } } private void verifyUpgrade() throws IOException { From 7faea1b62a48ec44661d33f6ecd48246651f194f Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Tue, 19 Aug 2025 15:27:14 +0000 Subject: [PATCH 5/8] [CI] Auto commit changes from spotless --- .../org/elasticsearch/upgrades/TransformSurvivesUpgradeIT.java | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TransformSurvivesUpgradeIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TransformSurvivesUpgradeIT.java index 17cd9068869af..8fcaf6071f524 100644 --- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TransformSurvivesUpgradeIT.java +++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TransformSurvivesUpgradeIT.java @@ -9,7 +9,6 @@ import org.apache.http.HttpHost; import org.apache.http.entity.ContentType; import org.apache.http.entity.StringEntity; -import org.elasticsearch.Build; import org.elasticsearch.Version; import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; From 2de81258e8d377f11ce9301b6944542b5179d9d6 Mon Sep 17 00:00:00 2001 From: David Kyle Date: Fri, 3 Oct 2025 12:13:56 +0100 Subject: [PATCH 6/8] update version check --- .../org/elasticsearch/upgrades/TransformSurvivesUpgradeIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TransformSurvivesUpgradeIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TransformSurvivesUpgradeIT.java index 8fcaf6071f524..5898cb5711b9d 100644 --- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TransformSurvivesUpgradeIT.java +++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/TransformSurvivesUpgradeIT.java @@ -238,7 +238,7 @@ private void verifyUpgradeFailsIfMixedCluster() { return; } var oldestVersion = Version.fromString(UPGRADE_FROM_VERSION); - if (oldestVersion.onOrAfter(Version.V_9_2_0)) { + if (oldestVersion.onOrAfter(Version.V_9_3_0)) { final Request upgradeTransformRequest = new Request("POST", getTransformEndpoint() + "_upgrade"); Exception ex = expectThrows(Exception.class, () -> client().performRequest(upgradeTransformRequest)); assertThat(ex.getMessage(), containsString("Cannot upgrade transforms while cluster upgrade is in progress")); From 84329e405a2444a4aeb9187a7e7c9264d80fd0c1 Mon Sep 17 00:00:00 2001 From: David Kyle Date: Fri, 3 Oct 2025 15:51:28 +0100 Subject: [PATCH 7/8] ignore system indices --- .../IndexBasedTransformConfigManager.java | 19 +++++++++++-------- .../upgrades/MLModelDeploymentsUpgradeIT.java | 4 ++-- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/IndexBasedTransformConfigManager.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/IndexBasedTransformConfigManager.java index 2d121d95b4da8..ce55622938c6a 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/IndexBasedTransformConfigManager.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/IndexBasedTransformConfigManager.java @@ -295,16 +295,19 @@ public void deleteOldIndices(ActionListener listener) { // use the transform context as we access system indexes try (ThreadContext.StoredContext ctx = client.threadPool().getThreadContext().stashWithOrigin(TRANSFORM_ORIGIN)) { - indicesToDelete.addAll( - Arrays.asList( - indexNameExpressionResolver.concreteIndexNames( - state, - IndicesOptions.lenientExpandHidden(), - TransformInternalIndexConstants.INDEX_NAME_PATTERN - ) - ) + var matchingIndexes = indexNameExpressionResolver.concreteIndices( + state, + IndicesOptions.lenientExpandHidden(), + TransformInternalIndexConstants.INDEX_NAME_PATTERN ); + for (var index : matchingIndexes) { + var meta = state.getMetadata().indexMetadata(index); + if (meta.isSystem() == false) { // ignore system indices as these are automatically managed + indicesToDelete.add(meta.getIndex().getName()); + } + } + indicesToDelete.addAll( Arrays.asList( indexNameExpressionResolver.concreteIndexNames( diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MLModelDeploymentsUpgradeIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MLModelDeploymentsUpgradeIT.java index adeaeae31be45..06a7f9f4047fe 100644 --- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MLModelDeploymentsUpgradeIT.java +++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MLModelDeploymentsUpgradeIT.java @@ -73,10 +73,10 @@ public void setUpLogging() throws IOException { request.setJsonEntity(""" { "persistent": { - "logger.org.elasticsearch.xpack.ml.inference": "TRACE", + "logger.org.elasticsearch.xpack.ml.inference": "DEBUG", "logger.org.elasticsearch.xpack.ml.inference.assignments": "DEBUG", "logger.org.elasticsearch.xpack.ml.process": "DEBUG", - "logger.org.elasticsearch.xpack.ml.action": "TRACE" + "logger.org.elasticsearch.xpack.ml.action": "DEBUG" } } """); From 462aa3c31e9f75a37b6a6b1605abcf8178495cb0 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Wed, 8 Oct 2025 10:01:07 +0000 Subject: [PATCH 8/8] [CI] Auto commit changes from spotless --- .../xpack/transform/action/TransportUpgradeTransformsAction.java | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportUpgradeTransformsAction.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportUpgradeTransformsAction.java index baab715fab887..f8d6c399f9fda 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportUpgradeTransformsAction.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/action/TransportUpgradeTransformsAction.java @@ -11,7 +11,6 @@ import org.apache.logging.log4j.Logger; import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.ResourceNotFoundException; -import org.elasticsearch.TransportVersion; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.master.TransportMasterNodeAction;