Merge branch 'upstream/master' into reuse-date-stream-timestamp

* upstream/master: (24 commits)
  Implement framework for migrating system indices (elastic#78951)
  Improve transient settings deprecation message (elastic#79504)
  Remove getValue and getValues from Field (elastic#79516)
  Store Template's mappings as bytes for disk serialization (elastic#78746)
  [ML] Add queue_capacity setting to start deployment API (elastic#79433)
  [ML] muting rest compat test issue elastic#79518 (elastic#79519)
  Avoid redundant available indices check (elastic#76540)
  Re-enable BWC tests
  TEST Ensure password 14 chars length on Kerberos FIPS tests (elastic#79496)
  [DOCS] Temporarily remove APM links (elastic#79411)
  Fix CCSDuelIT for skipped shards (elastic#79490)
  Add other time accounting in HotThreads (elastic#79392)
  Add deprecation info API entries for deprecated monitoring settings (elastic#78799)
  Add note in breaking changes for nameid_format (elastic#77785)
  Use 'migration' instead of 'upgrade' in GET system feature migration status responses (elastic#79302)
  Upgrade lucene version 8b68bf60c98 (elastic#79461)
  Use Strings#EMPTY_ARRAY (elastic#79452)
  Quicker shared cache file preallocation (elastic#79447)
  [ML] Removing some code that's obsolete for 8.0 (elastic#79444)
  Ensure indexing_data CCR requests are compressed (elastic#79413)
  ...

Author: weizijun

build-tools-internal/version.properties

@@ -1,5 +1,5 @@
 elasticsearch     = 8.0.0
-lucene            = 9.0.0-snapshot-cfd9f9f98f7
+lucene            = 9.0.0-snapshot-8b68bf60c98
 
 bundled_jdk_vendor = adoptium
 bundled_jdk = 17+35

build.gradle

@@ -132,9 +132,9 @@ tasks.register("verifyVersions") {
  * after the backport of the backcompat code is complete.
  */
 
-boolean bwc_tests_enabled = false
+boolean bwc_tests_enabled = true
 // place a PR link here when committing bwc changes:
-String bwc_tests_disabled_issue = "https://github.com/elastic/elasticsearch/pull/79385"
+String bwc_tests_disabled_issue = ""
 /*
  * FIPS 140-2 behavior was fixed in 7.11.0. Before that there is no way to run elasticsearch in a
  * JVM that is properly configured to be in fips mode with BCFIPS. For now we need to disable

client/rest-high-level/src/main/java/org/elasticsearch/client/migration/GetFeatureUpgradeStatusResponse.java

@@ -25,7 +25,7 @@
 public class GetFeatureUpgradeStatusResponse {
 
     private static final ParseField FEATURE_UPGRADE_STATUSES = new ParseField("features");
-    private static final ParseField UPGRADE_STATUS = new ParseField("upgrade_status");
+    private static final ParseField UPGRADE_STATUS = new ParseField("migration_status");
 
     private final List<FeatureUpgradeStatus> featureUpgradeStatuses;
     private final String upgradeStatus;
@@ -76,7 +76,7 @@ public static class FeatureUpgradeStatus {
 
         private static final ParseField FEATURE_NAME = new ParseField("feature_name");
         private static final ParseField MINIMUM_INDEX_VERSION = new ParseField("minimum_index_version");
-        private static final ParseField UPGRADE_STATUS = new ParseField("upgrade_status");
+        private static final ParseField UPGRADE_STATUS = new ParseField("migration_status");
         private static final ParseField INDEX_VERSIONS = new ParseField("indices");
 
         @SuppressWarnings("unchecked")

client/rest-high-level/src/test/java/org/elasticsearch/client/MigrationIT.java

@@ -23,8 +23,8 @@
 
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.is;
-import static org.hamcrest.Matchers.nullValue;
 
 public class MigrationIT extends ESRestHighLevelClientTestCase {
 
@@ -42,7 +42,7 @@ public void testGetDeprecationInfo() throws IOException {
     public void testGetFeatureUpgradeStatus() throws IOException {
         GetFeatureUpgradeStatusRequest request = new GetFeatureUpgradeStatusRequest();
         GetFeatureUpgradeStatusResponse response = highLevelClient().migration().getFeatureUpgradeStatus(request, RequestOptions.DEFAULT);
-        assertThat(response.getUpgradeStatus(), equalTo("NO_UPGRADE_NEEDED"));
+        assertThat(response.getUpgradeStatus(), equalTo("NO_MIGRATION_NEEDED"));
         assertThat(response.getFeatureUpgradeStatuses().size(), greaterThanOrEqualTo(1));
         Optional<GetFeatureUpgradeStatusResponse.FeatureUpgradeStatus> optionalTasksStatus = response.getFeatureUpgradeStatuses().stream()
             .filter(status -> "tasks".equals(status.getFeatureName()))
@@ -52,19 +52,16 @@ public void testGetFeatureUpgradeStatus() throws IOException {
 
         GetFeatureUpgradeStatusResponse.FeatureUpgradeStatus tasksStatus = optionalTasksStatus.get();
 
-        assertThat(tasksStatus.getUpgradeStatus(), equalTo("NO_UPGRADE_NEEDED"));
+        assertThat(tasksStatus.getUpgradeStatus(), equalTo("NO_MIGRATION_NEEDED"));
         assertThat(tasksStatus.getMinimumIndexVersion(), equalTo(Version.CURRENT.toString()));
         assertThat(tasksStatus.getFeatureName(), equalTo("tasks"));
     }
 
     public void testPostFeatureUpgradeStatus() throws IOException {
         PostFeatureUpgradeRequest request = new PostFeatureUpgradeRequest();
         PostFeatureUpgradeResponse response = highLevelClient().migration().postFeatureUpgrade(request, RequestOptions.DEFAULT);
-        assertThat(response.isAccepted(), equalTo(true));
-        assertThat(response.getFeatures().size(), equalTo(1));
-        PostFeatureUpgradeResponse.Feature feature = response.getFeatures().get(0);
-        assertThat(feature.getFeatureName(), equalTo("security"));
-        assertThat(response.getReason(), nullValue());
-        assertThat(response.getElasticsearchException(), nullValue());
+        assertThat(response.isAccepted(), equalTo(false));
+        assertThat(response.getFeatures(), hasSize(0));
+        assertThat(response.getReason(), equalTo("No system indices require migration"));
     }
 }

client/rest-high-level/src/test/java/org/elasticsearch/client/indices/GetComponentTemplatesResponseTests.java

@@ -13,6 +13,7 @@
 import org.elasticsearch.cluster.metadata.Template;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.test.ESTestCase;
 
@@ -79,7 +80,7 @@ private static void toXContent(GetComponentTemplatesResponse response, XContentB
             builder.startObject();
             builder.field("name", e.getKey());
             builder.field("component_template");
-            e.getValue().toXContent(builder, null);
+            e.getValue().toXContent(builder, ToXContent.EMPTY_PARAMS);
             builder.endObject();
         }
         builder.endArray();

client/rest-high-level/src/test/java/org/elasticsearch/client/indices/GetComposableIndexTemplatesResponseTests.java

@@ -10,6 +10,7 @@
 
 import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
 import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
@@ -53,7 +54,7 @@ private static void toXContent(GetComposableIndexTemplatesResponse response, XCo
             builder.startObject();
             builder.field("name", e.getKey());
             builder.field("index_template");
-            e.getValue().toXContent(builder, null);
+            e.getValue().toXContent(builder, ToXContent.EMPTY_PARAMS);
             builder.endObject();
         }
         builder.endArray();

client/rest-high-level/src/test/java/org/elasticsearch/client/migration/GetFeatureUpgradeStatusResponseTests.java

@@ -40,10 +40,14 @@ protected org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStat
                     randomAlphaOfLengthBetween(3, 20),
                     randomFrom(Version.CURRENT, Version.CURRENT.minimumCompatibilityVersion()),
                     randomFrom(org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusResponse.UpgradeStatus.values()),
-                    randomList(4,
-                        () -> new org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusResponse.IndexVersion(
+                    randomList(
+                        4,
+                        () -> new org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusResponse.IndexInfo(
                             randomAlphaOfLengthBetween(3, 20),
-                            randomFrom(Version.CURRENT, Version.CURRENT.minimumCompatibilityVersion())))
+                            randomFrom(Version.CURRENT, Version.CURRENT.minimumCompatibilityVersion()),
+                            null
+                        )
+                    )
                 )),
             randomFrom(org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusResponse.UpgradeStatus.values())
         );
@@ -78,12 +82,12 @@ protected void assertInstances(
             assertThat(clientStatus.getIndexVersions(), hasSize(serverTestStatus.getIndexVersions().size()));
 
             for (int j = 0; i < clientStatus.getIndexVersions().size(); i++) {
-                org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusResponse.IndexVersion serverIndexVersion
+                org.elasticsearch.action.admin.cluster.migration.GetFeatureUpgradeStatusResponse.IndexInfo serverIndexInfo
                     = serverTestStatus.getIndexVersions().get(j);
                 GetFeatureUpgradeStatusResponse.IndexVersion clientIndexVersion = clientStatus.getIndexVersions().get(j);
 
-                assertThat(clientIndexVersion.getIndexName(), equalTo(serverIndexVersion.getIndexName()));
-                assertThat(clientIndexVersion.getVersion(), equalTo(serverIndexVersion.getVersion().toString()));
+                assertThat(clientIndexVersion.getIndexName(), equalTo(serverIndexInfo.getIndexName()));
+                assertThat(clientIndexVersion.getVersion(), equalTo(serverIndexInfo.getVersion().toString()));
             }
         }
     }

docs/reference/migration/apis/feature_upgrade.asciidoc

@@ -24,7 +24,7 @@ and to trigger an automated system upgrade that might potentially involve downti
 ==== {api-prereq-title}
 
 * If the {es} {security-features} are enabled, you must have the `manage`
-<<privileges-list-cluster,cluster privilege>> to use this API. (TODO: true?)
+<<privileges-list-cluster,cluster privilege>> to use this API.
 
 [[feature-upgrade-api-example]]
 ==== {api-examples-title}
@@ -46,79 +46,80 @@ Example response:
     {
       "feature_name" : "async_search",
       "minimum_index_version" : "8.0.0",
-      "upgrade_status" : "NO_UPGRADE_NEEDED",
+      "migration_status" : "NO_MIGRATION_NEEDED",
       "indices" : [ ]
     },
     {
       "feature_name" : "enrich",
       "minimum_index_version" : "8.0.0",
-      "upgrade_status" : "NO_UPGRADE_NEEDED",
+      "migration_status" : "NO_MIGRATION_NEEDED",
       "indices" : [ ]
     },
     {
       "feature_name" : "fleet",
       "minimum_index_version" : "8.0.0",
-      "upgrade_status" : "NO_UPGRADE_NEEDED",
+      "migration_status" : "NO_MIGRATION_NEEDED",
       "indices" : [ ]
     },
     {
       "feature_name" : "geoip",
       "minimum_index_version" : "8.0.0",
-      "upgrade_status" : "NO_UPGRADE_NEEDED",
+      "migration_status" : "NO_MIGRATION_NEEDED",
       "indices" : [ ]
     },
     {
       "feature_name" : "kibana",
       "minimum_index_version" : "8.0.0",
-      "upgrade_status" : "NO_UPGRADE_NEEDED",
+      "migration_status" : "NO_MIGRATION_NEEDED",
       "indices" : [ ]
     },
     {
       "feature_name" : "logstash_management",
       "minimum_index_version" : "8.0.0",
-      "upgrade_status" : "NO_UPGRADE_NEEDED",
+      "migration_status" : "NO_MIGRATION_NEEDED",
       "indices" : [ ]
     },
     {
       "feature_name" : "machine_learning",
       "minimum_index_version" : "8.0.0",
-      "upgrade_status" : "NO_UPGRADE_NEEDED",
+      "migration_status" : "NO_MIGRATION_NEEDED",
       "indices" : [ ]
     },
     {
       "feature_name" : "searchable_snapshots",
       "minimum_index_version" : "8.0.0",
-      "upgrade_status" : "NO_UPGRADE_NEEDED",
+      "migration_status" : "NO_MIGRATION_NEEDED",
       "indices" : [ ]
     },
     {
       "feature_name" : "security",
       "minimum_index_version" : "8.0.0",
-      "upgrade_status" : "NO_UPGRADE_NEEDED",
+      "migration_status" : "NO_MIGRATION_NEEDED",
       "indices" : [ ]
     },
     {
       "feature_name" : "tasks",
       "minimum_index_version" : "8.0.0",
-      "upgrade_status" : "NO_UPGRADE_NEEDED",
+      "migration_status" : "NO_MIGRATION_NEEDED",
       "indices" : [ ]
     },
     {
       "feature_name" : "transform",
       "minimum_index_version" : "8.0.0",
-      "upgrade_status" : "NO_UPGRADE_NEEDED",
+      "migration_status" : "NO_MIGRATION_NEEDED",
       "indices" : [ ]
     },
     {
       "feature_name" : "watcher",
       "minimum_index_version" : "8.0.0",
-      "upgrade_status" : "NO_UPGRADE_NEEDED",
+      "migration_status" : "NO_MIGRATION_NEEDED",
       "indices" : [ ]
     }
   ],
-  "upgrade_status" : "NO_UPGRADE_NEEDED"
+  "migration_status" : "NO_MIGRATION_NEEDED"
 }
 --------------------------------------------------
+// TESTRESPONSE[s/"minimum_index_version" : "8.0.0"/"minimum_index_version" : $body.$_path/]
 
 This response tells us that Elasticsearch security needs its internal
 indices upgraded before we can upgrade the cluster to 8.0.
@@ -143,6 +144,7 @@ Example response:
   ]
 }
 --------------------------------------------------
+// TESTRESPONSE[skip: can't actually upgrade system indices in these tests]
 
 This tells us that the security index is being upgraded. To check the
 overall status of the upgrade, call the endpoint with GET.

docs/reference/migration/migrate_8_0.asciidoc

@@ -25,7 +25,6 @@ coming[8.0.0]
 * <<breaking_80_ingest_changes>>
 * <<breaking_80_java_changes>>
 * <<breaking_80_mappings_changes>>
-* <<breaking_80_monitoring_changes>>
 * <<breaking_80_network_changes>>
 * <<breaking_80_node_changes>>
 * <<breaking_80_packaging_changes>>
@@ -128,7 +127,6 @@ include::migrate_8_0/indices.asciidoc[]
 include::migrate_8_0/ingest.asciidoc[]
 include::migrate_8_0/java.asciidoc[]
 include::migrate_8_0/mappings.asciidoc[]
-include::migrate_8_0/monitoring.asciidoc[]
 include::migrate_8_0/network.asciidoc[]
 include::migrate_8_0/node.asciidoc[]
 include::migrate_8_0/packaging.asciidoc[]