Merge branch 'master' into remove-failing-sanity-check

Author: joegallo

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/RestTestBasePlugin.java

@@ -34,6 +34,8 @@ public class RestTestBasePlugin implements Plugin<Project> {
     private static final String TESTS_REST_CLUSTER = "tests.rest.cluster";
     private static final String TESTS_CLUSTER = "tests.cluster";
     private static final String TESTS_CLUSTER_NAME = "tests.clustername";
+    private static final String TESTS_CLUSTER_READINESS = "tests.cluster.readiness";
+
     private ProviderFactory providerFactory;
 
     @Inject
@@ -66,6 +68,7 @@ public void apply(Project project) {
                 runnerNonInputProperties.systemProperty(TESTS_REST_CLUSTER, () -> String.join(",", cluster.getAllHttpSocketURI()));
                 runnerNonInputProperties.systemProperty(TESTS_CLUSTER, () -> String.join(",", cluster.getAllTransportPortURI()));
                 runnerNonInputProperties.systemProperty(TESTS_CLUSTER_NAME, cluster::getName);
+                runnerNonInputProperties.systemProperty(TESTS_CLUSTER_READINESS, () -> String.join(",", cluster.getAllReadinessPortURI()));
             } else {
                 if (systemProperty(TESTS_CLUSTER) == null || systemProperty(TESTS_CLUSTER_NAME) == null) {
                     throw new IllegalArgumentException(

build-tools/src/main/java/org/elasticsearch/gradle/test/JavaRestTestPlugin.java

@@ -62,6 +62,7 @@ public void apply(Project project) {
                 nonInputProperties.systemProperty("tests.rest.cluster", () -> String.join(",", cluster.getAllHttpSocketURI()));
                 nonInputProperties.systemProperty("tests.cluster", () -> String.join(",", cluster.getAllTransportPortURI()));
                 nonInputProperties.systemProperty("tests.clustername", () -> cluster.getName());
+                nonInputProperties.systemProperty("tests.cluster.readiness", () -> String.join(",", cluster.getAllReadinessPortURI()));
                 task.getJvmArgumentProviders().add(nonInputProperties);
             });
 

build-tools/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchCluster.java

@@ -146,6 +146,14 @@ public void setNumberOfNodes(int numberOfNodes) {
         }
     }
 
+    public void setReadinessEnabled(boolean enabled) {
+        if (enabled) {
+            for (ElasticsearchNode node : nodes) {
+                node.setting("readiness.port", "0"); // ephemeral port
+            }
+        }
+    }
+
     @Internal
     public ElasticsearchNode getFirstNode() {
         return nodes.getAt(clusterName + "-0");
@@ -455,6 +463,13 @@ public String getTransportPortURI() {
         return getFirstNode().getTransportPortURI();
     }
 
+    @Override
+    @Internal
+    public String getReadinessPortURI() {
+        waitForAllConditions();
+        return getFirstNode().getReadinessPortURI();
+    }
+
     @Override
     @Internal
     public List<String> getAllHttpSocketURI() {
@@ -469,6 +484,13 @@ public List<String> getAllTransportPortURI() {
         return nodes.stream().flatMap(each -> each.getAllTransportPortURI().stream()).collect(Collectors.toList());
     }
 
+    @Override
+    @Internal
+    public List<String> getAllReadinessPortURI() {
+        waitForAllConditions();
+        return nodes.stream().flatMap(each -> each.getAllReadinessPortURI().stream()).collect(Collectors.toList());
+    }
+
     public void waitForAllConditions() {
         writeUnicastHostsFiles();
 

build-tools/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java

@@ -159,6 +159,7 @@ public class ElasticsearchNode implements TestClusterConfiguration {
     private final Path confPathLogs;
     private final Path transportPortFile;
     private final Path httpPortsFile;
+    private final Path readinessPortsFile;
     private final Path esLogFile;
     private final Path esStdinFile;
     private final Path tmpDir;
@@ -209,6 +210,7 @@ public class ElasticsearchNode implements TestClusterConfiguration {
         confPathLogs = workingDir.resolve("logs");
         transportPortFile = confPathLogs.resolve("transport.ports");
         httpPortsFile = confPathLogs.resolve("http.ports");
+        readinessPortsFile = confPathLogs.resolve("readiness.ports");
         esLogFile = confPathLogs.resolve(clusterName + ".log");
         esStdinFile = workingDir.resolve("es.stdin");
         tmpDir = workingDir.resolve("tmp");
@@ -955,6 +957,12 @@ public String getTransportPortURI() {
         return getTransportPortInternal().get(0);
     }
 
+    @Override
+    @Internal
+    public String getReadinessPortURI() {
+        return getReadinessPortInternal().get(0);
+    }
+
     @Override
     @Internal
     public List<String> getAllHttpSocketURI() {
@@ -969,6 +977,13 @@ public List<String> getAllTransportPortURI() {
         return getTransportPortInternal();
     }
 
+    @Override
+    @Internal
+    public List<String> getAllReadinessPortURI() {
+        waitForAllConditions();
+        return getReadinessPortInternal();
+    }
+
     @Internal
     public File getServerLog() {
         return confPathLogs.resolve(defaultConfig.get("cluster.name") + "_server.json").toFile();
@@ -989,6 +1004,9 @@ public synchronized void stop(boolean tailLogs) {
             if (Files.exists(transportPortFile)) {
                 Files.delete(transportPortFile);
             }
+            if (Files.exists(readinessPortsFile)) {
+                Files.delete(readinessPortsFile);
+            }
         } catch (IOException e) {
             throw new UncheckedIOException(e);
         }
@@ -1011,6 +1029,9 @@ public synchronized void stop(boolean tailLogs) {
             if (Files.exists(transportPortFile)) {
                 Files.delete(transportPortFile);
             }
+            if (Files.exists(readinessPortsFile)) {
+                Files.delete(readinessPortsFile);
+            }
         } catch (IOException e) {
             throw new UncheckedIOException(e);
         }
@@ -1409,6 +1430,14 @@ private List<String> getHttpPortInternal() {
         }
     }
 
+    private List<String> getReadinessPortInternal() {
+        try {
+            return readPortsFile(readinessPortsFile);
+        } catch (IOException e) {
+            return new ArrayList<>();
+        }
+    }
+
     private List<String> readPortsFile(Path file) throws IOException {
         try (Stream<String> lines = Files.lines(file, StandardCharsets.UTF_8)) {
             return lines.map(String::trim).collect(Collectors.toList());

build-tools/src/main/java/org/elasticsearch/gradle/testclusters/TestClusterConfiguration.java

@@ -113,10 +113,14 @@ public interface TestClusterConfiguration {
 
     String getTransportPortURI();
 
+    String getReadinessPortURI();
+
     List<String> getAllHttpSocketURI();
 
     List<String> getAllTransportPortURI();
 
+    List<String> getAllReadinessPortURI();
+
     void stop(boolean tailLogs);
 
     void setNameCustomization(Function<String, String> nameSupplier);

distribution/src/config/elasticsearch.yml

@@ -75,6 +75,12 @@
 #
 # For more information, consult the discovery and cluster formation module documentation.
 #
+# --------------------------------- Readiness ----------------------------------
+#
+# Enable an unauthenticated TCP readiness endpoint on localhost
+#
+#readiness.port: 9399
+#
 # ---------------------------------- Various -----------------------------------
 #
 # Allow wildcard deletion of indices:

docs/changelog/84375.yaml

@@ -0,0 +1,6 @@
+pr: 84375
+summary: Introduce an unauthenticated endpoint for readiness checks
+area: Infra/Core
+type: feature
+issues:
+ - 81168

docs/changelog/85383.yaml

@@ -0,0 +1,5 @@
+pr: 85383
+summary: Fixes to making old ML indices hidden
+area: Machine Learning
+type: bug
+issues: []

modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java

@@ -119,7 +119,6 @@ public void testGetAdditionalIndexSettingsDataStreamAlreadyCreatedTimeSettingsMi
         mb.put(
             new DataStream(
                 ds.getName(),
-                ds.getTimeStampField(),
                 ds.getIndices(),
                 ds.getGeneration(),
                 ds.getMetadata(),

modules/data-streams/src/test/java/org/elasticsearch/datastreams/MetadataDataStreamRolloverServiceTests.java

@@ -53,7 +53,6 @@ public void testRolloverClusterStateForDataStream() throws Exception {
         String dataStreamName = "logs-my-app";
         final DataStream dataStream = new DataStream(
             dataStreamName,
-            new DataStream.TimestampField("@timestamp"),
             List.of(new Index(DataStream.getDefaultBackingIndexName(dataStreamName, 1, now.toEpochMilli()), "uuid")),
             1,
             null,
@@ -152,7 +151,6 @@ public void testRolloverAndMigrateDataStream() throws Exception {
         IndexMode dsIndexMode = randomBoolean() ? null : IndexMode.STANDARD;
         final DataStream dataStream = new DataStream(
             dataStreamName,
-            new DataStream.TimestampField("@timestamp"),
             List.of(new Index(DataStream.getDefaultBackingIndexName(dataStreamName, 1, now.toEpochMilli()), "uuid")),
             1,
             null,
@@ -237,7 +235,6 @@ public void testChangingIndexModeFromTimeSeriesToSomethingElseNoEffectOnExisting
         String dataStreamName = "logs-my-app";
         final DataStream dataStream = new DataStream(
             dataStreamName,
-            new DataStream.TimestampField("@timestamp"),
             List.of(new Index(DataStream.getDefaultBackingIndexName(dataStreamName, 1, now.toEpochMilli()), "uuid")),
             1,
             null,