Unmute docker tests, add more logging and increase startup timeout (#131203)

Unmuting 33 DockerTests with added additional logging on failures.
Also, bumping docker startup timeout to account for slow security 
auto-configuration, and additionally moving CA certificates inside 
the retry while loop.

Resolves #129808
Resolves #123437
Resolves #120911
Resolves #120914
Resolves #120910
Resolves #120918
Resolves #120925
Resolves #124990
Resolves #116636
Resolves #125680
Resolves #116147
Resolves #126232
Resolves #126145
Resolves #116866
Resolves #126936
Resolves #127081
Resolves #127158
Resolves #127138
Resolves #128018
Resolves #128031
Resolves #128110
Resolves #128006
Resolves #128174
Resolves #128144
Resolves #128165
Resolves #128075
Resolves #128117
Resolves #128121
Resolves #128115
Resolves #128230
Resolves #128120
Resolves #128996
Resolves #128867

Author: slobodanadamovic

muted-tests.yml

@@ -140,21 +140,6 @@ tests:
 - class: org.elasticsearch.xpack.security.authc.service.ServiceAccountIT
   method: testAuthenticateShouldNotFallThroughInCaseOfFailure
   issue: https://github.com/elastic/elasticsearch/issues/120902
-- class: org.elasticsearch.packaging.test.DockerTests
-  method: test050BasicApiTests
-  issue: https://github.com/elastic/elasticsearch/issues/120911
-- class: org.elasticsearch.packaging.test.DockerTests
-  method: test140CgroupOsStatsAreAvailable
-  issue: https://github.com/elastic/elasticsearch/issues/120914
-- class: org.elasticsearch.packaging.test.DockerTests
-  method: test070BindMountCustomPathConfAndJvmOptions
-  issue: https://github.com/elastic/elasticsearch/issues/120910
-- class: org.elasticsearch.packaging.test.DockerTests
-  method: test071BindMountCustomPathWithDifferentUID
-  issue: https://github.com/elastic/elasticsearch/issues/120918
-- class: org.elasticsearch.packaging.test.DockerTests
-  method: test171AdditionalCliOptionsAreForwarded
-  issue: https://github.com/elastic/elasticsearch/issues/120925
 - class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT
   method: test {p0=nodes.stats/11_indices_metrics/indices mappings exact count test for indices level}
   issue: https://github.com/elastic/elasticsearch/issues/120950
@@ -171,9 +156,6 @@ tests:
 - class: org.elasticsearch.blocks.SimpleBlocksIT
   method: testConcurrentAddBlock
   issue: https://github.com/elastic/elasticsearch/issues/122324
-- class: org.elasticsearch.packaging.test.DockerTests
-  method: test151MachineDependentHeapWithSizeOverride
-  issue: https://github.com/elastic/elasticsearch/issues/123437
 - class: org.elasticsearch.action.admin.cluster.node.tasks.CancellableTasksIT
   method: testChildrenTasksCancelledOnTimeout
   issue: https://github.com/elastic/elasticsearch/issues/123568
@@ -207,12 +189,6 @@ tests:
 - class: org.elasticsearch.packaging.test.BootstrapCheckTests
   method: test10Install
   issue: https://github.com/elastic/elasticsearch/issues/124957
-- class: org.elasticsearch.packaging.test.DockerTests
-  method: test011SecurityEnabledStatus
-  issue: https://github.com/elastic/elasticsearch/issues/124990
-- class: org.elasticsearch.packaging.test.DockerTests
-  method: test012SecurityCanBeDisabled
-  issue: https://github.com/elastic/elasticsearch/issues/116636
 - class: org.elasticsearch.smoketest.MlWithSecurityIT
   method: test {yaml=ml/data_frame_analytics_crud/Test get stats on newly created config}
   issue: https://github.com/elastic/elasticsearch/issues/121726
@@ -222,9 +198,6 @@ tests:
 - class: org.elasticsearch.smoketest.MlWithSecurityIT
   method: test {yaml=ml/data_frame_analytics_cat_apis/Test cat data frame analytics single job with header}
   issue: https://github.com/elastic/elasticsearch/issues/125642
-- class: org.elasticsearch.packaging.test.DockerTests
-  method: test010Install
-  issue: https://github.com/elastic/elasticsearch/issues/125680
 - class: org.elasticsearch.xpack.test.rest.XPackRestIT
   method: test {p0=transform/transforms_start_stop/Test schedule_now on an already started transform}
   issue: https://github.com/elastic/elasticsearch/issues/120720
@@ -240,9 +213,6 @@ tests:
 - class: org.elasticsearch.xpack.test.rest.XPackRestIT
   method: test {p0=transform/transforms_stats/Test get transform stats with timeout}
   issue: https://github.com/elastic/elasticsearch/issues/125975
-- class: org.elasticsearch.packaging.test.DockerTests
-  method: test021InstallPlugin
-  issue: https://github.com/elastic/elasticsearch/issues/116147
 - class: org.elasticsearch.action.RejectionActionIT
   method: testSimulatedSearchRejectionLoad
   issue: https://github.com/elastic/elasticsearch/issues/125901
@@ -252,9 +222,6 @@ tests:
 - class: org.elasticsearch.search.basic.SearchWithRandomDisconnectsIT
   method: testSearchWithRandomDisconnects
   issue: https://github.com/elastic/elasticsearch/issues/122707
-- class: org.elasticsearch.packaging.test.DockerTests
-  method: test020PluginsListWithNoPlugins
-  issue: https://github.com/elastic/elasticsearch/issues/126232
 - class: org.elasticsearch.xpack.test.rest.XPackRestIT
   method: test {p0=transform/transforms_reset/Test force reseting a running transform}
   issue: https://github.com/elastic/elasticsearch/issues/126240
@@ -264,15 +231,9 @@ tests:
 - class: org.elasticsearch.xpack.test.rest.XPackRestIT
   method: test {p0=ml/start_data_frame_analytics/Test start classification analysis when the dependent variable cardinality is too low}
   issue: https://github.com/elastic/elasticsearch/issues/126299
-- class: org.elasticsearch.packaging.test.DockerTests
-  method: test023InstallPluginUsingConfigFile
-  issue: https://github.com/elastic/elasticsearch/issues/126145
 - class: org.elasticsearch.smoketest.MlWithSecurityIT
   method: test {yaml=ml/start_data_frame_analytics/Test start classification analysis when the dependent variable cardinality is too low}
   issue: https://github.com/elastic/elasticsearch/issues/123200
-- class: org.elasticsearch.packaging.test.DockerTests
-  method: test022InstallPluginsFromLocalArchive
-  issue: https://github.com/elastic/elasticsearch/issues/116866
 - class: org.elasticsearch.smoketest.MlWithSecurityIT
   method: test {yaml=ml/trained_model_cat_apis/Test cat trained models}
   issue: https://github.com/elastic/elasticsearch/issues/125750
@@ -315,15 +276,6 @@ tests:
 - class: org.elasticsearch.cli.keystore.AddStringKeyStoreCommandTests
   method: testStdinWithMultipleValues
   issue: https://github.com/elastic/elasticsearch/issues/126882
-- class: org.elasticsearch.packaging.test.DockerTests
-  method: test024InstallPluginFromArchiveUsingConfigFile
-  issue: https://github.com/elastic/elasticsearch/issues/126936
-- class: org.elasticsearch.packaging.test.DockerTests
-  method: test026InstallBundledRepositoryPlugins
-  issue: https://github.com/elastic/elasticsearch/issues/127081
-- class: org.elasticsearch.packaging.test.DockerTests
-  method: test026InstallBundledRepositoryPluginsViaConfigFile
-  issue: https://github.com/elastic/elasticsearch/issues/127158
 - class: org.elasticsearch.xpack.remotecluster.CrossClusterEsqlRCS2EnrichUnavailableRemotesIT
   method: testEsqlEnrichWithSkipUnavailable
   issue: https://github.com/elastic/elasticsearch/issues/127368
@@ -339,48 +291,18 @@ tests:
 - class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT
   method: test {p0=search/350_point_in_time/point-in-time with index filter}
   issue: https://github.com/elastic/elasticsearch/issues/127741
-- class: org.elasticsearch.packaging.test.DockerTests
-  method: test025SyncPluginsUsingProxy
-  issue: https://github.com/elastic/elasticsearch/issues/127138
 - class: org.elasticsearch.xpack.esql.action.CrossClusterQueryWithPartialResultsIT
   method: testOneRemoteClusterPartial
   issue: https://github.com/elastic/elasticsearch/issues/124055
 - class: org.elasticsearch.xpack.esql.qa.multi_node.EsqlSpecIT
   method: test {lookup-join.MvJoinKeyOnTheLookupIndex ASYNC}
   issue: https://github.com/elastic/elasticsearch/issues/128030
-- class: org.elasticsearch.packaging.test.DockerTests
-  method: test042KeystorePermissionsAreCorrect
-  issue: https://github.com/elastic/elasticsearch/issues/128018
-- class: org.elasticsearch.packaging.test.DockerTests
-  method: test072RunEsAsDifferentUserAndGroup
-  issue: https://github.com/elastic/elasticsearch/issues/128031
-- class: org.elasticsearch.packaging.test.DockerTests
-  method: test122CanUseDockerLoggingConfig
-  issue: https://github.com/elastic/elasticsearch/issues/128110
-- class: org.elasticsearch.packaging.test.DockerTests
-  method: test041AmazonCaCertsAreInTheKeystore
-  issue: https://github.com/elastic/elasticsearch/issues/128006
-- class: org.elasticsearch.packaging.test.DockerTests
-  method: test130JavaHasCorrectOwnership
-  issue: https://github.com/elastic/elasticsearch/issues/128174
-- class: org.elasticsearch.packaging.test.DockerTests
-  method: test600Interrupt
-  issue: https://github.com/elastic/elasticsearch/issues/128144
 - class: org.elasticsearch.packaging.test.EnrollmentProcessTests
   method: test20DockerAutoFormCluster
   issue: https://github.com/elastic/elasticsearch/issues/128113
-- class: org.elasticsearch.packaging.test.DockerTests
-  method: test121CanUseStackLoggingConfig
-  issue: https://github.com/elastic/elasticsearch/issues/128165
-- class: org.elasticsearch.packaging.test.DockerTests
-  method: test080ConfigurePasswordThroughEnvironmentVariableFile
-  issue: https://github.com/elastic/elasticsearch/issues/128075
 - class: org.elasticsearch.ingest.geoip.GeoIpDownloaderCliIT
   method: testInvalidTimestamp
   issue: https://github.com/elastic/elasticsearch/issues/128284
-- class: org.elasticsearch.packaging.test.DockerTests
-  method: test120DockerLogsIncludeElasticsearchLogs
-  issue: https://github.com/elastic/elasticsearch/issues/128117
 - class: org.elasticsearch.packaging.test.TemporaryDirectoryConfigTests
   method: test21AcceptsCustomPathInDocker
   issue: https://github.com/elastic/elasticsearch/issues/128114
@@ -396,27 +318,9 @@ tests:
 - class: org.elasticsearch.xpack.esql.action.CrossClusterQueryWithPartialResultsIT
   method: testFailToStartRequestOnRemoteCluster
   issue: https://github.com/elastic/elasticsearch/issues/128545
-- class: org.elasticsearch.packaging.test.DockerTests
-  method: test124CanRestartContainerWithStackLoggingConfig
-  issue: https://github.com/elastic/elasticsearch/issues/128121
-- class: org.elasticsearch.packaging.test.DockerTests
-  method: test085EnvironmentVariablesAreRespectedUnderDockerExec
-  issue: https://github.com/elastic/elasticsearch/issues/128115
 - class: org.elasticsearch.compute.operator.LimitOperatorTests
   method: testEarlyTermination
   issue: https://github.com/elastic/elasticsearch/issues/128721
-- class: org.elasticsearch.packaging.test.DockerTests
-  method: test040JavaUsesTheOsProvidedKeystore
-  issue: https://github.com/elastic/elasticsearch/issues/128230
-- class: org.elasticsearch.packaging.test.DockerTests
-  method: test150MachineDependentHeap
-  issue: https://github.com/elastic/elasticsearch/issues/128120
-- class: org.elasticsearch.packaging.test.DockerTests
-  method: test073RunEsAsDifferentUserAndGroupWithoutBindMounting
-  issue: https://github.com/elastic/elasticsearch/issues/128996
-- class: org.elasticsearch.packaging.test.DockerTests
-  method: test081SymlinksAreFollowedWithEnvironmentVariableFiles
-  issue: https://github.com/elastic/elasticsearch/issues/128867
 - class: org.elasticsearch.xpack.esql.qa.single_node.GenerativeForkIT
   method: test {lookup-join.EnrichLookupStatsBug ASYNC}
   issue: https://github.com/elastic/elasticsearch/issues/129228
@@ -505,9 +409,6 @@ tests:
 - class: org.elasticsearch.search.SearchWithRejectionsIT
   method: testOpenContextsAfterRejections
   issue: https://github.com/elastic/elasticsearch/issues/130821
-- class: org.elasticsearch.packaging.test.DockerTests
-  method: test082CannotUseEnvVarsAndFiles
-  issue: https://github.com/elastic/elasticsearch/issues/129808
 - class: org.elasticsearch.xpack.esql.action.EsqlActionBreakerIT
   method: testExtractFields
   issue: https://github.com/elastic/elasticsearch/issues/130501

qa/packaging/src/test/java/org/elasticsearch/packaging/test/PackagingTestCase.java

@@ -145,6 +145,10 @@ public abstract class PackagingTestCase extends Assert {
         @Override
         protected void failed(Throwable e, Description description) {
             failed = true;
+            if (installation != null && installation.distribution.isDocker()) {
+                logger.warn("Test {} failed. Printing logs for failed test...", description.getMethodName());
+                FileUtils.logAllLogs(installation.logs, logger);
+            }
         }
     };
 

qa/packaging/src/test/java/org/elasticsearch/packaging/util/ServerUtils.java

@@ -151,7 +151,19 @@ private static HttpResponse execute(Request request, String username, String pas
             executor.auth(username, password);
             executor.authPreemptive(new HttpHost("localhost", 9200));
         }
-        return executor.execute(request).returnResponse();
+        try {
+            return executor.execute(request).returnResponse();
+        } catch (Exception e) {
+            logger.warn(
+                "Failed to execute request [{}] with username/password [{}/{}] and caCert [{}]",
+                request.toString(),
+                username,
+                password,
+                caCert,
+                e
+            );
+            throw e;
+        }
     }
 
     // polls every two seconds for Elasticsearch to be running on 9200
@@ -239,14 +251,13 @@ public static void waitForElasticsearch(
         long timeElapsed = 0;
         boolean started = false;
         Throwable thrownException = null;
-        if (caCert == null) {
-            caCert = getCaCert(installation);
-        }
 
         while (started == false && timeElapsed < waitTime) {
             if (System.currentTimeMillis() - lastRequest > requestInterval) {
+                if (caCert == null) {
+                    caCert = getCaCert(installation);
+                }
                 try {
-
                     final HttpResponse response = execute(
                         Request.Get((caCert != null ? "https" : "http") + "://localhost:9200/_cluster/health")
                             .connectTimeout((int) timeoutLength)
@@ -277,7 +288,7 @@ public static void waitForElasticsearch(
                     }
                     started = true;
 
-                } catch (IOException e) {
+                } catch (Exception e) {
                     if (thrownException == null) {
                         thrownException = e;
                     } else {

qa/packaging/src/test/java/org/elasticsearch/packaging/util/docker/Docker.java

@@ -73,7 +73,7 @@ public class Docker {
     public static final Shell sh = new Shell();
     public static final DockerShell dockerShell = new DockerShell();
     public static final int STARTUP_SLEEP_INTERVAL_MILLISECONDS = 1000;
-    public static final int STARTUP_ATTEMPTS_MAX = 30;
+    public static final int STARTUP_ATTEMPTS_MAX = 45;
 
     /**
      * The length of the command exceeds what we can use for COLUMNS so we use