Shuffle test order and repeat test runs once. (#12275)

The tests we write must be resilient to the order in which they are run in the
harness. That is, they must not rely on global state set by other tests that
have already run in the process. Also, these tests must themselves be
repeatable. That is, they must correctly clean up after themselves and be able
to run successfully again in the same process.

This patch adds some safeguards against (but does NOT guarantee) the addition of
tests that violate the dictum.

Additionally, test failures must be easily reproducible for folks investigating
the test failure. Also, tests that assert correctness of unrelated code must not
stop progress on the authors patch.

This changes does not hinder reproducibility of test failures because the random
seed is printed in the logs before running each test. Developers attempting to
reproduce the failure locally can do the same via the following invocation
`--gtest_shuffle --gtest_repeat=<the count> --gtest_random_seed=<seed from failing run>`.

This change does introduce potential burden on patch authors that may see
failures in unrelated code as a newly failing shuffle seed is used on their
runs. To ameliorate this, we will formulate guidance for them to aggressively
mark such tests as disabled and file bugs to enable the same.

The test seed is intentionally kept low because it’s purpose is to test that
individual tests are repeatable. It must not be used as a replacement for
fuzzing.

Author: chinmaygarde

shell/platform/embedder/tests/embedder_unittests.cc

@@ -218,6 +218,7 @@ TEST_F(EmbedderTest, CanSpecifyCustomTaskRunner) {
   kill_latch.Wait();
 
   ASSERT_TRUE(signaled_once);
+  signaled_once = false;
 }
 
 TEST(EmbedderTestNoFixture, CanGetCurrentTimeInNanoseconds) {

testing/run_tests.py

@@ -71,39 +71,44 @@ def RunEngineExecutable(build_dir, executable_name, filter, flags=[], cwd=buildr
 def RunCCTests(build_dir, filter):
   print "Running Engine Unit-tests."
 
-  RunEngineExecutable(build_dir, 'client_wrapper_glfw_unittests', filter)
+  shuffle_flags = [
+    "--gtest_shuffle",
+    "--gtest_repeat=2",
+  ]
+
+  RunEngineExecutable(build_dir, 'client_wrapper_glfw_unittests', filter, shuffle_flags)
 
-  RunEngineExecutable(build_dir, 'client_wrapper_unittests', filter)
+  RunEngineExecutable(build_dir, 'client_wrapper_unittests', filter, shuffle_flags)
 
   # https://github.com/flutter/flutter/issues/36294
   if not IsWindows():
-    RunEngineExecutable(build_dir, 'embedder_unittests', filter)
+    RunEngineExecutable(build_dir, 'embedder_unittests', filter, shuffle_flags)
 
   flow_flags = ['--gtest_filter=-PerformanceOverlayLayer.Gold']
   if IsLinux():
     flow_flags = [
       '--golden-dir=%s' % golden_dir,
       '--font-file=%s' % roboto_font_path,
     ]
-  RunEngineExecutable(build_dir, 'flow_unittests', filter, flow_flags)
+  RunEngineExecutable(build_dir, 'flow_unittests', filter, flow_flags + shuffle_flags)
 
-  RunEngineExecutable(build_dir, 'fml_unittests', filter, [ time_sensitve_test_flag ])
+  RunEngineExecutable(build_dir, 'fml_unittests', filter, [ time_sensitve_test_flag ] + shuffle_flags)
 
-  RunEngineExecutable(build_dir, 'runtime_unittests', filter)
+  RunEngineExecutable(build_dir, 'runtime_unittests', filter, shuffle_flags)
 
   # https://github.com/flutter/flutter/issues/36295
   if not IsWindows():
-    RunEngineExecutable(build_dir, 'shell_unittests', filter)
+    RunEngineExecutable(build_dir, 'shell_unittests', filter, shuffle_flags)
 
-  RunEngineExecutable(build_dir, 'ui_unittests', filter)
+  RunEngineExecutable(build_dir, 'ui_unittests', filter, shuffle_flags)
 
   # These unit-tests are Objective-C and can only run on Darwin.
   if IsMac():
-    RunEngineExecutable(build_dir, 'flutter_channels_unittests', filter)
+    RunEngineExecutable(build_dir, 'flutter_channels_unittests', filter, shuffle_flags)
 
   # https://github.com/flutter/flutter/issues/36296
   if IsLinux():
-    RunEngineExecutable(build_dir, 'txt_unittests', filter, [ fonts_dir_flag ])
+    RunEngineExecutable(build_dir, 'txt_unittests', filter, [ fonts_dir_flag ] + shuffle_flags)
 
 
 def RunEngineBenchmarks(build_dir, filter):