Work around the glibc bug that causes rare Chrome crashes (flutter#67466)

Work around the glibc bug that causes rare Chrome crashes

Author: yjbanov

dev/try_builders.json

@@ -165,7 +165,7 @@
          "repo":"flutter",
          "task_name":"linux_web_tests",
          "enabled":true,
-         "run_if":["dev/", "packages/flutter/", "packages/flutter_goldens_client/", "packages/flutter_goldens/", "packages/flutter_test/", "packages/flutter_tools/lib/src/test/", "packages/flutter_web_plugins/", "bin/"]
+         "run_if":["dev/", "packages/", "bin/"]
       },
       {
          "name":"Linux web_integration_tests",

packages/flutter_tools/lib/src/web/chrome.dart

@@ -35,6 +35,18 @@ const String kWindowsExecutable = r'Google\Chrome\Application\chrome.exe';
 /// The expected Edge executable name on Windows.
 const String kWindowsEdgeExecutable = r'Microsoft\Edge\Application\msedge.exe';
 
+/// Used by [ChromiumLauncher] to detect a glibc bug and retry launching the
+/// browser.
+///
+/// Once every few thousands of launches we hit this glibc bug:
+///
+/// https://sourceware.org/bugzilla/show_bug.cgi?id=19329.
+///
+/// When this happens Chrome spits out something like the following then exits with code 127:
+///
+///     Inconsistency detected by ld.so: ../elf/dl-tls.c: 493: _dl_allocate_tls_init: Assertion `listp->slotinfo[cnt].gen <= GL(dl_tls_generation)' failed!
+const String _kGlibcError = 'Inconsistency detected by ld.so';
+
 typedef BrowserFinder = String Function(Platform, FileSystem);
 
 /// Find the chrome executable on the current platform.
@@ -168,6 +180,12 @@ class ChromiumLauncher {
     }
 
     final String chromeExecutable = _browserFinder(_platform, _fileSystem);
+
+    if (_logger.isVerbose) {
+      final ProcessResult versionResult = await _processManager.run(<String>[chromeExecutable, '--version']);
+      _logger.printTrace('Using ${versionResult.stdout}');
+    }
+
     final Directory userDataDir = _fileSystem.systemTempDirectory
       .createTempSync('flutter_tools_chrome_device.');
 
@@ -204,27 +222,7 @@ class ChromiumLauncher {
       url,
     ];
 
-    final Process process = await _processManager.start(args);
-
-    process.stdout
-      .transform(utf8.decoder)
-      .transform(const LineSplitter())
-      .listen((String line) {
-        _logger.printTrace('[CHROME]: $line');
-      });
-
-    // Wait until the DevTools are listening before trying to connect. This is
-    // only required for flutter_test --platform=chrome and not flutter run.
-    await process.stderr
-      .transform(utf8.decoder)
-      .transform(const LineSplitter())
-      .map((String line) {
-        _logger.printTrace('[CHROME]:$line');
-        return line;
-      })
-      .firstWhere((String line) => line.startsWith('DevTools listening'), orElse: () {
-        return 'Failed to spawn stderr';
-      });
+    final Process process = await _spawnChromiumProcess(args);
 
     // When the process exits, copy the user settings back to the provided data-dir.
     if (cacheDir != null) {
@@ -241,6 +239,63 @@ class ChromiumLauncher {
     ), skipCheck);
   }
 
+  Future<Process> _spawnChromiumProcess(List<String> args) async {
+    // Keep attempting to launch the browser until one of:
+    // - Chrome launched successfully, in which case we just return from the loop.
+    // - The tool detected an unretriable Chrome error, in which case we throw ToolExit.
+    while (true) {
+      final Process process = await _processManager.start(args);
+
+      process.stdout
+        .transform(utf8.decoder)
+        .transform(const LineSplitter())
+        .listen((String line) {
+          _logger.printTrace('[CHROME]: $line');
+        });
+
+      // Wait until the DevTools are listening before trying to connect. This is
+      // only required for flutter_test --platform=chrome and not flutter run.
+      bool hitGlibcBug = false;
+      await process.stderr
+        .transform(utf8.decoder)
+        .transform(const LineSplitter())
+        .map((String line) {
+          _logger.printTrace('[CHROME]:$line');
+          if (line.contains(_kGlibcError)) {
+            hitGlibcBug = true;
+          }
+          return line;
+        })
+        .firstWhere((String line) => line.startsWith('DevTools listening'), orElse: () {
+          if (hitGlibcBug) {
+            _logger.printTrace(
+              'Encountered glibc bug https://sourceware.org/bugzilla/show_bug.cgi?id=19329. '
+              'Will try launching browser again.',
+            );
+            return null;
+          }
+          _logger.printTrace('Failed to launch browser. Command used to launch it: ${args.join(' ')}');
+          throw ToolExit(
+            'Failed to launch browser. Make sure you are using an up-to-date '
+            'Chrome or Edge. Otherwise, consider using -d web-server instead '
+            'and filing an issue at https://github.com/flutter/flutter/issues.',
+          );
+        });
+
+      if (!hitGlibcBug) {
+        return process;
+      }
+
+      // A precaution that avoids accumulating browser processes, in case the
+      // glibc bug doesn't cause the browser to quit and we keep looping and
+      // launching more processes.
+      unawaited(process.exitCode.timeout(const Duration(seconds: 1), onTimeout: () {
+        process.kill();
+        return null;
+      }));
+    }
+  }
+
   // This is a JSON file which contains configuration from the browser session,
   // such as window position. It is located under the Chrome data-dir folder.
   String get _preferencesPath => _fileSystem.path.join('Default', 'preferences');

packages/flutter_tools/test/general.shard/web/chrome_test.dart

@@ -81,7 +81,7 @@ void main() {
         processManager,
         chromeLauncher,
       ),
-      throwsToolExit(),
+      throwsToolExit(message: 'Only one instance of chrome can be started'),
     );
   });
 
@@ -209,13 +209,17 @@ void main() {
     localStorageContentsDirectory.childFile('LOCK').writeAsBytesSync(<int>[]);
     localStorageContentsDirectory.childFile('LOG').writeAsStringSync('contents');
 
-    processManager.addCommand(FakeCommand(command: const <String>[
-      'example_chrome',
-      '--user-data-dir=/.tmp_rand0/flutter_tools_chrome_device.rand0',
-      '--remote-debugging-port=1234',
-      ...kChromeArgs,
-      'example_url',
-    ], completer: exitCompleter));
+    processManager.addCommand(FakeCommand(
+      command: const <String>[
+        'example_chrome',
+        '--user-data-dir=/.tmp_rand0/flutter_tools_chrome_device.rand0',
+        '--remote-debugging-port=1234',
+        ...kChromeArgs,
+        'example_url',
+      ],
+      completer: exitCompleter,
+      stderr: kDevtoolsStderr,
+    ));
 
     await chromeLauncher.launch(
       'example_url',
@@ -244,6 +248,71 @@ void main() {
     expect(storageDir.childFile('LOG'), exists);
     expect(storageDir.childFile('LOG').readAsStringSync(), 'contents');
   });
+
+  testWithoutContext('can retry launch when glibc bug happens', () async {
+    const List<String> args = <String>[
+      'example_chrome',
+      '--user-data-dir=/.tmp_rand0/flutter_tools_chrome_device.rand0',
+      '--remote-debugging-port=1234',
+      ...kChromeArgs,
+      '--headless',
+      '--disable-gpu',
+      '--no-sandbox',
+      '--window-size=2400,1800',
+      'example_url',
+    ];
+
+    // Pretend to hit glibc bug 3 times.
+    for (int i = 0; i < 3; i++) {
+      processManager.addCommand(const FakeCommand(
+        command: args,
+        stderr: 'Inconsistency detected by ld.so: ../elf/dl-tls.c: 493: '
+                '_dl_allocate_tls_init: Assertion `listp->slotinfo[cnt].gen '
+                '<= GL(dl_tls_generation)\' failed!',
+      ));
+    }
+
+    // Succeed on the 4th try.
+    processManager.addCommand(const FakeCommand(
+      command: args,
+      stderr: kDevtoolsStderr,
+    ));
+
+    expect(
+      () async => await chromeLauncher.launch(
+        'example_url',
+        skipCheck: true,
+        headless: true,
+      ),
+      returnsNormally,
+    );
+  });
+
+  testWithoutContext('gives up retrying when a non-glibc error happens', () async {
+    processManager.addCommand(const FakeCommand(
+      command: <String>[
+        'example_chrome',
+        '--user-data-dir=/.tmp_rand0/flutter_tools_chrome_device.rand0',
+        '--remote-debugging-port=1234',
+        ...kChromeArgs,
+        '--headless',
+        '--disable-gpu',
+        '--no-sandbox',
+        '--window-size=2400,1800',
+        'example_url',
+      ],
+      stderr: 'nothing in the std error indicating glibc error',
+    ));
+
+    expect(
+      () async => await chromeLauncher.launch(
+        'example_url',
+        skipCheck: true,
+        headless: true,
+      ),
+      throwsToolExit(message: 'Failed to launch browser.'),
+    );
+  });
 }
 
 class MockFileSystemUtils extends Mock implements FileSystemUtils {}