Migrate verify_codesigned. (flutter#139328)

This is part of the migration of adhoc tests to shard tests.

Bug: flutter#139153

Author: godofredoc

.ci.yaml

@@ -3714,26 +3714,24 @@ targets:
   - name: Mac_x64 verify_binaries_codesigned
     enabled_branches:
       - flutter-\d+\.\d+-candidate\.\d+
-    recipe: flutter/flutter
+    recipe: flutter/flutter_drone
     presubmit: false
     timeout: 60
     properties:
       tags: >
         ["framework", "hostonly", "shard", "mac"]
-      validation: verify_binaries_codesigned
-      validation_name: Verify x64 binaries codesigned
+      shard: verify_binaries_codesigned
 
   - name: Mac_arm64 verify_binaries_codesigned
     enabled_branches:
       - flutter-\d+\.\d+-candidate\.\d+
-    recipe: flutter/flutter
+    recipe: flutter/flutter_drone
     presubmit: false
     timeout: 60
     properties:
       tags: >
         ["framework", "hostonly", "shard", "mac"]
-      validation: verify_binaries_codesigned
-      validation_name: Verify arm64 binaries codesigned
+      shard: verify_binaries_codesigned
 
   - name: Mac web_tool_tests
     recipe: flutter/flutter_drone

dev/bots/test.dart

@@ -52,13 +52,16 @@ import 'dart:core' as system show print;
 import 'dart:core' hide print;
 import 'dart:io' as system show exit;
 import 'dart:io' hide exit;
+import 'dart:io' as io;
 import 'dart:math' as math;
 import 'dart:typed_data';
 
 import 'package:archive/archive.dart';
 import 'package:file/file.dart' as fs;
 import 'package:file/local.dart';
+import 'package:meta/meta.dart';
 import 'package:path/path.dart' as path;
+import 'package:process/process.dart';
 
 import 'browser.dart';
 import 'run_command.dart';
@@ -252,6 +255,7 @@ Future<void> main(List<String> args) async {
       'analyze': _runAnalyze,
       'fuchsia_precache': _runFuchsiaPrecache,
       'docs': _runDocs,
+      'verify_binaries_codesigned': _runVerifyCodesigned,
       kTestHarnessShardName: _runTestHarnessTests, // Used for testing this script; also run as part of SHARD=framework_tests, SUBSHARD=misc.
     });
   } catch (error, stackTrace) {
@@ -1644,6 +1648,320 @@ Future<void> _runDocs() async {
   );
 }
 
+// Verifies binaries are codesigned.
+Future<void> _runVerifyCodesigned() async {
+  printProgress('${green}Running binaries codesign verification$reset');
+  await runCommand(
+    'flutter',
+    <String>[
+      'precache',
+      '--android',
+      '--ios',
+      '--macos'
+    ],
+    workingDirectory: flutterRoot,
+  );
+
+  await verifyExist(flutterRoot);
+  await verifySignatures(flutterRoot);
+}
+
+const List<String> expectedEntitlements = <String>[
+  'com.apple.security.cs.allow-jit',
+  'com.apple.security.cs.allow-unsigned-executable-memory',
+  'com.apple.security.cs.allow-dyld-environment-variables',
+  'com.apple.security.network.client',
+  'com.apple.security.network.server',
+  'com.apple.security.cs.disable-library-validation',
+];
+
+/// Binaries that are expected to be codesigned and have entitlements.
+///
+/// This list should be kept in sync with the actual contents of Flutter's
+/// cache.
+Future<List<String>> binariesWithEntitlements(String flutterRoot) async {
+  return <String> [
+    'artifacts/engine/android-arm-profile/darwin-x64/gen_snapshot',
+    'artifacts/engine/android-arm-release/darwin-x64/gen_snapshot',
+    'artifacts/engine/android-arm64-profile/darwin-x64/gen_snapshot',
+    'artifacts/engine/android-arm64-release/darwin-x64/gen_snapshot',
+    'artifacts/engine/android-x64-profile/darwin-x64/gen_snapshot',
+    'artifacts/engine/android-x64-release/darwin-x64/gen_snapshot',
+    'artifacts/engine/darwin-x64-profile/gen_snapshot',
+    'artifacts/engine/darwin-x64-profile/gen_snapshot_arm64',
+    'artifacts/engine/darwin-x64-profile/gen_snapshot_x64',
+    'artifacts/engine/darwin-x64-release/gen_snapshot',
+    'artifacts/engine/darwin-x64-release/gen_snapshot_arm64',
+    'artifacts/engine/darwin-x64-release/gen_snapshot_x64',
+    'artifacts/engine/darwin-x64/flutter_tester',
+    'artifacts/engine/darwin-x64/gen_snapshot',
+    'artifacts/engine/darwin-x64/gen_snapshot_arm64',
+    'artifacts/engine/darwin-x64/gen_snapshot_x64',
+    'artifacts/engine/ios-profile/gen_snapshot_arm64',
+    'artifacts/engine/ios-release/gen_snapshot_arm64',
+    'artifacts/engine/ios/gen_snapshot_arm64',
+    'artifacts/libimobiledevice/idevicescreenshot',
+    'artifacts/libimobiledevice/idevicesyslog',
+    'artifacts/libimobiledevice/libimobiledevice-1.0.6.dylib',
+    'artifacts/libplist/libplist-2.0.3.dylib',
+    'artifacts/openssl/libcrypto.1.1.dylib',
+    'artifacts/openssl/libssl.1.1.dylib',
+    'artifacts/usbmuxd/iproxy',
+    'artifacts/usbmuxd/libusbmuxd-2.0.6.dylib',
+    'dart-sdk/bin/dart',
+    'dart-sdk/bin/dartaotruntime',
+    'dart-sdk/bin/utils/gen_snapshot',
+    'dart-sdk/bin/utils/wasm-opt',
+  ]
+  .map((String relativePath) => path.join(flutterRoot, 'bin', 'cache', relativePath)).toList();
+}
+
+/// Binaries that are only expected to be codesigned.
+///
+/// This list should be kept in sync with the actual contents of Flutter's
+/// cache.
+Future<List<String>> binariesWithoutEntitlements(String flutterRoot) async {
+  return <String>[
+    'artifacts/engine/darwin-x64-profile/FlutterMacOS.framework/Versions/A/FlutterMacOS',
+    'artifacts/engine/darwin-x64-release/FlutterMacOS.framework/Versions/A/FlutterMacOS',
+    'artifacts/engine/darwin-x64/FlutterMacOS.framework/Versions/A/FlutterMacOS',
+    'artifacts/engine/darwin-x64/font-subset',
+    'artifacts/engine/darwin-x64/impellerc',
+    'artifacts/engine/darwin-x64/libpath_ops.dylib',
+    'artifacts/engine/darwin-x64/libtessellator.dylib',
+    'artifacts/engine/ios-profile/Flutter.xcframework/ios-arm64/Flutter.framework/Flutter',
+    'artifacts/engine/ios-profile/Flutter.xcframework/ios-arm64_x86_64-simulator/Flutter.framework/Flutter',
+    'artifacts/engine/ios-profile/extension_safe/Flutter.xcframework/ios-arm64/Flutter.framework/Flutter',
+    'artifacts/engine/ios-profile/extension_safe/Flutter.xcframework/ios-arm64_x86_64-simulator/Flutter.framework/Flutter',
+    'artifacts/engine/ios-release/Flutter.xcframework/ios-arm64/Flutter.framework/Flutter',
+    'artifacts/engine/ios-release/Flutter.xcframework/ios-arm64_x86_64-simulator/Flutter.framework/Flutter',
+    'artifacts/engine/ios-release/extension_safe/Flutter.xcframework/ios-arm64/Flutter.framework/Flutter',
+    'artifacts/engine/ios-release/extension_safe/Flutter.xcframework/ios-arm64_x86_64-simulator/Flutter.framework/Flutter',
+    'artifacts/engine/ios/Flutter.xcframework/ios-arm64/Flutter.framework/Flutter',
+    'artifacts/engine/ios/Flutter.xcframework/ios-arm64_x86_64-simulator/Flutter.framework/Flutter',
+    'artifacts/engine/ios/extension_safe/Flutter.xcframework/ios-arm64/Flutter.framework/Flutter',
+    'artifacts/engine/ios/extension_safe/Flutter.xcframework/ios-arm64_x86_64-simulator/Flutter.framework/Flutter',
+    'artifacts/ios-deploy/ios-deploy',
+  ]
+  .map((String relativePath) => path.join(flutterRoot, 'bin', 'cache', relativePath)).toList();
+}
+
+/// Verify the existence of all expected binaries in cache.
+///
+/// This function ignores code signatures and entitlements, and is intended to
+/// be run on every commit. It should throw if either new binaries are added
+/// to the cache or expected binaries removed. In either case, this class'
+/// [binariesWithEntitlements] or [binariesWithoutEntitlements] lists should
+/// be updated accordingly.
+Future<void> verifyExist(
+  String flutterRoot,
+  {@visibleForTesting ProcessManager processManager = const LocalProcessManager()
+}) async {
+  final Set<String> foundFiles = <String>{};
+  final String cacheDirectory =  path.join(flutterRoot, 'bin', 'cache');
+
+
+
+  for (final String binaryPath
+      in await findBinaryPaths(cacheDirectory, processManager: processManager)) {
+    if ((await binariesWithEntitlements(flutterRoot)).contains(binaryPath)) {
+      foundFiles.add(binaryPath);
+    } else if ((await binariesWithoutEntitlements(flutterRoot)).contains(binaryPath)) {
+      foundFiles.add(binaryPath);
+    } else {
+      throw Exception(
+          'Found unexpected binary in cache: $binaryPath');
+    }
+  }
+
+  final List<String> allExpectedFiles = await binariesWithEntitlements(flutterRoot) + await binariesWithoutEntitlements(flutterRoot);
+  if (foundFiles.length < allExpectedFiles.length) {
+    final List<String> unfoundFiles = allExpectedFiles
+        .where(
+          (String file) => !foundFiles.contains(file),
+        )
+        .toList();
+    print(
+      'Expected binaries not found in cache:\n\n${unfoundFiles.join('\n')}\n\n'
+      'If this commit is removing binaries from the cache, this test should be fixed by\n'
+      'removing the relevant entry from either the "binariesWithEntitlements" or\n'
+      '"binariesWithoutEntitlements" getters in dev/tools/lib/codesign.dart.',
+    );
+    throw Exception('Did not find all expected binaries!');
+  }
+
+  print('All expected binaries present.');
+}
+
+/// Verify code signatures and entitlements of all binaries in the cache.
+Future<void> verifySignatures(
+  String flutterRoot,
+  {@visibleForTesting ProcessManager processManager = const LocalProcessManager()}
+) async {
+  final List<String> unsignedBinaries = <String>[];
+  final List<String> wrongEntitlementBinaries = <String>[];
+  final List<String> unexpectedBinaries = <String>[];
+  final String cacheDirectory =  path.join(flutterRoot, 'bin', 'cache');
+
+  for (final String binaryPath
+      in await findBinaryPaths(cacheDirectory, processManager: processManager)) {
+    bool verifySignature = false;
+    bool verifyEntitlements = false;
+    if ((await binariesWithEntitlements(flutterRoot)).contains(binaryPath)) {
+      verifySignature = true;
+      verifyEntitlements = true;
+    }
+    if ((await binariesWithoutEntitlements(flutterRoot)).contains(binaryPath)) {
+      verifySignature = true;
+    }
+    if (!verifySignature && !verifyEntitlements) {
+      unexpectedBinaries.add(binaryPath);
+      print('Unexpected binary $binaryPath found in cache!');
+      continue;
+    }
+    print('Verifying the code signature of $binaryPath');
+    final io.ProcessResult codeSignResult = await processManager.run(
+      <String>[
+        'codesign',
+        '-vvv',
+        binaryPath,
+      ],
+    );
+    if (codeSignResult.exitCode != 0) {
+      unsignedBinaries.add(binaryPath);
+      print(
+        'File "$binaryPath" does not appear to be codesigned.\n'
+        'The `codesign` command failed with exit code ${codeSignResult.exitCode}:\n'
+        '${codeSignResult.stderr}\n',
+      );
+      continue;
+    }
+    if (verifyEntitlements) {
+      print('Verifying entitlements of $binaryPath');
+      if (!(await hasExpectedEntitlements(binaryPath, flutterRoot, processManager: processManager))) {
+        wrongEntitlementBinaries.add(binaryPath);
+      }
+    }
+  }
+
+  // First print all deviations from expectations
+  if (unsignedBinaries.isNotEmpty) {
+    print('Found ${unsignedBinaries.length} unsigned binaries:');
+    unsignedBinaries.forEach(print);
+  }
+
+  if (wrongEntitlementBinaries.isNotEmpty) {
+    print('Found ${wrongEntitlementBinaries.length} binaries with unexpected entitlements:');
+    wrongEntitlementBinaries.forEach(print);
+  }
+
+  if (unexpectedBinaries.isNotEmpty) {
+    print('Found ${unexpectedBinaries.length} unexpected binaries in the cache:');
+    unexpectedBinaries.forEach(print);
+  }
+
+  // Finally, exit on any invalid state
+  if (unsignedBinaries.isNotEmpty) {
+    throw Exception('Test failed because unsigned binaries detected.');
+  }
+
+  if (wrongEntitlementBinaries.isNotEmpty) {
+    throw Exception(
+      'Test failed because files found with the wrong entitlements:\n'
+      '${wrongEntitlementBinaries.join('\n')}',
+    );
+  }
+
+  if (unexpectedBinaries.isNotEmpty) {
+    throw Exception('Test failed because unexpected binaries found in the cache.');
+  }
+  print('Verified that binaries are codesigned and have expected entitlements.');
+}
+
+/// Find every binary file in the given [rootDirectory].
+Future<List<String>> findBinaryPaths(
+  String rootDirectory,
+  {@visibleForTesting ProcessManager processManager = const LocalProcessManager()
+}) async {
+  final List<String> allBinaryPaths = <String>[];
+  final io.ProcessResult result = await processManager.run(
+    <String>[
+      'find',
+      rootDirectory,
+      '-type',
+      'f',
+    ],
+  );
+  final List<String> allFiles = (result.stdout as String)
+      .split('\n')
+      .where((String s) => s.isNotEmpty)
+      .toList();
+
+  await Future.forEach(allFiles, (String filePath) async {
+    if (await isBinary(filePath, processManager: processManager)) {
+      allBinaryPaths.add(filePath);
+      print('Found: $filePath\n');
+    }
+  });
+  return allBinaryPaths;
+}
+
+/// Check mime-type of file at [filePath] to determine if it is binary.
+Future<bool> isBinary(
+  String filePath,
+  {@visibleForTesting ProcessManager processManager = const LocalProcessManager()}
+) async {
+  final io.ProcessResult result = await processManager.run(
+    <String>[
+      'file',
+      '--mime-type',
+      '-b', // is binary
+      filePath,
+    ],
+  );
+  return (result.stdout as String).contains('application/x-mach-binary');
+}
+
+/// Check if the binary has the expected entitlements.
+Future<bool> hasExpectedEntitlements(
+  String binaryPath,
+  String flutterRoot,
+  {@visibleForTesting ProcessManager processManager = const LocalProcessManager()}
+) async {
+  final io.ProcessResult entitlementResult = await processManager.run(
+    <String>[
+      'codesign',
+      '--display',
+      '--entitlements',
+      ':-',
+      binaryPath,
+    ],
+  );
+
+  if (entitlementResult.exitCode != 0) {
+    print(
+      'The `codesign --entitlements` command failed with exit code ${entitlementResult.exitCode}:\n'
+      '${entitlementResult.stderr}\n',
+    );
+    return false;
+  }
+
+  bool passes = true;
+  final String output = entitlementResult.stdout as String;
+  for (final String entitlement in expectedEntitlements) {
+    final bool entitlementExpected =
+        (await binariesWithEntitlements(flutterRoot)).contains(binaryPath);
+    if (output.contains(entitlement) != entitlementExpected) {
+      print(
+        'File "$binaryPath" ${entitlementExpected ? 'does not have expected' : 'has unexpected'} '
+        'entitlement $entitlement.',
+      );
+      passes = false;
+    }
+  }
+  return passes;
+}
+
 /// Runs the skp_generator from the flutter/tests repo.
 ///
 /// See also the customer_tests shard.