From 0df391423fba9ab5ff56f33af7e2d29fec8ea73f Mon Sep 17 00:00:00 2001 From: Craig Labenz Date: Thu, 4 Jan 2024 13:07:27 -0800 Subject: [PATCH 1/4] adds sdks_finder command to builder utility --- tool/builder/bin/main.dart | 16 +- tool/builder/lib/extensions.dart | 35 +++ tool/builder/lib/repo_finder.dart | 13 ++ tool/builder/lib/sdks_finder.dart | 345 ++++++++++++++++++++++++++++++ tool/builder/pubspec.yaml | 3 +- 5 files changed, 400 insertions(+), 12 deletions(-) create mode 100644 tool/builder/lib/extensions.dart create mode 100644 tool/builder/lib/sdks_finder.dart diff --git a/tool/builder/bin/main.dart b/tool/builder/bin/main.dart index f00d48a1..ccf00cfb 100644 --- a/tool/builder/bin/main.dart +++ b/tool/builder/bin/main.dart @@ -2,24 +2,18 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -import 'dart:io' as io; import 'package:args/command_runner.dart'; import 'package:builder/download_model.dart'; +import 'package:builder/sdks_finder.dart'; import 'package:builder/sync_headers.dart'; -import 'package:logging/logging.dart'; final runner = CommandRunner( 'build', 'Performs build operations for google/flutter-mediapipe that ' 'depend on contents in this repository', ) - ..addCommand(SyncHeadersCommand()) - ..addCommand(DownloadModelCommand()); + ..addCommand(DownloadModelCommand()) + ..addCommand(SdksFinderCommand()) + ..addCommand(SyncHeadersCommand()); -void main(List arguments) { - Logger.root.onRecord.listen((LogRecord record) { - io.stdout - .writeln('${record.level.name}: ${record.time}: ${record.message}'); - }); - runner.run(arguments); -} +void main(List arguments) => runner.run(arguments); diff --git a/tool/builder/lib/extensions.dart b/tool/builder/lib/extensions.dart new file mode 100644 index 00000000..5f9081a0 --- /dev/null +++ b/tool/builder/lib/extensions.dart @@ -0,0 +1,35 @@ +import 'dart:convert'; +import 'dart:io'; + +extension EasyOutput on Process { + Future> get processedStdErr => _process(this.stderr); + + Future> get processedStdOut => _process(this.stdout); + + Future> _process(Stream> stream) async { + return utf8.decoder + .convert((await stream.toList()) + .fold>([], (arr, el) => arr..addAll(el))) + .split('\n'); + } +} + +/// Returns the last full chunk from a Url-like String. +/// +/// From "/an/awesome/url/", returns "url". +/// From "/an/awesome/url", returns "url". +/// From "/an/awesome/url/", with a depth of 1, returns "awesome" +/// From "/an/awesome/url", with a depth of 1, returns "awesome" +String lastChunk(String url, {int depth = 0}) { + final indexOffset = (url.endsWith('/')) ? -2 - depth : -1 - depth; + final splitUrl = url.split('/'); + return splitUrl[splitUrl.length + indexOffset]; +} + +extension DefaultableMap on Map { + void setDefault(K key, V def) { + if (!containsKey(key)) { + this[key] = def; + } + } +} diff --git a/tool/builder/lib/repo_finder.dart b/tool/builder/lib/repo_finder.dart index f81bd677..4faaa66b 100644 --- a/tool/builder/lib/repo_finder.dart +++ b/tool/builder/lib/repo_finder.dart @@ -5,6 +5,7 @@ import 'dart:io' as io; import 'package:args/args.dart'; import 'package:args/command_runner.dart'; +import 'package:logging/logging.dart'; import 'package:path/path.dart' as path; import 'package:io/ansi.dart'; @@ -34,6 +35,18 @@ mixin RepoFinderMixin on Command { ); } + void addVerboseOption(ArgParser argParser) => + argParser.addFlag('verbose', abbr: 'v', defaultsTo: false); + + void setUpLogging() { + final bool verbose = argResults!['verbose']; + Logger.root.level = verbose ? Level.FINEST : Level.INFO; + Logger.root.onRecord.listen((LogRecord record) { + io.stdout.writeln( + '[${record.loggerName}][${record.level.name}] ${record.message}'); + }); + } + /// Looks upward for the root of the `google/mediapipe` repository. This assumes /// the `dart build` command is executed from within said repository. If it is /// not executed from within, then this searching algorithm will reach the root diff --git a/tool/builder/lib/sdks_finder.dart b/tool/builder/lib/sdks_finder.dart new file mode 100644 index 00000000..94f1eeec --- /dev/null +++ b/tool/builder/lib/sdks_finder.dart @@ -0,0 +1,345 @@ +import 'dart:convert'; +import 'dart:io'; + +import 'package:args/command_runner.dart'; +import 'package:builder/extensions.dart'; +import 'package:builder/repo_finder.dart'; +import 'package:io/ansi.dart'; +import 'package:logging/logging.dart'; +import 'package:native_assets_cli/native_assets_cli.dart'; +import 'package:path/path.dart' as path; + +final _log = Logger('SDKsFinder'); + +/// Structure of flattened build locations suitable for JSON serialization. +/// Format is: +/// { +/// : { +/// : , +/// ... +/// }, +/// ... +/// } +typedef _FlatResults = Map>; + +/// Container for the three flavors of MediaPipe tasks. +enum MediaPipeSdk { + libaudio, + libtext, + libvision; + + String toPackageDir() => switch (this) { + MediaPipeSdk.libaudio => 'mediapipe-task-audio', + MediaPipeSdk.libtext => 'mediapipe-task-text', + MediaPipeSdk.libvision => 'mediapipe-task-vision', + }; +} + +/// Scans the GCS buckets where MediaPipe SDKs are stored, identifies the latest +/// builds for each supported build target, and writes the output to a +/// designated file. +/// +/// [SdksFinderCommand] depends on `gsutil` and a Google-corp account that has +/// permissions to read the necessary buckets, so the idea is to run this +/// command on Googlers' work machines whenever MediaPipe rebuilds SDKs. The +/// manual nature of this command is suboptimal, but there is no precedent for +/// fully automating this portion of the MediaPipe release process, so this +/// command helps by automating a portion of the task. +/// +/// The cache-busting mechanism of Flutter's native assets feature is a hash +/// of the contents of any build dependencies, so if this command leads to any +/// new build state, this must be reflected by *some change* to the associated +/// library's `build.dart` script. +/// +/// Operationally, [SdksFinderCommand]'s implementation involves orchestrating +/// one [_OsFinder] instance for each supported [OS] value, which in turn +/// identifies the correct build that has been uploaded to GCS by MediaPipe +/// release machinery. +/// +/// Usage: +/// ```sh +/// $ cd path/to/flutter-mediapipe +/// $ dart tool/builder/bin/main.dart sdks [-v] +/// ``` +class SdksFinderCommand extends Command with RepoFinderMixin { + SdksFinderCommand() { + addVerboseOption(argParser); + } + @override + String description = + 'Downloads the appropriate MediaPipe SDKs for the current build target'; + + @override + String name = 'sdks'; + + static const _gcsPrefix = 'https://storage.googleapis.com'; + + /// Google Storage bucket which houses all MediaPipe SDK uploads. + static const _bucketName = 'mediapipe-nightly-public/prod/mediapipe'; + + final _finders = <_OsFinder>[ + _OsFinder(OS.android), + _OsFinder(OS.macOS), + // TODO: Add other values as their support is ready + ]; + + @override + Future run() async { + setUpLogging(); + _checkGsUtil(); + final results = _SdkLocations(); + + for (final finder in _finders) { + await for (final sdkLocation in finder.find()) { + results.add(sdkLocation); + } + } + for (final MediaPipeSdk sdk in MediaPipeSdk.values) { + _log.info('Saving locations for ${sdk.name}'); + _writeResults(sdk, results.toMap(sdk)); + } + } + + void _writeResults(MediaPipeSdk sdk, _FlatResults results) { + final file = _getOutputFile(sdk); + _log.fine('Writing data to "${file.absolute.path}"'); + var encoder = JsonEncoder.withIndent(' '); + file.writeAsStringSync('''// Generated file. Do not manually edit. +final Map> sdkDownloadUrls = ${encoder.convert(results).replaceAll('"', "'")}; +'''); + } + + File _getOutputFile(MediaPipeSdk sdk) { + return File( + path.joinAll([ + findFlutterMediaPipeRoot().absolute.path, + 'packages/${sdk.toPackageDir()}', + 'sdk_downloads.dart', + ]), + ); + } + + void _checkGsUtil() async { + final process = await Process.start('which', ['gsutil']); + await process.exitCode; + if ((await process.processedStdOut).isEmpty) { + stderr.writeln( + wrapWith( + 'gsutil command not found. Visit: ' + 'https://cloud.google.com/storage/docs/gsutil_install', + [red], + ), + ); + exit(1); + } + } +} + +/// Main workhorse of the SdksFinderCommand. Navigates folders in GCS to find +/// the location of the latest builds for each [MediaPipeSdk] / [Architecture] +/// combination for the given [OS]. +/// +/// Usage: +/// ```dart +/// final macOsFinder = _OsFinder(OS.macOS); +/// await for (final _SdkLocation sdkLoc in macOsFinder.find()) { +/// doSomethingWithLocation(sdkLoc); +/// } +/// ``` +class _OsFinder { + _OsFinder(this.os); + + /// OS-specific upload directories located immediately inside + /// [SdksFinderCommand._bucketName]. + static const _gcsFolderPaths = { + OS.android: 'gcp_ubuntu_flutter', + OS.iOS: null, + OS.macOS: 'macos_flutter', + }; + + /// File extensions for OS-specific SDKs. + static const _sdkExtensions = { + OS.android: 'so', + OS.iOS: 'dylib', + OS.macOS: 'dylib', + }; + + /// Folders for specific [Target] values, where a Target is an OS/architecture + /// combination. + static const targetFolders = >{ + OS.android: {'android_arm64': Architecture.arm64}, + OS.iOS: {}, + OS.macOS: {'arm64': Architecture.arm64, 'x86_64': Architecture.x64}, + }; + + final OS os; + + String get folderPath => '${_gcsFolderPaths[os]!}/release'; + String get extension => _sdkExtensions[os]!; + + /// Scans the appropriate GCS location for all build Ids for the given OS and + /// returns the highest integer found. + Future _getBuildNumber(String path) async { + int highestBuildNumber = 0; + + for (final folder in await _gsUtil(path)) { + late int buildId; + try { + // Grab last chunk, since we're looking for a folder of the + // structure: `.../release/:int/` + buildId = int.parse(lastChunk(folder)); + } catch (e) { + // Probably the `{id}_$folder$` directory + continue; + } + if (buildId > highestBuildNumber) { + highestBuildNumber = buildId; + } + } + _log.fine('Highest build number for $os is $highestBuildNumber'); + return highestBuildNumber; + } + + /// Extracts the date within a build directory, which is where the final + /// artifacts can be found. + /// + /// Usage: + /// ```dart + /// final path = await _getDateOfBuildNumber('.../gcp_ubuntu_flutter/release/', 17); + /// print(path); + /// >>> ".../gcp_ubuntu_flutter/release/17/20231212-090734/" + /// ``` + Future _getDateOfBuildNumber(String path) async { + final foldersInBuild = await _gsUtil(path); + if (foldersInBuild.isEmpty || foldersInBuild.length > 2) { + final paths = + foldersInBuild.map((path) => ' • $path').toList().join('\n'); + _log.warning('Unexpectedly found ${foldersInBuild.length} entries inside ' + 'build folder: $path. Expected 1 or 2, of formats "/[date]/" and ' + 'optionally "/[date]_\$folder\$". Found:\n\n$paths\n'); + } + for (final folderPath in foldersInBuild) { + if (folderPath.endsWith('/')) { + final buildDateFolderPath = lastChunk(folderPath); + _log.fine('$folderPath :: $buildDateFolderPath'); + return buildDateFolderPath; + } + } + throw Exception( + 'Unexpected structure of build folder: "$path". Did not find match.', + ); + } + + /// Receives a [Path] like ".../[os_folder]/release/[build_number]/[date]/" + /// and yields all matching architecture folders within. + Stream _getArchitectectures(String path) async* { + for (final pathWithinBuild in await _gsUtil(path)) { + final maybeArchitecture = lastChunk(pathWithinBuild); + if (targetFolders[os]!.containsKey(maybeArchitecture)) { + yield maybeArchitecture; + } + } + } + + /// Combines the path, file name, and extension into the final, complete path. + /// Additionally, checks whether that file actually exists and returns the + /// String value if it does, or `null` if it does not. + Future _getAndCheckFullPath(String path, MediaPipeSdk sdk) async { + final pathToCheck = '$path/${sdk.name}.${_sdkExtensions[os]!}'; + final output = await _gsUtil(pathToCheck); + if (output.isEmpty) { + return null; + } + return pathToCheck; + } + + Stream<_SdkLocation> find() async* { + _log.info('Finding SDKs for $os'); + String path = folderPath; + final buildNumber = await _getBuildNumber(path); + path = '$path/$buildNumber'; + _log.finest('$os :: build number :: $path'); + final buildDate = await _getDateOfBuildNumber(path); + path = '$path/$buildDate'; + _log.finest('$os :: date :: $path'); + + await for (final String archPath in _getArchitectectures(path)) { + String pathWithArch = '$path/$archPath'; + _log.finest('$os :: $archPath :: $pathWithArch'); + for (final sdk in MediaPipeSdk.values) { + final maybeFinalPath = await _getAndCheckFullPath(pathWithArch, sdk); + _log.finest('$os :: maybeFinalPath :: $maybeFinalPath'); + if (maybeFinalPath != null) { + _log.fine('Found "$maybeFinalPath"'); + yield _SdkLocation( + os: os, + arch: targetFolders[os]![archPath]!, + sdk: sdk, + fullPath: '${SdksFinderCommand._gcsPrefix}/' + '${SdksFinderCommand._bucketName}/$maybeFinalPath', + ); + } + } + } + } +} + +// Runs `gsutil ls` against the path, optionally with the `-r` flag. +Future> _gsUtil(String path, {bool recursive = false}) async { + assert( + !path.startsWith('http'), + 'gsutil requires URIs with the "gs" scheme, which this function will add.', + ); + final cmd = [ + 'ls', + if (recursive) '-r', + 'gs://${SdksFinderCommand._bucketName}/$path', + ]; + _log.finest('Running: `gsutil ${cmd.join(' ')}`'); + final process = await Process.start('gsutil', cmd); + await process.exitCode; + final processStdout = await process.processedStdOut; + final filtered = (processStdout).where((String line) => line != '').toList(); + return filtered; +} + +/// Simple container for the location of a specific MediaPipe SDK in GCS. +class _SdkLocation { + _SdkLocation({ + required this.os, + required this.arch, + required this.sdk, + required this.fullPath, + }); + final OS os; + final Architecture arch; + final MediaPipeSdk sdk; + final String fullPath; + + @override + String toString() => '_SdkLocation(os: $os, arch: $arch, sdk: $sdk, ' + 'fullPath: $fullPath)'; +} + +/// Container for multiple [_SdkLocation] objects with support for quickly +/// extracting all records for a given MediaPipe task (audio, vision, or text). +class _SdkLocations { + final Map> _locations = {}; + + void add(_SdkLocation loc) { + _locations.setDefault(loc.sdk, <_SdkLocation>[]); + _locations[loc.sdk]!.add(loc); + } + + _FlatResults toMap(MediaPipeSdk sdk) { + if (!_locations.containsKey(sdk)) return {}; + + final _FlatResults results = >{}; + for (_SdkLocation loc in _locations[sdk]!) { + results.setDefault(loc.os.toString(), {}); + results[loc.os.toString()]![loc.arch.toString()] = loc.fullPath; + } + + return results; + } +} diff --git a/tool/builder/pubspec.yaml b/tool/builder/pubspec.yaml index 43d8dce8..33ca0c0c 100644 --- a/tool/builder/pubspec.yaml +++ b/tool/builder/pubspec.yaml @@ -4,7 +4,7 @@ description: Performs build operations for google/flutter-mediapipe that depend version: 1.0.0 # repository: https://github.com/my_org/my_repo environment: - sdk: ^3.2.0-162.0.dev + sdk: ^3.1.5 # Add regular dependencies here. dependencies: @@ -12,6 +12,7 @@ dependencies: http: ^1.1.0 io: ^1.0.4 logging: ^1.2.0 + native_assets_cli: ^0.3.2 path: ^1.8.0 process: ^5.0.0 From 30eed667e18961ee77dc4e724a99d36435eaf9fd Mon Sep 17 00:00:00 2001 From: Craig Labenz Date: Thu, 4 Jan 2024 13:07:39 -0800 Subject: [PATCH 2/4] propagates changes to existing commands --- tool/builder/lib/download_model.dart | 2 ++ tool/builder/lib/sync_headers.dart | 2 ++ 2 files changed, 4 insertions(+) diff --git a/tool/builder/lib/download_model.dart b/tool/builder/lib/download_model.dart index 14dc889d..a7c9e6d2 100644 --- a/tool/builder/lib/download_model.dart +++ b/tool/builder/lib/download_model.dart @@ -61,6 +61,7 @@ class DownloadModelCommand extends Command with RepoFinderMixin { 'if you use the `custommodel` option, but optional if you use the ' '`model` option.', ); + addVerboseOption(argParser); } static final Map _standardModelSources = { @@ -79,6 +80,7 @@ class DownloadModelCommand extends Command with RepoFinderMixin { @override Future run() async { + setUpLogging(); final io.Directory flutterMediaPipeDirectory = findFlutterMediaPipeRoot(); late final String modelSource; diff --git a/tool/builder/lib/sync_headers.dart b/tool/builder/lib/sync_headers.dart index aefe1e97..c4b383e7 100644 --- a/tool/builder/lib/sync_headers.dart +++ b/tool/builder/lib/sync_headers.dart @@ -57,10 +57,12 @@ class SyncHeadersCommand extends Command with RepoFinderMixin { 'at destination locations.', ); addSourceOption(argParser); + addVerboseOption(argParser); } @override Future run() async { + setUpLogging(); final io.Directory flutterMediaPipeDirectory = findFlutterMediaPipeRoot(); final io.Directory mediaPipeDirectory = findMediaPipeRoot( flutterMediaPipeDirectory, From c81cfb15f4de54fe1e878880c12c1a89511a6539 Mon Sep 17 00:00:00 2001 From: Craig Labenz Date: Thu, 4 Jan 2024 13:07:58 -0800 Subject: [PATCH 3/4] adds manifest files generated by new sdks_finder command --- packages/mediapipe-task-audio/sdk_downloads.dart | 2 ++ packages/mediapipe-task-text/sdk_downloads.dart | 10 ++++++++++ packages/mediapipe-task-vision/sdk_downloads.dart | 10 ++++++++++ 3 files changed, 22 insertions(+) create mode 100644 packages/mediapipe-task-audio/sdk_downloads.dart create mode 100644 packages/mediapipe-task-text/sdk_downloads.dart create mode 100644 packages/mediapipe-task-vision/sdk_downloads.dart diff --git a/packages/mediapipe-task-audio/sdk_downloads.dart b/packages/mediapipe-task-audio/sdk_downloads.dart new file mode 100644 index 00000000..1f0512e2 --- /dev/null +++ b/packages/mediapipe-task-audio/sdk_downloads.dart @@ -0,0 +1,2 @@ +// Generated file. Do not manually edit. +final Map> sdkDownloadUrls = {}; diff --git a/packages/mediapipe-task-text/sdk_downloads.dart b/packages/mediapipe-task-text/sdk_downloads.dart new file mode 100644 index 00000000..03548a31 --- /dev/null +++ b/packages/mediapipe-task-text/sdk_downloads.dart @@ -0,0 +1,10 @@ +// Generated file. Do not manually edit. +final Map> sdkDownloadUrls = { + 'android': { + 'arm64': 'https://storage.googleapis.com/mediapipe-nightly-public/prod/mediapipe/gcp_ubuntu_flutter/release/17/20231212-090734/android_arm64/libtext.so' + }, + 'macos': { + 'arm64': 'https://storage.googleapis.com/mediapipe-nightly-public/prod/mediapipe/macos_flutter/release/7/20231204-130423/arm64/libtext.dylib', + 'x64': 'https://storage.googleapis.com/mediapipe-nightly-public/prod/mediapipe/macos_flutter/release/7/20231204-130423/x86_64/libtext.dylib' + } +}; diff --git a/packages/mediapipe-task-vision/sdk_downloads.dart b/packages/mediapipe-task-vision/sdk_downloads.dart new file mode 100644 index 00000000..efcab84e --- /dev/null +++ b/packages/mediapipe-task-vision/sdk_downloads.dart @@ -0,0 +1,10 @@ +// Generated file. Do not manually edit. +final Map> sdkDownloadUrls = { + 'android': { + 'arm64': 'https://storage.googleapis.com/mediapipe-nightly-public/prod/mediapipe/gcp_ubuntu_flutter/release/17/20231212-090734/android_arm64/libvision.so' + }, + 'macos': { + 'arm64': 'https://storage.googleapis.com/mediapipe-nightly-public/prod/mediapipe/macos_flutter/release/7/20231204-130423/arm64/libvision.dylib', + 'x64': 'https://storage.googleapis.com/mediapipe-nightly-public/prod/mediapipe/macos_flutter/release/7/20231204-130423/x86_64/libvision.dylib' + } +}; From 69327425460a55e4163764bdda7d9d12fb41ac0d Mon Sep 17 00:00:00 2001 From: Craig Labenz Date: Tue, 9 Jan 2024 12:10:05 -0800 Subject: [PATCH 4/4] updates in response to code review --- Makefile | 4 +++ README.md | 18 ++++++++++++ tool/builder/lib/sdks_finder.dart | 49 ++++++++++++++++++++++++++----- 3 files changed, 64 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index 84f0a670..3543ba3f 100644 --- a/Makefile +++ b/Makefile @@ -11,6 +11,10 @@ test: generate_text test_text generate_core test_core # Runs `ffigen` for all packages and all tests for all packages test_only: test_core test_text +# Runs `sdks_finder` to update manifest files +sdks: + dart tool/builder/bin/main.dart sdks + # Core --- # Runs `ffigen` for `mediapipe_core` diff --git a/README.md b/README.md index 16bbaad8..817b5d83 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,21 @@ # Flutter-MediaPipe This repository will be home to the source code for the mediapipe_task_vision, mediapipe_task_audio, and mediapipe_task_text plugins for Flutter. + +## Releasing + +### Updating MediaPipe SDKs + +Anytime MediaPipe releases new versions of their SDKs, this package will need to be updated to incorporate those latest builds. SDK versions are pinned in the `sdk_downloads.json` files in each package, which are updated by running the following command from the root of the repository: + +``` +$ make sdks +``` + +The Google Cloud Storage bucket in question only gives read-list access to a specific list of Googlers' accounts, so this command must be run from such a Googler's corp machines. + +After this, create and merge a PR with the changes and then proceed to `Releasing to pub.dev`. + +### Releasing to pub.dev + +TODO \ No newline at end of file diff --git a/tool/builder/lib/sdks_finder.dart b/tool/builder/lib/sdks_finder.dart index 94f1eeec..30401dfe 100644 --- a/tool/builder/lib/sdks_finder.dart +++ b/tool/builder/lib/sdks_finder.dart @@ -1,5 +1,6 @@ import 'dart:convert'; import 'dart:io'; +import 'dart:io' as io; import 'package:args/command_runner.dart'; import 'package:builder/extensions.dart'; @@ -47,9 +48,12 @@ enum MediaPipeSdk { /// command helps by automating a portion of the task. /// /// The cache-busting mechanism of Flutter's native assets feature is a hash -/// of the contents of any build dependencies, so if this command leads to any -/// new build state, this must be reflected by *some change* to the associated -/// library's `build.dart` script. +/// of the contents of any build dependencies. The output files of this command +/// are included in the build dependencies (as specified by the contents of each +/// package's `build.dart` file), so if this command generates new SDK locations +/// in those files, Flutter's CLI will have a cache miss, will re-run +/// `build.dart` during the build phase, and in turn will download the newest +/// versions of the MediaPipe SDKs onto the developer's machine. /// /// Operationally, [SdksFinderCommand]'s implementation involves orchestrating /// one [_OsFinder] instance for each supported [OS] value, which in turn @@ -67,7 +71,7 @@ class SdksFinderCommand extends Command with RepoFinderMixin { } @override String description = - 'Downloads the appropriate MediaPipe SDKs for the current build target'; + 'Updates MediaPipe SDK manifest files for the current build target'; @override String name = 'sdks'; @@ -120,9 +124,26 @@ final Map> sdkDownloadUrls = ${encoder.convert(resul } void _checkGsUtil() async { + if (!io.Platform.isMacOS && !io.Platform.isLinux) { + // `which` is not available on Windows, so allow the command to attempt + // to run on Windows + // TODO: possibly add Windows-specific support + return; + } final process = await Process.start('which', ['gsutil']); - await process.exitCode; - if ((await process.processedStdOut).isEmpty) { + final exitCode = await process.exitCode; + final List processStdOut = await process.processedStdOut; + if (exitCode != 0) { + stderr.writeln( + wrapWith( + 'Warning: Unexpected exit code $exitCode checking for gsutil. Output:' + '${processStdOut.join('\n')}', + [yellow], + ), + ); + // Not exiting here, since this could be a false-negative. + } + if (processStdOut.isEmpty) { stderr.writeln( wrapWith( 'gsutil command not found. Visit: ' @@ -297,7 +318,21 @@ Future> _gsUtil(String path, {bool recursive = false}) async { ]; _log.finest('Running: `gsutil ${cmd.join(' ')}`'); final process = await Process.start('gsutil', cmd); - await process.exitCode; + final exitCode = await process.exitCode; + if (exitCode > 1) { + // Exit codes of 1 appear when `gsutil` checks for a file that does not + // exist, which for our purposes does not constitute an actual error, and is + // handled later when `process.processedStdOut` is empty. + stderr.writeln( + wrapWith( + 'Warning: Unexpected exit code $exitCode running ' + '`gsutil ${cmd.join(' ')}`. Output: ' + '${(await process.processedStdOut).join('\n')}', + [red], + ), + ); + exit(exitCode); + } final processStdout = await process.processedStdOut; final filtered = (processStdout).where((String line) => line != '').toList(); return filtered;