[FFI] MediaPipe SDKs finder automation (#16)

* adds sdks_finder command to builder utility

* propagates changes to existing commands

* adds manifest files generated by new sdks_finder command

* updates in response to code review

Author: craiglabenz

Makefile

@@ -11,6 +11,10 @@ test: generate_text test_text generate_core test_core
 # Runs `ffigen` for all packages and all tests for all packages
 test_only: test_core test_text
 
+# Runs `sdks_finder` to update manifest files
+sdks:
+	dart tool/builder/bin/main.dart sdks
+
 # Core ---
 
 # Runs `ffigen` for `mediapipe_core`

README.md

@@ -1,3 +1,21 @@
 # Flutter-MediaPipe
 
 This repository will be home to the source code for the mediapipe_task_vision, mediapipe_task_audio, and mediapipe_task_text plugins for Flutter.
+
+## Releasing
+
+### Updating MediaPipe SDKs
+
+Anytime MediaPipe releases new versions of their SDKs, this package will need to be updated to incorporate those latest builds. SDK versions are pinned in the `sdk_downloads.json` files in each package, which are updated by running the following command from the root of the repository:
+
+```
+$ make sdks
+```
+
+The Google Cloud Storage bucket in question only gives read-list access to a specific list of Googlers' accounts, so this command must be run from such a Googler's corp machines.
+
+After this, create and merge a PR with the changes and then proceed to `Releasing to pub.dev`.
+
+### Releasing to pub.dev
+
+TODO

packages/mediapipe-task-audio/sdk_downloads.dart

@@ -0,0 +1,2 @@
+// Generated file. Do not manually edit.
+final Map<String, Map<String, String>> sdkDownloadUrls = {};

packages/mediapipe-task-text/sdk_downloads.dart

@@ -0,0 +1,10 @@
+// Generated file. Do not manually edit.
+final Map<String, Map<String, String>> sdkDownloadUrls = {
+  'android': {
+    'arm64': 'https://storage.googleapis.com/mediapipe-nightly-public/prod/mediapipe/gcp_ubuntu_flutter/release/17/20231212-090734/android_arm64/libtext.so'
+  },
+  'macos': {
+    'arm64': 'https://storage.googleapis.com/mediapipe-nightly-public/prod/mediapipe/macos_flutter/release/7/20231204-130423/arm64/libtext.dylib',
+    'x64': 'https://storage.googleapis.com/mediapipe-nightly-public/prod/mediapipe/macos_flutter/release/7/20231204-130423/x86_64/libtext.dylib'
+  }
+};

packages/mediapipe-task-vision/sdk_downloads.dart

@@ -0,0 +1,10 @@
+// Generated file. Do not manually edit.
+final Map<String, Map<String, String>> sdkDownloadUrls = {
+  'android': {
+    'arm64': 'https://storage.googleapis.com/mediapipe-nightly-public/prod/mediapipe/gcp_ubuntu_flutter/release/17/20231212-090734/android_arm64/libvision.so'
+  },
+  'macos': {
+    'arm64': 'https://storage.googleapis.com/mediapipe-nightly-public/prod/mediapipe/macos_flutter/release/7/20231204-130423/arm64/libvision.dylib',
+    'x64': 'https://storage.googleapis.com/mediapipe-nightly-public/prod/mediapipe/macos_flutter/release/7/20231204-130423/x86_64/libvision.dylib'
+  }
+};

tool/builder/bin/main.dart

@@ -2,24 +2,18 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
-import 'dart:io' as io;
 import 'package:args/command_runner.dart';
 import 'package:builder/download_model.dart';
+import 'package:builder/sdks_finder.dart';
 import 'package:builder/sync_headers.dart';
-import 'package:logging/logging.dart';
 
 final runner = CommandRunner(
   'build',
   'Performs build operations for google/flutter-mediapipe that '
       'depend on contents in this repository',
 )
-  ..addCommand(SyncHeadersCommand())
-  ..addCommand(DownloadModelCommand());
+  ..addCommand(DownloadModelCommand())
+  ..addCommand(SdksFinderCommand())
+  ..addCommand(SyncHeadersCommand());
 
-void main(List<String> arguments) {
-  Logger.root.onRecord.listen((LogRecord record) {
-    io.stdout
-        .writeln('${record.level.name}: ${record.time}: ${record.message}');
-  });
-  runner.run(arguments);
-}
+void main(List<String> arguments) => runner.run(arguments);

tool/builder/lib/download_model.dart

@@ -61,6 +61,7 @@ class DownloadModelCommand extends Command with RepoFinderMixin {
             'if you use the `custommodel` option, but optional if you use the '
             '`model` option.',
       );
+    addVerboseOption(argParser);
   }
 
   static final Map<String, String> _standardModelSources = {
@@ -79,6 +80,7 @@ class DownloadModelCommand extends Command with RepoFinderMixin {
 
   @override
   Future<void> run() async {
+    setUpLogging();
     final io.Directory flutterMediaPipeDirectory = findFlutterMediaPipeRoot();
 
     late final String modelSource;

tool/builder/lib/extensions.dart

@@ -0,0 +1,35 @@
+import 'dart:convert';
+import 'dart:io';
+
+extension EasyOutput on Process {
+  Future<List<String>> get processedStdErr => _process(this.stderr);
+
+  Future<List<String>> get processedStdOut => _process(this.stdout);
+
+  Future<List<String>> _process(Stream<List<int>> stream) async {
+    return utf8.decoder
+        .convert((await stream.toList())
+            .fold<List<int>>([], (arr, el) => arr..addAll(el)))
+        .split('\n');
+  }
+}
+
+/// Returns the last full chunk from a Url-like String.
+///
+/// From "/an/awesome/url/", returns "url".
+/// From "/an/awesome/url", returns "url".
+/// From "/an/awesome/url/", with a depth of 1, returns "awesome"
+/// From "/an/awesome/url", with a depth of 1, returns "awesome"
+String lastChunk(String url, {int depth = 0}) {
+  final indexOffset = (url.endsWith('/')) ? -2 - depth : -1 - depth;
+  final splitUrl = url.split('/');
+  return splitUrl[splitUrl.length + indexOffset];
+}
+
+extension DefaultableMap<K, V> on Map {
+  void setDefault(K key, V def) {
+    if (!containsKey(key)) {
+      this[key] = def;
+    }
+  }
+}

tool/builder/lib/repo_finder.dart

@@ -5,6 +5,7 @@
 import 'dart:io' as io;
 import 'package:args/args.dart';
 import 'package:args/command_runner.dart';
+import 'package:logging/logging.dart';
 import 'package:path/path.dart' as path;
 import 'package:io/ansi.dart';
 
@@ -34,6 +35,18 @@ mixin RepoFinderMixin on Command {
     );
   }
 
+  void addVerboseOption(ArgParser argParser) =>
+      argParser.addFlag('verbose', abbr: 'v', defaultsTo: false);
+
+  void setUpLogging() {
+    final bool verbose = argResults!['verbose'];
+    Logger.root.level = verbose ? Level.FINEST : Level.INFO;
+    Logger.root.onRecord.listen((LogRecord record) {
+      io.stdout.writeln(
+          '[${record.loggerName}][${record.level.name}] ${record.message}');
+    });
+  }
+
   /// Looks upward for the root of the `google/mediapipe` repository. This assumes
   /// the `dart build` command is executed from within said repository. If it is
   /// not executed from within, then this searching algorithm will reach the root