Add GenAI inference task

.github/workflows/main.yaml

@@ -5,7 +5,7 @@ permissions: read-all
 
 on:
   pull_request:
-    branches: [ main ]
+    branches: [main]
   push:
     branches: [main, ffi-wrapper, ffi-wrapper-text-pkg]
   schedule:

Makefile

@@ -57,4 +57,17 @@ test_text:
 	cd packages/mediapipe-task-text/example && flutter test
 
 example_text:
-	cd packages/mediapipe-task-text/example && flutter run -d macos
+	cd packages/mediapipe-task-text/example && flutter run -d macos
+
+# GenAI ---
+generate_genai:
+	cd packages/mediapipe-task-genai && dart --enable-experiment=native-assets run ffigen --config=ffigen.yaml
+
+# Example genai invocation.
+# Note that `GEMMA_4B_CPU_URI` can either be a local path or web URL. Similar values exist for
+# 8B and GPU variants.
+#
+# For desktop development, standard environment variables like this work great.
+# $ GEMMA_4B_CPU_URI=/path/to/gemma-2b-it-cpu-int4.bin flutter run -d [macos, windows, linux]
+# For emulator or attached device testing, use `--dart-define` for the same values.
+# $ flutter run -d [<device_id>] --dart-define=GEMMA_4B_CPU_URI=https://url/to.com/gemma-2b-it-cpu-int4.bin

packages/mediapipe-core/lib/src/interface/task_options.dart

@@ -6,9 +6,12 @@ import 'dart:typed_data';
 
 import 'package:equatable/equatable.dart';
 
-/// {@template TaskOptions}
+/// {@template BaseOptions}
 /// Root class for options classes for MediaPipe tasks.
-///
+/// {@endtemplate}
+abstract class Options extends Equatable {}
+
+/// {@template TaskOptions}
 /// Implementing classes will contain two [BaseInnerTaskOptions] subclasses,
 /// including a descendent of the universal options struct, [BaseBaseOptions].
 /// The second field will be task-specific.
@@ -17,7 +20,7 @@ import 'package:equatable/equatable.dart';
 /// This implementation is not immutable to track whether `dispose` has been
 /// called. All values used by pkg:equatable are in fact immutable.
 // ignore: must_be_immutable
-abstract class BaseTaskOptions extends Equatable {
+abstract class BaseTaskOptions extends Options {
   /// {@macro TaskOptions}
   BaseTaskOptions();
 

packages/mediapipe-core/lib/src/io/ffi_utils.dart

@@ -90,11 +90,11 @@ extension DartAwarePointerChars on Pointer<Pointer<Char>> {
   ///
   /// See also:
   ///  * [toDartString], for a non-list equivalent.
-  List<String?> toDartStrings(int length) {
+  List<String> toDartStrings(int length) {
     if (isNullPointer) {
       throw Exception('Unexpectedly called `toDartStrings` on nullptr');
     }
-    final dartStrings = <String?>[];
+    final dartStrings = <String>[];
     int counter = 0;
     while (counter < length) {
       dartStrings.add(this[counter].toDartString());

packages/mediapipe-core/lib/src/io/task_options.dart

@@ -15,7 +15,7 @@ import 'third_party/mediapipe/generated/mediapipe_common_bindings.dart'
 /// should manage their [InnerTaskOptions] fields. The two suggested methods are
 /// [copyToNative] and [dispose].
 /// {@endtemplate}
-mixin TaskOptions<T extends Struct> on BaseTaskOptions {
+mixin TaskOptions<T extends Struct> on Options {
   /// {@template TaskOptions.copyToNative}
   /// Copies these task options into native memory. Any fields of type
   /// [InnerTaskOptions] should have their `assignToStruct` method called.

packages/mediapipe-core/test/io/task_options_test.dart

@@ -86,13 +86,17 @@ void main() {
         expect(ptr.ref.score_threshold, lessThan(0.90001));
         expect(ptr.ref.category_allowlist_count, 3);
         expect(
-          ptr.ref.category_allowlist.toDartStrings(3),
+          ptr.ref.category_allowlist.toDartStrings(
+            3,
+          ),
           ['good', 'great', 'best'],
         );
 
         expect(ptr.ref.category_denylist_count, 4);
         expect(
-          ptr.ref.category_denylist.toDartStrings(4),
+          ptr.ref.category_denylist.toDartStrings(
+            4,
+          ),
           ['bad', 'terrible', 'worst', 'honestly come on'],
         );
       });

packages/mediapipe-task-audio/sdk_downloads.dart

@@ -1,2 +1,4 @@
 // Generated file. Do not manually edit.
-final Map<String, Map<String, String>> sdkDownloadUrls = {};
+// Used by the flutter toolchain (via build.dart) during compilation of any
+// Flutter app using this package.
+final Map<String, Map<String, Map<String, String>>> sdkDownloadUrls = {};

packages/mediapipe-task-genai/.gitignore

@@ -0,0 +1,29 @@
+# Miscellaneous
+*.class
+*.log
+*.pyc
+*.swp
+.DS_Store
+.atom/
+.buildlog/
+.history
+.svn/
+migrate_working_dir/
+
+# IntelliJ related
+*.iml
+*.ipr
+*.iws
+.idea/
+
+# The .vscode folder contains launch configuration and tasks you configure in
+# VS Code which you may wish to be included in version control, so this line
+# is commented out by default.
+#.vscode/
+
+# Flutter/Dart/Pub related
+# Libraries should not include pubspec.lock, per https://dart.dev/guides/libraries/private-files#pubspeclock.
+/pubspec.lock
+**/doc/api/
+.dart_tool/
+build/

packages/mediapipe-task-genai/.metadata

@@ -0,0 +1,10 @@
+# This file tracks properties of this Flutter project.
+# Used by Flutter tool to assess capabilities and perform upgrades etc.
+#
+# This file should be version controlled and should not be manually edited.
+
+version:
+  revision: "bd909542a33ab1d5249363a2434ae50ee468094f"
+  channel: "master"
+
+project_type: package

packages/mediapipe-task-genai/CHANGELOG.md

@@ -0,0 +1,3 @@
+## 0.0.1
+
+* TODO: Describe initial release.

packages/mediapipe-task-genai/LICENSE

@@ -0,0 +1 @@
+TODO: Add your license here.

packages/mediapipe-task-genai/README.md

@@ -0,0 +1,39 @@
+<!--
+This README describes the package. If you publish this package to pub.dev,
+this README's contents appear on the landing page for your package.
+
+For information about how to write a good package README, see the guide for
+[writing package pages](https://dart.dev/guides/libraries/writing-package-pages).
+
+For general information about developing packages, see the Dart guide for
+[creating packages](https://dart.dev/guides/libraries/create-library-packages)
+and the Flutter guide for
+[developing packages and plugins](https://flutter.dev/developing-packages).
+-->
+
+TODO: Put a short description of the package here that helps potential users
+know whether this package might be useful for them.
+
+## Features
+
+TODO: List what your package can do. Maybe include images, gifs, or videos.
+
+## Getting started
+
+TODO: List prerequisites and provide or point to information on how to
+start using the package.
+
+## Usage
+
+TODO: Include short and useful examples for package users. Add longer examples
+to `/example` folder.
+
+```dart
+const like = 'sample';
+```
+
+## Additional information
+
+TODO: Tell users more about the package: where to find more information, how to
+contribute to the package, how to file issues, what response they can expect
+from the package authors, and more.

packages/mediapipe-task-genai/analysis_options.yaml

@@ -0,0 +1,9 @@
+include: ../analysis_options.yaml
+
+linter:
+  rules:
+    - public_member_api_docs # see https://github.com/flutter/flutter/wiki/Style-guide-for-Flutter-repo#documentation-dartdocs-javadocs-etc
+
+analyzer:
+  exclude:
+    - "**/mediapipe_genai_bindings.dart"

packages/mediapipe-task-genai/build.dart

@@ -0,0 +1,122 @@
+import 'dart:io';
+import 'package:native_assets_cli/native_assets_cli.dart';
+import 'package:http/http.dart' as http;
+import 'package:path/path.dart' as path;
+
+import 'sdk_downloads.dart';
+
+late File logFile;
+
+final logs = <(DateTime, String)>[];
+void log(String msg) {
+  logs.add((DateTime.now(), msg));
+  if (!logFile.parent.existsSync()) {
+    logFile.parent.createSync();
+  }
+
+  if (logFile.existsSync()) {
+    logFile.deleteSync();
+  }
+  logFile.createSync();
+  logFile.writeAsStringSync(logs
+      .map<String>((rec) => '[${rec.$1.toIso8601String()}] ${rec.$2}')
+      .toList()
+      .join('\n\n'));
+}
+
+Future<void> main(List<String> args) async {
+  final buildConfig = await BuildConfig.fromArgs(args);
+  logFile = File(
+    path.joinAll([
+      Directory.current.path, // root dir of app using `mediapipe-task-xyz`
+      'build/${buildConfig.dryRun ? "dryrun" : "live-run"}-build-log.txt',
+    ]),
+  );
+
+  log(args.join(' '));
+  final String targetOs = buildConfig.targetOs.toString();
+
+  log('dir.current: ${Directory.current.absolute.path}');
+
+  // Throw if target runtime is unsupported.
+  if (!sdkDownloadUrls.containsKey(targetOs)) {
+    throw Exception('Unsupported target OS: $targetOs. '
+        'Supported values are: ${sdkDownloadUrls.keys.toSet()}');
+  }
+
+  final buildOutput = BuildOutput();
+  buildOutput.dependencies.dependencies
+      .add(buildConfig.packageRoot.resolve('build.dart'));
+  buildOutput.dependencies.dependencies
+      .add(buildConfig.packageRoot.resolve('sdk_downloads.dart'));
+
+  final modelName = 'libllm_inference_engine';
+  final Iterable<String> archKeys;
+  if (buildConfig.dryRun) {
+    archKeys = sdkDownloadUrls[targetOs]![modelName]!.keys;
+  } else {
+    archKeys = [buildConfig.targetArchitecture.toString()];
+  }
+  for (String arch in archKeys) {
+    arch = getArchAlias(arch);
+    log("arch: $arch");
+    log("sdkDownloadUrls[targetOs]: ${sdkDownloadUrls[targetOs]}");
+    log("sdkDownloadUrls[targetOs]['$modelName']: ${sdkDownloadUrls[targetOs]![modelName]}");
+    log("sdkDownloadUrls[targetOs]['$modelName'][$arch]: ${sdkDownloadUrls[targetOs]![modelName]![arch]}");
+
+    if (!sdkDownloadUrls[targetOs]!['libllm_inference_engine']!
+        .containsKey(arch)) {
+      continue;
+    }
+    final assetUrl =
+        sdkDownloadUrls[targetOs]!['libllm_inference_engine']![arch]!;
+    final downloadFileLocation = buildConfig.outDir.resolve(
+      '${arch}_${assetUrl.split('/').last}',
+    );
+    log('downloadFileLocation: $downloadFileLocation');
+    buildOutput.assets.add(
+      Asset(
+        id: 'package:mediapipe_genai/src/io/third_party/mediapipe/generated/mediapipe_genai_bindings.dart',
+        linkMode: LinkMode.dynamic,
+        target: Target.fromArchitectureAndOs(
+            Architecture.fromString(arch), buildConfig.targetOs),
+        path: AssetAbsolutePath(downloadFileLocation),
+      ),
+    );
+    if (!buildConfig.dryRun) {
+      downloadAsset(assetUrl, downloadFileLocation);
+    }
+  }
+
+  await buildOutput.writeToFile(outDir: buildConfig.outDir);
+}
+
+Future<void> downloadAsset(String assetUrl, Uri destinationFile) async {
+  final downloadUri = Uri.parse(assetUrl);
+  final downloadedFile = File(destinationFile.toFilePath());
+  log('Saving file to ${downloadedFile.absolute.path}');
+
+  final downloadResponse = await http.get(downloadUri);
+  log('Download response: ${downloadResponse.statusCode}');
+
+  if (downloadResponse.statusCode == 200) {
+    if (downloadedFile.existsSync()) {
+      downloadedFile.deleteSync();
+    }
+    downloadedFile.createSync();
+    log('Saved file to ${downloadedFile.absolute.path}\n');
+    downloadedFile.writeAsBytes(downloadResponse.bodyBytes);
+  } else {
+    log('${downloadResponse.statusCode} :: ${downloadResponse.body}');
+    throw Exception(
+        '${downloadResponse.statusCode} :: ${downloadResponse.body}');
+  }
+}
+
+/// Translates native-assets architecture names into MediaPipe architecture names
+String getArchAlias(String arch) {
+  return <String, String>{
+        'arm': 'arm64',
+      }[arch] ??
+      arch;
+}

packages/mediapipe-task-genai/example/.gitignore

@@ -0,0 +1,43 @@
+# Miscellaneous
+*.class
+*.log
+*.pyc
+*.swp
+.DS_Store
+.atom/
+.buildlog/
+.history
+.svn/
+migrate_working_dir/
+
+# IntelliJ related
+*.iml
+*.ipr
+*.iws
+.idea/
+
+# The .vscode folder contains launch configuration and tasks you configure in
+# VS Code which you may wish to be included in version control, so this line
+# is commented out by default.
+#.vscode/
+
+# Flutter/Dart/Pub related
+**/doc/api/
+**/ios/Flutter/.last_build_id
+.dart_tool/
+.flutter-plugins
+.flutter-plugins-dependencies
+.pub-cache/
+.pub/
+/build/
+
+# Symbolication related
+app.*.symbols
+
+# Obfuscation related
+app.*.map.json
+
+# Android Studio will place build artifacts here
+/android/app/debug
+/android/app/profile
+/android/app/release

packages/mediapipe-task-genai/example/.metadata

@@ -0,0 +1,30 @@
+# This file tracks properties of this Flutter project.
+# Used by Flutter tool to assess capabilities and perform upgrades etc.
+#
+# This file should be version controlled and should not be manually edited.
+
+version:
+  revision: "86135b7774e32fa7b0ad0d116511471f36048579"
+  channel: "master"
+
+project_type: app
+
+# Tracks metadata for the flutter migrate command
+migration:
+  platforms:
+    - platform: root
+      create_revision: 86135b7774e32fa7b0ad0d116511471f36048579
+      base_revision: 86135b7774e32fa7b0ad0d116511471f36048579
+    - platform: ios
+      create_revision: 86135b7774e32fa7b0ad0d116511471f36048579
+      base_revision: 86135b7774e32fa7b0ad0d116511471f36048579
+
+  # User provided section
+
+  # List of Local paths (relative to this file) that should be
+  # ignored by the migrate tool.
+  #
+  # Files that are not part of the templates will be ignored by default.
+  unmanaged_files:
+    - 'lib/main.dart'
+    - 'ios/Runner.xcodeproj/project.pbxproj'