Adds example to mediapipe-task-text

.github/workflows/main.yaml

@@ -7,7 +7,7 @@ on:
   push:
     branches: [main]
   pull_request:
-    branches: [main, ffi-wrapper, ffi-wrapper-core-pkg]
+    branches: [main, ffi-wrapper, ffi-wrapper-text-pkg]
   workflow_dispatch:
   schedule:
     - cron: "0 0 * * *" # Every day at midnight
@@ -25,7 +25,9 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        flutter_version: [stable, beta, master]
+        flutter_version: [master]
+        # TODO(craiglabenz): reverse once CI works
+        # flutter_version: [stable, beta, master]
         # TODO(craiglabenz): Add `ubuntu-latest` and `windows-latest` when those artifacts exist
         os: [macos-latest]
     steps:

Makefile

@@ -5,17 +5,25 @@ headers:
 # Runs `ffigen` for all packages
 generate: generate_core generate_text
 
-# Runs `ffigen` for all packages, compiles the faked C artifacts, and runs all tests
+# Runs `ffigen` for all packages and runs all tests
 test: generate_text test_text generate_core test_core
 
-# Runs `ffigen` for all packages and all tests for all packages
+# Runs all tests for all packages
 test_only: test_core test_text
 
 # Rebuilds the MediaPipe task for macOS
 # Assumes google/mediapipe and google/flutter-mediapipe are siblings on the file system
-compile_text_classifier_macos:
+compile_text_classifier_macos_arm:
 	cd ../mediapipe && bazel build --linkopt -s --config darwin_arm64 --strip always --define MEDIAPIPE_DISABLE_GPU=1 mediapipe/tasks/c/text/text_classifier:libtext_classifier.dylib
-	cd ../mediapipe && sudo cp bazel-bin/mediapipe/tasks/c/text/text_classifier/libtext_classifier.dylib ../flutter-mediapipe/packages/mediapipe-task-text/example/assets
+	cd ../mediapipe && sudo cp bazel-bin/mediapipe/tasks/c/text/text_classifier/libtext_classifier.dylib ../flutter-mediapipe/packages/mediapipe-task-text/example/assets/libtext_classifier_arm64.dylib
+
+compile_text_classifier_macos_x86:
+	cd ../mediapipe && bazel build --linkopt -s --config darwin_x86_64 --strip always --define MEDIAPIPE_DISABLE_GPU=1 mediapipe/tasks/c/text/text_classifier:libtext_classifier.dylib
+	cd ../mediapipe && sudo cp bazel-bin/mediapipe/tasks/c/text/text_classifier/libtext_classifier.dylib ../flutter-mediapipe/packages/mediapipe-task-text/example/assets/libtext_classifier_x64.dylib
+
+# Runs `sdks_finder` to update manifest files
+sdks:
+	dart tool/builder/bin/main.dart sdks
 
 # Core ---
 
@@ -33,9 +41,10 @@ test_core:
 generate_text:
 	cd packages/mediapipe-task-text && dart --enable-experiment=native-assets run ffigen --config=ffigen.yaml
 
-# Runs `ffigen` for `mediapipe_text` and all text tests
-test_text: compile_fake_text test_text_only
-
 # Runs all text tests
-test_text_only:
+test_text:
 	cd packages/mediapipe-task-text && flutter test
+	cd packages/mediapipe-task-text/example && flutter test
+
+example_text:
+	cd packages/mediapipe-task-text/example && flutter run -d macos

README.md

@@ -1,3 +1,21 @@
 # Flutter-MediaPipe
 
 This repository will be home to the source code for the mediapipe_task_vision, mediapipe_task_audio, and mediapipe_task_text plugins for Flutter.
+
+## Releasing
+
+### Updating MediaPipe SDKs
+
+Anytime MediaPipe releases new versions of their SDKs, this package will need to be updated to incorporate those latest builds. SDK versions are pinned in the `sdk_downloads.json` files in each package, which are updated by running the following command from the root of the repository:
+
+```
+$ make sdks
+```
+
+The Google Cloud Storage bucket in question only gives read-list access to a specific list of Googlers' accounts, so this command must be run from such a Googler's corp machines.
+
+After this, create and merge a PR with the changes and then proceed to `Releasing to pub.dev`.
+
+### Releasing to pub.dev
+
+TODO

packages/mediapipe-core/lib/generated/core_symbols.yaml

@@ -6,6 +6,8 @@ files:
     symbols:
       c:@S@BaseOptions:
         name: BaseOptions
+      c:@S@Categories:
+        name: Categories
       c:@S@Category:
         name: Category
       c:@S@ClassificationResult:

packages/mediapipe-core/lib/src/containers.dart

@@ -40,19 +40,19 @@ class Category {
 
   /// Accepts a pointer to a list of structs, and a count representing the length
   /// of the list, and returns a list of pure-Dart [Category] instances.
-  static List<Category> fromStructs(
+  static List<Category> structsToDart(
     Pointer<bindings.Category> structs,
     int count,
   ) {
     final categories = <Category>[];
     for (int i = 0; i < count; i++) {
-      categories.add(fromStruct(structs[i]));
+      categories.add(structToDart(structs[i]));
     }
     return categories;
   }
 
   /// Accepts a pointer to a single struct and returns a pure-Dart [Category] instance.
-  static Category fromStruct(bindings.Category struct) {
+  static Category structToDart(bindings.Category struct) {
     return Category(
       index: struct.index,
       score: struct.score,
@@ -62,19 +62,25 @@ class Category {
   }
 
   /// Releases all C memory associated with a list of [bindings.Category] pointers.
-  /// This method is important to call after calling [Category.fromStructs] to
+  /// This method is important to call after calling [Category.structsToDart] to
   /// convert that C memory into pure-Dart objects.
   static void freeStructs(Pointer<bindings.Category> structs, int count) {
     int index = 0;
     while (index < count) {
       bindings.Category obj = structs[index];
-      calloc.free(obj.category_name);
-      calloc.free(obj.display_name);
+      Category.freeStruct(obj);
       index++;
     }
     calloc.free(structs);
   }
 
+  /// Releases the string fields associated with a single [bindings.Category]
+  /// struct.
+  static void freeStruct(bindings.Category struct) {
+    calloc.free(struct.category_name);
+    calloc.free(struct.display_name);
+  }
+
   @override
   String toString() => 'Category(index=$index, score=$score, '
       'categoryName=$categoryName, displayName=$displayName)';
@@ -108,22 +114,22 @@ class Classifications {
 
   /// Accepts a pointer to a list of structs, and a count representing the length
   /// of the list, and returns a list of pure-Dart [Classifications] instances.
-  static List<Classifications> fromStructs(
+  static List<Classifications> structsToDart(
     Pointer<bindings.Classifications> structs,
     int count,
   ) {
     final classifications = <Classifications>[];
     for (int i = 0; i < count; i++) {
-      classifications.add(fromStruct(structs[i]));
+      classifications.add(structToDart(structs[i]));
     }
     return classifications;
   }
 
   /// Accepts a pointer to a single struct and returns a pure-Dart [Classifications]
   /// instance.
-  static Classifications fromStruct(bindings.Classifications struct) {
+  static Classifications structToDart(bindings.Classifications struct) {
     return Classifications(
-      categories: Category.fromStructs(
+      categories: Category.structsToDart(
         struct.categories,
         struct.categories_count,
       ),
@@ -133,7 +139,7 @@ class Classifications {
   }
 
   /// Releases all C memory associated with a list of [bindings.Classifications]
-  /// pointers. This method is important to call after calling [Classifications.fromStructs]
+  /// pointers. This method is important to call after calling [Classifications.structsToDart]
   /// to convert that C memory into pure-Dart objects.
   static void freeStructs(
     Pointer<bindings.Classifications> structs,

packages/mediapipe-core/lib/src/task_options.dart

@@ -21,51 +21,89 @@ import 'third_party/mediapipe/generated/mediapipe_common_bindings.dart'
 ///    classifier's desired behavior.
 class BaseOptions extends Equatable {
   /// Generative constructor that creates a [BaseOptions] instance.
-  const BaseOptions({this.modelAssetBuffer, this.modelAssetPath})
-      : assert(
+  const BaseOptions._({
+    this.modelAssetBuffer,
+    this.modelAssetPath,
+    required _BaseOptionsType type,
+  })  : assert(
           !(modelAssetBuffer == null && modelAssetPath == null),
           'You must supply either `modelAssetBuffer` or `modelAssetPath`',
         ),
         assert(
           !(modelAssetBuffer != null && modelAssetPath != null),
           'You must only supply one of `modelAssetBuffer` and `modelAssetPath`',
-        );
+        ),
+        _type = type;
+
+  /// Constructor for [BaseOptions] classes using a file system path.
+  ///
+  /// In practice, this is unsupported, as assets in Flutter are bundled into
+  /// the build output and not available on disk. However, it can potentially
+  /// be helpful for testing / development purposes.
+  factory BaseOptions.path(String path) => BaseOptions._(
+        modelAssetPath: path,
+        type: _BaseOptionsType.path,
+      );
+
+  /// Constructor for [BaseOptions] classes using an in-memory pointer to the
+  /// MediaPipe SDK.
+  ///
+  /// In practice, this is the only option supported for production builds.
+  factory BaseOptions.memory(Uint8List buffer) {
+    return BaseOptions._(
+      modelAssetBuffer: buffer,
+      type: _BaseOptionsType.memory,
+    );
+  }
 
   /// The model asset file contents as bytes;
   final Uint8List? modelAssetBuffer;
 
   /// Path to the model asset file.
   final String? modelAssetPath;
 
+  final _BaseOptionsType _type;
+
   /// Converts this pure-Dart representation into C-memory suitable for the
   /// MediaPipe SDK to instantiate various classifiers.
   Pointer<bindings.BaseOptions> toStruct() {
     final struct = calloc<bindings.BaseOptions>();
 
-    if (modelAssetPath != null) {
-      struct.ref.model_asset_path = prepareString(modelAssetPath!);
-    }
-    if (modelAssetBuffer != null) {
-      struct.ref.model_asset_buffer = prepareUint8List(modelAssetBuffer!);
-      struct.ref.model_asset_buffer_count = modelAssetBuffer!.lengthInBytes;
+    switch (_type) {
+      case _BaseOptionsType.path:
+        {
+          struct.ref.model_asset_path = prepareString(modelAssetPath!);
+        }
+      case _BaseOptionsType.memory:
+        {
+          struct.ref.model_asset_buffer = prepareUint8List(modelAssetBuffer!);
+          struct.ref.model_asset_buffer_count = modelAssetBuffer!.lengthInBytes;
+        }
     }
+
     return struct;
   }
 
-  @override
-  List<Object?> get props => [modelAssetBuffer, modelAssetPath];
-
   /// Releases all C memory held by this [bindings.BaseOptions] struct.
   static void freeStruct(bindings.BaseOptions struct) {
-    if (struct.model_asset_buffer.address != 0) {
-      calloc.free(struct.model_asset_buffer);
-    }
-    if (struct.model_asset_path.address != 0) {
+    if (struct.model_asset_path.isNotNullPointer) {
       calloc.free(struct.model_asset_path);
     }
+    if (struct.model_asset_buffer.isNotNullPointer) {
+      calloc.free(struct.model_asset_buffer);
+    }
   }
+
+  @override
+  List<Object?> get props => [
+        modelAssetBuffer,
+        modelAssetPath,
+        modelAssetBuffer?.lengthInBytes,
+      ];
 }
 
+enum _BaseOptionsType { path, memory }
+
 /// Dart representation of MediaPipe's "ClassifierOptions" concept.
 ///
 /// Classifier options shared across MediaPipe classification tasks.

packages/mediapipe-core/lib/src/test_utils.dart

@@ -0,0 +1,54 @@
+import 'dart:ffi';
+import 'package:ffi/ffi.dart';
+
+import 'package:mediapipe_core/mediapipe_core.dart';
+import 'package:mediapipe_core/src/third_party/mediapipe/generated/mediapipe_common_bindings.dart'
+    as core_bindings;
+
+/// Hydrates a faked [core_bindings.Category] object.
+void populateCategory(
+  core_bindings.Category category, {
+  int index = 1,
+  double score = 0.9,
+  String? categoryName = 'Positive',
+  String? displayName,
+}) {
+  category.index = index;
+  category.score = score;
+
+  if (categoryName != null) {
+    category.category_name = prepareString(categoryName);
+  }
+  if (displayName != null) {
+    category.display_name = prepareString(displayName);
+  }
+}
+
+/// Hydrates a faked [core_bindings.Classifications] object.
+///
+/// If [categories] is passed, [numCategories] must indicate its length;
+/// otherwise, this function generates [numCategories] faked
+/// [core_bindings.Category] objects.
+void populateClassifications(
+  core_bindings.Classifications classifications, {
+  Pointer<core_bindings.Category>? categories,
+  int numCategories = 2,
+  int headIndex = 1,
+  String headName = 'Head',
+}) {
+  if (isNotNullOrNullPointer(categories)) {
+    classifications.categories = categories!;
+    classifications.categories_count = numCategories;
+  } else {
+    final ptrs = calloc<core_bindings.Category>(numCategories);
+    int count = 0;
+    while (count < numCategories) {
+      populateCategory(ptrs[count]);
+      count++;
+    }
+    classifications.categories = ptrs;
+    classifications.categories_count = numCategories;
+  }
+  classifications.head_name = prepareString(headName);
+  classifications.head_index = headIndex;
+}

packages/mediapipe-core/lib/src/third_party/mediapipe/generated/mediapipe_common_bindings.dart

@@ -22,10 +22,10 @@ import 'dart:ffi' as ffi;
 final class BaseOptions extends ffi.Struct {
   external ffi.Pointer<ffi.Char> model_asset_buffer;
 
-  external ffi.Pointer<ffi.Char> model_asset_path;
-
   @ffi.UnsignedInt()
   external int model_asset_buffer_count;
+
+  external ffi.Pointer<ffi.Char> model_asset_path;
 }
 
 final class __mbstate_t extends ffi.Union {
@@ -152,6 +152,13 @@ final class Category extends ffi.Struct {
   external ffi.Pointer<ffi.Char> display_name;
 }
 
+final class Categories extends ffi.Struct {
+  external ffi.Pointer<Category> categories;
+
+  @ffi.Uint32()
+  external int categories_count;
+}
+
 final class Classifications extends ffi.Struct {
   external ffi.Pointer<Category> categories;