moves async delivery of task to isolates

as Flutter's compute method does not allow for reuse of inner resources

Author: craiglabenz

.vscode/settings.json

@@ -0,0 +1,8 @@
+{
+    "files.associations": {
+        "base_options.h": "c",
+        "__config": "c",
+        "text_classifier.h": "c",
+        "category.h": "c"
+    }
+}

Makefile

@@ -11,6 +11,12 @@ test: generate_text test_text generate_core test_core
 # Runs `ffigen` for all packages and all tests for all packages
 test_only: test_core test_text
 
+# Rebuilds the MediaPipe task for macOS
+# Assumes google/mediapipe and google/flutter-mediapipe are siblings on the file system
+compile_text_classifier_macos:
+	cd ../mediapipe && bazel build --linkopt -s --config darwin_arm64 --strip always --define MEDIAPIPE_DISABLE_GPU=1 mediapipe/tasks/c/text/text_classifier:libtext_classifier.dylib
+	cd ../mediapipe && sudo cp bazel-bin/mediapipe/tasks/c/text/text_classifier/libtext_classifier.dylib ../flutter-mediapipe/packages/mediapipe-task-text/example/assets
+
 # Core ---
 
 # Runs `ffigen` for `mediapipe_core`
@@ -25,7 +31,7 @@ test_core:
 
 # Runs `ffigen` for `mediapipe_text`
 generate_text:
-	cd packages/mediapipe-task-text && dart run ffigen --config=ffigen.yaml
+	cd packages/mediapipe-task-text && dart --enable-experiment=native-assets run ffigen --config=ffigen.yaml
 
 # Compiles the faked C artifacts for testing
 compile_fake_text:

packages/mediapipe-core/ffigen.yaml

@@ -7,7 +7,7 @@ output:
     import-path: "package:mediapipe_core/src/third_party/mediapipe/generated/mediapipe_common_bindings.dart"
 headers:
   entry-points:
-    - "third_party/mediapipe/tasks/c/**"
+    - "third_party/mediapipe/tasks/c/**.h"
 preamble: |
   /* Copyright 2023 The MediaPipe Authors.
 

packages/mediapipe-core/lib/src/ffi_utils.dart

@@ -76,3 +76,21 @@ int _length(Pointer<Uint8> codeUnits) {
   }
   return length;
 }
+
+/// Offers convenience and readability extensions for detecting null pointers.
+extension NullAwarePtr on Pointer {
+  /// Returns true if this is a null pointer.
+  bool get isNullPointer => address == 0;
+
+  /// Returns true if this is not a null pointer.
+  bool get isNotNullPointer => address != 0;
+}
+
+/// Returns true if this nullable pointer is both not-null and not a `NullPtr`,
+/// which is to say, its address is 0x00000000.
+bool isNotNullOrNullPointer(Pointer? ptr) =>
+    ptr != null && ptr.isNotNullPointer;
+
+/// Returns true if this nullable pointer is either null OR is a `NullPtr`,
+/// which is to say, its address is 0x00000000.
+bool isNullOrNullPointer(Pointer? ptr) => ptr == null || ptr.isNullPointer;

packages/mediapipe-core/lib/src/task_options.dart

@@ -47,6 +47,7 @@ class BaseOptions extends Equatable {
     }
     if (modelAssetBuffer != null) {
       struct.ref.model_asset_buffer = prepareUint8List(modelAssetBuffer!);
+      struct.ref.model_asset_buffer_count = modelAssetBuffer!.lengthInBytes;
     }
     return struct;
   }

packages/mediapipe-core/lib/src/third_party/mediapipe/generated/mediapipe_common_bindings.dart

@@ -23,6 +23,9 @@ final class BaseOptions extends ffi.Struct {
   external ffi.Pointer<ffi.Char> model_asset_buffer;
 
   external ffi.Pointer<ffi.Char> model_asset_path;
+
+  @ffi.UnsignedInt()
+  external int model_asset_buffer_count;
 }
 
 final class __mbstate_t extends ffi.Union {

packages/mediapipe-core/third_party/mediapipe/tasks/c/core/base_options.h

@@ -20,14 +20,17 @@ limitations under the License.
 extern "C" {
 #endif
 
-// Base options for MediaPipe C Tasks.
-struct BaseOptions {
-  // The model asset file contents as a string.
-  char* model_asset_buffer;
-
-  // The path to the model asset to open and mmap in memory.
-  char* model_asset_path;
-};
+  // Base options for MediaPipe C Tasks.
+  struct BaseOptions {
+    // The model asset file contents as a string.
+    char* model_asset_buffer;
+
+    // The path to the model asset to open and mmap in memory.
+    char* model_asset_path;
+
+    // The size of the model assets buffer (or `0` if not set).
+    unsigned int model_asset_buffer_count;
+  };
 
 #ifdef __cplusplus
 }  // extern C

packages/mediapipe-task-text/ffigen.yaml

@@ -4,7 +4,7 @@ output:
   bindings: "lib/src/mediapipe_text_bindings.dart"
 headers:
   entry-points:
-    - "third_party/mediapipe/tasks/c/**"
+    - "third_party/mediapipe/tasks/c/**.h"
 import:
   symbol-files:
     - "package:mediapipe_core/generated/core_symbols.yaml"

packages/mediapipe-task-text/lib/src/mediapipe_text_bindings.dart

@@ -18,25 +18,39 @@ import 'dart:ffi' as ffi;
 import 'package:mediapipe_core/src/third_party/mediapipe/generated/mediapipe_common_bindings.dart'
     as imp1;
 
-@ffi.Native<ffi.Pointer<ffi.Void> Function(ffi.Pointer<TextClassifierOptions>)>(
-    symbol: 'text_classifier_create')
+@ffi.Native<
+    ffi.Pointer<ffi.Void> Function(ffi.Pointer<TextClassifierOptions>,
+        ffi.Pointer<ffi.Pointer<ffi.Char>>)>(symbol: 'text_classifier_create')
 external ffi.Pointer<ffi.Void> text_classifier_create(
   ffi.Pointer<TextClassifierOptions> options,
+  ffi.Pointer<ffi.Pointer<ffi.Char>> error_msg,
 );
 
 @ffi.Native<
-    ffi.Int Function(ffi.Pointer<ffi.Void>, ffi.Pointer<ffi.Char>,
-        ffi.Pointer<TextClassifierResult>)>(symbol: 'text_classifier_classify')
+    ffi.Int Function(
+        ffi.Pointer<ffi.Void>,
+        ffi.Pointer<ffi.Char>,
+        ffi.Pointer<TextClassifierResult>,
+        ffi.Pointer<ffi.Pointer<ffi.Char>>)>(symbol: 'text_classifier_classify')
 external int text_classifier_classify(
   ffi.Pointer<ffi.Void> classifier,
   ffi.Pointer<ffi.Char> utf8_str,
   ffi.Pointer<TextClassifierResult> result,
+  ffi.Pointer<ffi.Pointer<ffi.Char>> error_msg,
 );
 
-@ffi.Native<ffi.Void Function(ffi.Pointer<ffi.Void>)>(
-    symbol: 'text_classifier_close')
-external void text_classifier_close(
+@ffi.Native<ffi.Void Function(ffi.Pointer<TextClassifierResult>)>(
+    symbol: 'text_classifier_close_result')
+external void text_classifier_close_result(
+  ffi.Pointer<TextClassifierResult> result,
+);
+
+@ffi.Native<
+    ffi.Int Function(ffi.Pointer<ffi.Void>,
+        ffi.Pointer<ffi.Pointer<ffi.Char>>)>(symbol: 'text_classifier_close')
+external int text_classifier_close(
   ffi.Pointer<ffi.Void> classifier,
+  ffi.Pointer<ffi.Pointer<ffi.Char>> error_msg,
 );
 
 final class TextClassifierOptions extends ffi.Struct {

packages/mediapipe-task-text/lib/src/tasks/text_classification/containers/text_classifier_result.dart

@@ -1,5 +1,3 @@
-import 'dart:ffi';
-import 'package:ffi/ffi.dart';
 import 'package:mediapipe_core/mediapipe_core.dart';
 import 'package:mediapipe_text/src/mediapipe_text_bindings.dart' as bindings;
 
@@ -41,16 +39,6 @@ class TextClassifierResult {
     );
   }
 
-  /// Deallocates all memory associated with an [bindings.ImageClassifierResult]
-  /// in C memory.
-  static void freeStruct(Pointer<bindings.TextClassifierResult> struct) {
-    Classifications.freeStructs(
-      struct.ref.classifications,
-      struct.ref.classifications_count,
-    );
-    calloc.free(struct);
-  }
-
   /// Convenience helper for the first [Classifications] object.
   Classifications? get firstClassification =>
       classifications.isNotEmpty ? classifications.first : null;