From c9eb71d13000c5f41a4465316d5fe76862e704da Mon Sep 17 00:00:00 2001 From: Craig Labenz Date: Fri, 13 Oct 2023 10:07:35 -0700 Subject: [PATCH 1/8] adding bare structure for native assets --- packages/mediapipe-task-text/build.dart | 1 + packages/mediapipe-task-text/pubspec.yaml | 2 ++ 2 files changed, 3 insertions(+) create mode 100644 packages/mediapipe-task-text/build.dart diff --git a/packages/mediapipe-task-text/build.dart b/packages/mediapipe-task-text/build.dart new file mode 100644 index 00000000..f60f88b1 --- /dev/null +++ b/packages/mediapipe-task-text/build.dart @@ -0,0 +1 @@ +void main(List args) async {} diff --git a/packages/mediapipe-task-text/pubspec.yaml b/packages/mediapipe-task-text/pubspec.yaml index 830a0af7..ed851407 100644 --- a/packages/mediapipe-task-text/pubspec.yaml +++ b/packages/mediapipe-task-text/pubspec.yaml @@ -18,6 +18,8 @@ dependencies: logging: ^1.2.0 mediapipe_core: path: ../mediapipe-core + native_assets_cli: ^0.3.0 + native_toolchain_c: ^0.3.0 dev_dependencies: ffigen: ^9.0.1 From 8777ed395007da1935e9d1c4298e9afe75aea5df Mon Sep 17 00:00:00 2001 From: Craig Labenz Date: Tue, 24 Oct 2023 15:51:09 -0700 Subject: [PATCH 2/8] add MVP / first draft of build.dart --- packages/mediapipe-task-text/build.dart | 58 ++++++++++++++++++++++++- 1 file changed, 57 insertions(+), 1 deletion(-) diff --git a/packages/mediapipe-task-text/build.dart b/packages/mediapipe-task-text/build.dart index f60f88b1..93439679 100644 --- a/packages/mediapipe-task-text/build.dart +++ b/packages/mediapipe-task-text/build.dart @@ -1 +1,57 @@ -void main(List args) async {} +import 'dart:io'; +import 'package:http/http.dart' as http; +import 'package:native_assets_cli/native_assets_cli.dart'; +import 'package:native_toolchain_c/native_toolchain_c.dart'; + +const assetFilename = 'libtext_classifier.dylib'; +const assetLocation = + 'https://storage.cloud.google.com/random-storage-asdf/$assetFilename'; + +File outputFile = File( + '/Users/craiglabenz/Dev/git/google/flutter-mediapipe/packages/mediapipe-task-text/logs-build.txt'); + +final logs = []; + +void main(List args) async { + if (await outputFile.exists()) { + await outputFile.delete(); + } + await outputFile.create(); + + log(args.join(' ')); + _build(args); + outputFile.writeAsString(logs.join('\n')); +} + +Future _build(List args) async { + final buildConfig = await BuildConfig.fromArgs(args); + final buildOutput = BuildOutput(); + final downloadUri = Uri.parse(assetLocation); + final downloadFileLocation = buildConfig.outDir.resolve(assetFilename); + log('Downloading $downloadUri'); + final downloadResponse = await http.get(downloadUri); + final downloadedFile = File(downloadFileLocation.toFilePath()); + if (downloadResponse.statusCode == 200) { + if (downloadedFile.existsSync()) { + downloadedFile.deleteSync(); + } + downloadedFile.createSync(); + downloadedFile.writeAsBytes(downloadResponse.bodyBytes); + } + buildOutput.dependencies.dependencies + .add(buildConfig.packageRoot.resolve('build.dart')); + buildOutput.assets.add( + Asset( + id: 'package:mediapipe_text/src/mediapipe_text_bindings.dart', + linkMode: LinkMode.dynamic, + target: Target.macOSArm64, + path: AssetAbsolutePath(downloadFileLocation), + ), + ); + await buildOutput.writeToFile(outDir: buildConfig.outDir); + log(buildConfig.outDir.toString()); +} + +void log(String val) { + logs.add('\n[${DateTime.now().toIso8601String()}] $val\n'); +} From ce33a8dcde39f113b71d366bed49b17977dbfc8f Mon Sep 17 00:00:00 2001 From: Craig Labenz Date: Wed, 25 Oct 2023 12:13:01 -0700 Subject: [PATCH 3/8] build.dart updates --- packages/mediapipe-task-text/build.dart | 11 +++++++---- packages/mediapipe-task-text/example/pubspec.yaml | 1 - packages/mediapipe-task-text/pubspec.yaml | 1 + 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/packages/mediapipe-task-text/build.dart b/packages/mediapipe-task-text/build.dart index 93439679..f7b9c049 100644 --- a/packages/mediapipe-task-text/build.dart +++ b/packages/mediapipe-task-text/build.dart @@ -1,11 +1,10 @@ import 'dart:io'; -import 'package:http/http.dart' as http; import 'package:native_assets_cli/native_assets_cli.dart'; -import 'package:native_toolchain_c/native_toolchain_c.dart'; +import 'package:http/http.dart' as http; const assetFilename = 'libtext_classifier.dylib'; const assetLocation = - 'https://storage.cloud.google.com/random-storage-asdf/$assetFilename'; + 'https://storage.googleapis.com/random-storage-asdf/$assetFilename'; File outputFile = File( '/Users/craiglabenz/Dev/git/google/flutter-mediapipe/packages/mediapipe-task-text/logs-build.txt'); @@ -37,12 +36,16 @@ Future _build(List args) async { } downloadedFile.createSync(); downloadedFile.writeAsBytes(downloadResponse.bodyBytes); + } else { + log('${downloadResponse.statusCode} :: ${downloadResponse.body}'); + return; } buildOutput.dependencies.dependencies .add(buildConfig.packageRoot.resolve('build.dart')); buildOutput.assets.add( Asset( - id: 'package:mediapipe_text/src/mediapipe_text_bindings.dart', + // What should this `id` be? + id: 'package:mediapipe_text/mediapipe_text.dart', linkMode: LinkMode.dynamic, target: Target.macOSArm64, path: AssetAbsolutePath(downloadFileLocation), diff --git a/packages/mediapipe-task-text/example/pubspec.yaml b/packages/mediapipe-task-text/example/pubspec.yaml index 714a533a..1335c091 100644 --- a/packages/mediapipe-task-text/example/pubspec.yaml +++ b/packages/mediapipe-task-text/example/pubspec.yaml @@ -25,4 +25,3 @@ flutter: assets: - assets/bert_classifier.tflite - - assets/libtext_classifier.dylib diff --git a/packages/mediapipe-task-text/pubspec.yaml b/packages/mediapipe-task-text/pubspec.yaml index ed851407..a443216d 100644 --- a/packages/mediapipe-task-text/pubspec.yaml +++ b/packages/mediapipe-task-text/pubspec.yaml @@ -16,6 +16,7 @@ dependencies: sdk: flutter http: ^1.1.0 logging: ^1.2.0 + http: ^1.1.0 mediapipe_core: path: ../mediapipe-core native_assets_cli: ^0.3.0 From 33f1d0b5027810137a89c68a94f49440d6048465 Mon Sep 17 00:00:00 2001 From: Craig Labenz Date: Tue, 31 Oct 2023 09:19:00 -0700 Subject: [PATCH 4/8] update build.dart TODO: stream file --- packages/mediapipe-task-text/build.dart | 28 +++++++++++++------------ 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/packages/mediapipe-task-text/build.dart b/packages/mediapipe-task-text/build.dart index f7b9c049..95f7e13d 100644 --- a/packages/mediapipe-task-text/build.dart +++ b/packages/mediapipe-task-text/build.dart @@ -25,27 +25,29 @@ void main(List args) async { Future _build(List args) async { final buildConfig = await BuildConfig.fromArgs(args); final buildOutput = BuildOutput(); - final downloadUri = Uri.parse(assetLocation); final downloadFileLocation = buildConfig.outDir.resolve(assetFilename); - log('Downloading $downloadUri'); - final downloadResponse = await http.get(downloadUri); - final downloadedFile = File(downloadFileLocation.toFilePath()); - if (downloadResponse.statusCode == 200) { - if (downloadedFile.existsSync()) { - downloadedFile.deleteSync(); + if (!buildConfig.dryRun) { + final downloadUri = Uri.parse(assetLocation); + log('Downloading $downloadUri'); + final downloadResponse = await http.get(downloadUri); + final downloadedFile = File(downloadFileLocation.toFilePath()); + if (downloadResponse.statusCode == 200) { + if (downloadedFile.existsSync()) { + downloadedFile.deleteSync(); + } + downloadedFile.createSync(); + downloadedFile.writeAsBytes(downloadResponse.bodyBytes); + } else { + log('${downloadResponse.statusCode} :: ${downloadResponse.body}'); + return; } - downloadedFile.createSync(); - downloadedFile.writeAsBytes(downloadResponse.bodyBytes); - } else { - log('${downloadResponse.statusCode} :: ${downloadResponse.body}'); - return; } buildOutput.dependencies.dependencies .add(buildConfig.packageRoot.resolve('build.dart')); buildOutput.assets.add( Asset( // What should this `id` be? - id: 'package:mediapipe_text/mediapipe_text.dart', + id: 'package:mediapipe_text/src/mediapipe_text_bindings.dart', linkMode: LinkMode.dynamic, target: Target.macOSArm64, path: AssetAbsolutePath(downloadFileLocation), From 612a4ab99d4d59bef21302ec37cf8840657a4d17 Mon Sep 17 00:00:00 2001 From: Craig Labenz Date: Mon, 6 Nov 2023 08:34:27 -0800 Subject: [PATCH 5/8] model memory troubleshooting --- .../mediapipe-core/lib/src/task_options.dart | 65 ++++++++++++++----- .../generated/mediapipe_common_bindings.dart | 2 +- .../test/task_options_test.dart | 24 +------ .../mediapipe-task-text/example/lib/main.dart | 1 + .../containers/text_classifier_options.dart | 4 +- 5 files changed, 56 insertions(+), 40 deletions(-) diff --git a/packages/mediapipe-core/lib/src/task_options.dart b/packages/mediapipe-core/lib/src/task_options.dart index c0f6fa7b..6291efb2 100644 --- a/packages/mediapipe-core/lib/src/task_options.dart +++ b/packages/mediapipe-core/lib/src/task_options.dart @@ -21,31 +21,68 @@ import 'third_party/mediapipe/generated/mediapipe_common_bindings.dart' /// classifier's desired behavior. class BaseOptions extends Equatable { /// Generative constructor that creates a [BaseOptions] instance. - const BaseOptions({this.modelAssetBuffer, this.modelAssetPath}) - : assert( + const BaseOptions._({ + this.modelAssetBuffer, + this.modelAssetPath, + this.modelAssetBufferCount, + required _BaseOptionsType type, + }) : assert( !(modelAssetBuffer == null && modelAssetPath == null), 'You must supply either `modelAssetBuffer` or `modelAssetPath`', ), assert( !(modelAssetBuffer != null && modelAssetPath != null), 'You must only supply one of `modelAssetBuffer` and `modelAssetPath`', - ); + ), + assert( + (modelAssetBuffer == null) == (modelAssetBufferCount == null), + 'modelAssetBuffer and modelAssetBufferCount must only be submitted ' + 'together', + ), + _type = type; + + /// Constructor for [BaseOptions] classes using a file system path. + /// + /// In practice, this is unsupported, as assets in Flutter are bundled into + /// the build output and not available on disk. However, it can potentially + /// be helpful for testing / development purposes. + factory BaseOptions.path(String path) => BaseOptions._( + modelAssetPath: path, + type: _BaseOptionsType.path, + ); + + /// Constructor for [BaseOptions] classes using an in-memory pointer to the + /// MediaPipe SDK. + /// + /// In practice, this is the only option supported for production builds. + factory BaseOptions.memory(Uint8List buffer) { + return BaseOptions._( + modelAssetBuffer: buffer, + modelAssetBufferCount: buffer.lengthInBytes, + type: _BaseOptionsType.memory, + ); + } /// The model asset file contents as bytes; final Uint8List? modelAssetBuffer; + /// The size of the model assets buffer (or `0` if not set). + final int? modelAssetBufferCount; + /// Path to the model asset file. final String? modelAssetPath; + final _BaseOptionsType _type; + /// Converts this pure-Dart representation into C-memory suitable for the /// MediaPipe SDK to instantiate various classifiers. Pointer toStruct() { final struct = calloc(); - if (modelAssetPath != null) { + if (_type == _BaseOptionsType.path) { struct.ref.model_asset_path = prepareString(modelAssetPath!); } - if (modelAssetBuffer != null) { + if (_type == _BaseOptionsType.memory) { struct.ref.model_asset_buffer = prepareUint8List(modelAssetBuffer!); struct.ref.model_asset_buffer_count = modelAssetBuffer!.lengthInBytes; } @@ -53,19 +90,15 @@ class BaseOptions extends Equatable { } @override - List get props => [modelAssetBuffer, modelAssetPath]; - - /// Releases all C memory held by this [bindings.BaseOptions] struct. - static void freeStruct(bindings.BaseOptions struct) { - if (struct.model_asset_buffer.address != 0) { - calloc.free(struct.model_asset_buffer); - } - if (struct.model_asset_path.address != 0) { - calloc.free(struct.model_asset_path); - } - } + List get props => [ + modelAssetBuffer, + modelAssetPath, + modelAssetBufferCount, + ]; } +enum _BaseOptionsType { path, memory } + /// Dart representation of MediaPipe's "ClassifierOptions" concept. /// /// Classifier options shared across MediaPipe classification tasks. diff --git a/packages/mediapipe-core/lib/src/third_party/mediapipe/generated/mediapipe_common_bindings.dart b/packages/mediapipe-core/lib/src/third_party/mediapipe/generated/mediapipe_common_bindings.dart index f6217336..51a2a2ef 100644 --- a/packages/mediapipe-core/lib/src/third_party/mediapipe/generated/mediapipe_common_bindings.dart +++ b/packages/mediapipe-core/lib/src/third_party/mediapipe/generated/mediapipe_common_bindings.dart @@ -24,7 +24,7 @@ final class BaseOptions extends ffi.Struct { external ffi.Pointer model_asset_path; - @ffi.UnsignedInt() + @ffi.Int() external int model_asset_buffer_count; } diff --git a/packages/mediapipe-core/test/task_options_test.dart b/packages/mediapipe-core/test/task_options_test.dart index 143a9ff4..f0947679 100644 --- a/packages/mediapipe-core/test/task_options_test.dart +++ b/packages/mediapipe-core/test/task_options_test.dart @@ -9,32 +9,16 @@ import 'package:test/test.dart'; import 'package:mediapipe_core/mediapipe_core.dart'; void main() { - group('BaseOptions constructor should', () { - test('enforce exactly one of modelPath and modelBuffer', () { - expect( - () => BaseOptions( - modelAssetPath: 'abc', - modelAssetBuffer: Uint8List.fromList([1, 2, 3]), - ), - throwsA(TypeMatcher()), - ); - - expect(BaseOptions.new, throwsA(TypeMatcher())); - }); - }); - group('BaseOptions.toStruct/fromStruct should', () { test('allocate memory in C for a modelAssetPath', () { - final options = BaseOptions(modelAssetPath: 'abc'); + final options = BaseOptions.path('abc'); final struct = options.toStruct(); expect(toDartString(struct.ref.model_asset_path), 'abc'); expectNullPtr(struct.ref.model_asset_buffer); }); test('allocate memory in C for a modelAssetBuffer', () { - final options = BaseOptions( - modelAssetBuffer: Uint8List.fromList([1, 2, 3]), - ); + final options = BaseOptions.memory(Uint8List.fromList([1, 2, 3])); final struct = options.toStruct(); expect( toUint8List(struct.ref.model_asset_buffer), @@ -44,9 +28,7 @@ void main() { }); test('allocate memory in C for a modelAssetBuffer containing 0', () { - final options = BaseOptions( - modelAssetBuffer: Uint8List.fromList([1, 2, 0, 3]), - ); + final options = BaseOptions.memory(Uint8List.fromList([1, 2, 0, 3])); final struct = options.toStruct(); expect( toUint8List(struct.ref.model_asset_buffer), diff --git a/packages/mediapipe-task-text/example/lib/main.dart b/packages/mediapipe-task-text/example/lib/main.dart index acbf1237..26a4d060 100644 --- a/packages/mediapipe-task-text/example/lib/main.dart +++ b/packages/mediapipe-task-text/example/lib/main.dart @@ -38,6 +38,7 @@ class _MainAppState extends State { super.initState(); _controller.text = 'Hello, world!'; _initClassifier(); + Future.delayed(const Duration(milliseconds: 500)).then((_) => _classify()); } Future _initClassifier() async { diff --git a/packages/mediapipe-task-text/lib/src/tasks/text_classification/containers/text_classifier_options.dart b/packages/mediapipe-task-text/lib/src/tasks/text_classification/containers/text_classifier_options.dart index f7fb7e55..00a76d51 100644 --- a/packages/mediapipe-task-text/lib/src/tasks/text_classification/containers/text_classifier_options.dart +++ b/packages/mediapipe-task-text/lib/src/tasks/text_classification/containers/text_classifier_options.dart @@ -32,7 +32,7 @@ class TextClassifierOptions { }) { assert(!kIsWeb, 'fromAssetPath cannot be used on the web'); return TextClassifierOptions( - baseOptions: BaseOptions(modelAssetPath: assetPath), + baseOptions: BaseOptions.path(assetPath), classifierOptions: classifierOptions, ); } @@ -45,7 +45,7 @@ class TextClassifierOptions { ClassifierOptions classifierOptions = const ClassifierOptions(), }) => TextClassifierOptions( - baseOptions: BaseOptions(modelAssetBuffer: assetBuffer), + baseOptions: BaseOptions.memory(assetBuffer), classifierOptions: classifierOptions, ); From b89d9b7c60ca21aab4801eb72e8a7303da179bc5 Mon Sep 17 00:00:00 2001 From: Craig Labenz Date: Wed, 8 Nov 2023 09:19:44 -0800 Subject: [PATCH 6/8] vendoring script tweak --- packages/mediapipe-task-text/build.dart | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/mediapipe-task-text/build.dart b/packages/mediapipe-task-text/build.dart index 95f7e13d..092f2831 100644 --- a/packages/mediapipe-task-text/build.dart +++ b/packages/mediapipe-task-text/build.dart @@ -2,9 +2,10 @@ import 'dart:io'; import 'package:native_assets_cli/native_assets_cli.dart'; import 'package:http/http.dart' as http; -const assetFilename = 'libtext_classifier.dylib'; +const cloudAssetFilename = 'libtext_classifier-v0.0.3.dylib'; +const localAssetFilename = 'libtext_classifier.dylib'; const assetLocation = - 'https://storage.googleapis.com/random-storage-asdf/$assetFilename'; + 'https://storage.googleapis.com/random-storage-asdf/$cloudAssetFilename'; File outputFile = File( '/Users/craiglabenz/Dev/git/google/flutter-mediapipe/packages/mediapipe-task-text/logs-build.txt'); @@ -25,7 +26,7 @@ void main(List args) async { Future _build(List args) async { final buildConfig = await BuildConfig.fromArgs(args); final buildOutput = BuildOutput(); - final downloadFileLocation = buildConfig.outDir.resolve(assetFilename); + final downloadFileLocation = buildConfig.outDir.resolve(localAssetFilename); if (!buildConfig.dryRun) { final downloadUri = Uri.parse(assetLocation); log('Downloading $downloadUri'); From 806c082df3764983237d797f564eb312d5183627 Mon Sep 17 00:00:00 2001 From: Craig Labenz Date: Mon, 13 Nov 2023 14:58:47 -0800 Subject: [PATCH 7/8] remove development logging --- packages/mediapipe-task-text/build.dart | 22 ++-------------------- 1 file changed, 2 insertions(+), 20 deletions(-) diff --git a/packages/mediapipe-task-text/build.dart b/packages/mediapipe-task-text/build.dart index 092f2831..8ed79af7 100644 --- a/packages/mediapipe-task-text/build.dart +++ b/packages/mediapipe-task-text/build.dart @@ -7,20 +7,8 @@ const localAssetFilename = 'libtext_classifier.dylib'; const assetLocation = 'https://storage.googleapis.com/random-storage-asdf/$cloudAssetFilename'; -File outputFile = File( - '/Users/craiglabenz/Dev/git/google/flutter-mediapipe/packages/mediapipe-task-text/logs-build.txt'); - -final logs = []; - void main(List args) async { - if (await outputFile.exists()) { - await outputFile.delete(); - } - await outputFile.create(); - - log(args.join(' ')); _build(args); - outputFile.writeAsString(logs.join('\n')); } Future _build(List args) async { @@ -29,7 +17,6 @@ Future _build(List args) async { final downloadFileLocation = buildConfig.outDir.resolve(localAssetFilename); if (!buildConfig.dryRun) { final downloadUri = Uri.parse(assetLocation); - log('Downloading $downloadUri'); final downloadResponse = await http.get(downloadUri); final downloadedFile = File(downloadFileLocation.toFilePath()); if (downloadResponse.statusCode == 200) { @@ -39,8 +26,8 @@ Future _build(List args) async { downloadedFile.createSync(); downloadedFile.writeAsBytes(downloadResponse.bodyBytes); } else { - log('${downloadResponse.statusCode} :: ${downloadResponse.body}'); - return; + throw Exception( + '${downloadResponse.statusCode} :: ${downloadResponse.body}'); } } buildOutput.dependencies.dependencies @@ -55,9 +42,4 @@ Future _build(List args) async { ), ); await buildOutput.writeToFile(outDir: buildConfig.outDir); - log(buildConfig.outDir.toString()); -} - -void log(String val) { - logs.add('\n[${DateTime.now().toIso8601String()}] $val\n'); } From 5549f93a18dba22eb3698ab98cdb804b9c44af32 Mon Sep 17 00:00:00 2001 From: Craig Labenz Date: Mon, 13 Nov 2023 14:59:27 -0800 Subject: [PATCH 8/8] removes pointless build method --- packages/mediapipe-task-text/build.dart | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/packages/mediapipe-task-text/build.dart b/packages/mediapipe-task-text/build.dart index 8ed79af7..f35c7e3b 100644 --- a/packages/mediapipe-task-text/build.dart +++ b/packages/mediapipe-task-text/build.dart @@ -7,11 +7,7 @@ const localAssetFilename = 'libtext_classifier.dylib'; const assetLocation = 'https://storage.googleapis.com/random-storage-asdf/$cloudAssetFilename'; -void main(List args) async { - _build(args); -} - -Future _build(List args) async { +Future main(List args) async { final buildConfig = await BuildConfig.fromArgs(args); final buildOutput = BuildOutput(); final downloadFileLocation = buildConfig.outDir.resolve(localAssetFilename);