From 8ddef51ff94b9e2036277a92f500ba220573c525 Mon Sep 17 00:00:00 2001 From: Craig Labenz Date: Fri, 13 Oct 2023 09:10:34 -0700 Subject: [PATCH 01/17] adds mediapipe_core package --- packages/mediapipe-core/.gitignore | 7 + packages/mediapipe-core/CHANGELOG.md | 3 + packages/mediapipe-core/README.md | 39 ++ packages/mediapipe-core/analysis_options.yaml | 24 ++ packages/mediapipe-core/ffigen.yaml | 13 + .../lib/generated/core_symbols.yaml | 38 ++ .../mediapipe-core/lib/mediapipe_core.dart | 5 + .../mediapipe-core/lib/src/containers.dart | 109 ++++++ .../mediapipe-core/lib/src/ffi_utils.dart | 52 +++ .../lib/src/mediapipe_common_bindings.dart | 332 ++++++++++++++++++ .../mediapipe-core/lib/src/task_options.dart | 120 +++++++ packages/mediapipe-core/pubspec.yaml | 17 + .../test/task_options_test.dart | 116 ++++++ .../tasks/c/components/containers/category.h | 50 +++ .../containers/classification_result.h | 68 ++++ .../processors/classifier_options.h | 59 ++++ .../mediapipe/tasks/c/core/base_options.h | 36 ++ 17 files changed, 1088 insertions(+) create mode 100644 packages/mediapipe-core/.gitignore create mode 100644 packages/mediapipe-core/CHANGELOG.md create mode 100644 packages/mediapipe-core/README.md create mode 100644 packages/mediapipe-core/analysis_options.yaml create mode 100644 packages/mediapipe-core/ffigen.yaml create mode 100644 packages/mediapipe-core/lib/generated/core_symbols.yaml create mode 100644 packages/mediapipe-core/lib/mediapipe_core.dart create mode 100644 packages/mediapipe-core/lib/src/containers.dart create mode 100644 packages/mediapipe-core/lib/src/ffi_utils.dart create mode 100644 packages/mediapipe-core/lib/src/mediapipe_common_bindings.dart create mode 100644 packages/mediapipe-core/lib/src/task_options.dart create mode 100644 packages/mediapipe-core/pubspec.yaml create mode 100644 packages/mediapipe-core/test/task_options_test.dart create mode 100644 packages/mediapipe-core/third_party/mediapipe/tasks/c/components/containers/category.h create mode 100644 packages/mediapipe-core/third_party/mediapipe/tasks/c/components/containers/classification_result.h create mode 100644 packages/mediapipe-core/third_party/mediapipe/tasks/c/components/processors/classifier_options.h create mode 100644 packages/mediapipe-core/third_party/mediapipe/tasks/c/core/base_options.h diff --git a/packages/mediapipe-core/.gitignore b/packages/mediapipe-core/.gitignore new file mode 100644 index 00000000..3cceda55 --- /dev/null +++ b/packages/mediapipe-core/.gitignore @@ -0,0 +1,7 @@ +# https://dart.dev/guides/libraries/private-files +# Created by `dart pub` +.dart_tool/ + +# Avoid committing pubspec.lock for library packages; see +# https://dart.dev/guides/libraries/private-files#pubspeclock. +pubspec.lock diff --git a/packages/mediapipe-core/CHANGELOG.md b/packages/mediapipe-core/CHANGELOG.md new file mode 100644 index 00000000..effe43c8 --- /dev/null +++ b/packages/mediapipe-core/CHANGELOG.md @@ -0,0 +1,3 @@ +## 1.0.0 + +- Initial version. diff --git a/packages/mediapipe-core/README.md b/packages/mediapipe-core/README.md new file mode 100644 index 00000000..8b55e735 --- /dev/null +++ b/packages/mediapipe-core/README.md @@ -0,0 +1,39 @@ + + +TODO: Put a short description of the package here that helps potential users +know whether this package might be useful for them. + +## Features + +TODO: List what your package can do. Maybe include images, gifs, or videos. + +## Getting started + +TODO: List prerequisites and provide or point to information on how to +start using the package. + +## Usage + +TODO: Include short and useful examples for package users. Add longer examples +to `/example` folder. + +```dart +const like = 'sample'; +``` + +## Additional information + +TODO: Tell users more about the package: where to find more information, how to +contribute to the package, how to file issues, what response they can expect +from the package authors, and more. diff --git a/packages/mediapipe-core/analysis_options.yaml b/packages/mediapipe-core/analysis_options.yaml new file mode 100644 index 00000000..0e9249f3 --- /dev/null +++ b/packages/mediapipe-core/analysis_options.yaml @@ -0,0 +1,24 @@ +# This file configures the static analysis results for your project (errors, +# warnings, and lints). +# +# This enables the 'recommended' set of lints from `package:lints`. +# This set helps identify many issues that may lead to problems when running +# or consuming Dart code, and enforces writing Dart using a single, idiomatic +# style and format. +# +# If you want a smaller set of lints you can change this to specify +# 'package:lints/core.yaml'. These are just the most critical lints +# (the recommended set includes the core lints). +# The core lints are also what is used by pub.dev for scoring packages. + +include: package:lints/recommended.yaml + +# Uncomment the following section to specify additional rules. + +# linter: +# rules: +# - camel_case_types + +analyzer: + exclude: + - "lib/src/mediapipe_common_bindings.dart" diff --git a/packages/mediapipe-core/ffigen.yaml b/packages/mediapipe-core/ffigen.yaml new file mode 100644 index 00000000..10c8a880 --- /dev/null +++ b/packages/mediapipe-core/ffigen.yaml @@ -0,0 +1,13 @@ +name: "MediaPipeCommonBindings" +description: "Bindings for shared MediaPipe structs common across many tasks" +output: + bindings: "lib/src/mediapipe_common_bindings.dart" + symbol-file: + output: "package:mediapipe_core/generated/core_symbols.yaml" + import-path: "package:mediapipe_core/src/mediapipe_common_bindings.dart" +headers: + entry-points: + - "third_party/mediapipe/tasks/c/core/base_options.h" + - "third_party/mediapipe/tasks/c/components/containers/category.h" + - "third_party/mediapipe/tasks/c/components/containers/classification_result.h" + - "third_party/mediapipe/tasks/c/components/processors/classifier_options.h" diff --git a/packages/mediapipe-core/lib/generated/core_symbols.yaml b/packages/mediapipe-core/lib/generated/core_symbols.yaml new file mode 100644 index 00000000..497d7c36 --- /dev/null +++ b/packages/mediapipe-core/lib/generated/core_symbols.yaml @@ -0,0 +1,38 @@ +format_version: 1.0.0 +files: + package:mediapipe_core/src/mediapipe_common_bindings.dart: + used-config: + ffi-native: false + symbols: + c:@S@BaseOptions: + name: BaseOptions + c:@S@Category: + name: Category + c:@S@ClassificationResult: + name: ClassificationResult + c:@S@Classifications: + name: Classifications + c:@S@ClassifierOptions: + name: ClassifierOptions + c:@S@__darwin_pthread_handler_rec: + name: __darwin_pthread_handler_rec + c:@S@_opaque_pthread_attr_t: + name: _opaque_pthread_attr_t + c:@S@_opaque_pthread_cond_t: + name: _opaque_pthread_cond_t + c:@S@_opaque_pthread_condattr_t: + name: _opaque_pthread_condattr_t + c:@S@_opaque_pthread_mutex_t: + name: _opaque_pthread_mutex_t + c:@S@_opaque_pthread_mutexattr_t: + name: _opaque_pthread_mutexattr_t + c:@S@_opaque_pthread_once_t: + name: _opaque_pthread_once_t + c:@S@_opaque_pthread_rwlock_t: + name: _opaque_pthread_rwlock_t + c:@S@_opaque_pthread_rwlockattr_t: + name: _opaque_pthread_rwlockattr_t + c:@S@_opaque_pthread_t: + name: _opaque_pthread_t + c:@UA@__mbstate_t: + name: __mbstate_t diff --git a/packages/mediapipe-core/lib/mediapipe_core.dart b/packages/mediapipe-core/lib/mediapipe_core.dart new file mode 100644 index 00000000..2e62b181 --- /dev/null +++ b/packages/mediapipe-core/lib/mediapipe_core.dart @@ -0,0 +1,5 @@ +library; + +export 'src/containers.dart' show Category, Classifications; +export 'src/ffi_utils.dart'; +export 'src/task_options.dart' show BaseOptions, ClassifierOptions; diff --git a/packages/mediapipe-core/lib/src/containers.dart b/packages/mediapipe-core/lib/src/containers.dart new file mode 100644 index 00000000..f8f66e4d --- /dev/null +++ b/packages/mediapipe-core/lib/src/containers.dart @@ -0,0 +1,109 @@ +import 'dart:ffi'; +import 'package:ffi/ffi.dart'; +import 'package:mediapipe_core/mediapipe_core.dart'; +import 'mediapipe_common_bindings.dart' as bindings; + +class Category { + const Category({ + required this.index, + required this.score, + required this.categoryName, + required this.displayName, + }); + final int index; + final double score; + final String? categoryName; + final String? displayName; + + static List fromStructs( + Pointer structs, + int count, + ) { + final categories = []; + for (int i = 0; i < count; i++) { + categories.add(fromStruct(structs[i])); + } + return categories; + } + + static Category fromStruct(bindings.Category struct) { + return Category( + index: struct.index, + score: struct.score, + categoryName: toDartString(struct.category_name), + displayName: toDartString(struct.display_name), + ); + } + + static void freeStructs(Pointer structs, int count) { + int index = 0; + while (index < count) { + bindings.Category obj = structs[index]; + calloc.free(obj.category_name); + calloc.free(obj.display_name); + index++; + } + calloc.free(structs); + } + + @override + String toString() => 'Category(index=$index, score=$score, ' + 'categoryName=$categoryName, displayName=$displayName)'; +} + +class Classifications { + const Classifications({ + required this.categories, + required this.headIndex, + required this.headName, + }); + final List categories; + final int headIndex; + final String? headName; + + static List fromStructs( + Pointer structs, + int count, + ) { + final classifications = []; + for (int i = 0; i < count; i++) { + classifications.add(fromStruct(structs[i])); + } + return classifications; + } + + static Classifications fromStruct(bindings.Classifications struct) { + return Classifications( + categories: Category.fromStructs( + struct.categories, + struct.categories_count, + ), + headIndex: struct.head_index, + headName: toDartString(struct.head_name), + ); + } + + static void freeStructs( + Pointer structs, + int count, + ) { + int index = 0; + while (index < count) { + bindings.Classifications obj = structs[index]; + Category.freeStructs(obj.categories, obj.categories_count); + calloc.free(obj.head_name); + index++; + } + calloc.free(structs); + } + + Category? get firstCategory => + categories.isNotEmpty ? categories.first : null; + + @override + String toString() { + final categoryStrings = categories.map((cat) => cat.toString()).join(', '); + return 'Classification(categories=[$categoryStrings], ' + 'headIndex=$headIndex, headName=$headName)'; + } +} diff --git a/packages/mediapipe-core/lib/src/ffi_utils.dart b/packages/mediapipe-core/lib/src/ffi_utils.dart new file mode 100644 index 00000000..99327d83 --- /dev/null +++ b/packages/mediapipe-core/lib/src/ffi_utils.dart @@ -0,0 +1,52 @@ +import 'dart:ffi'; +import 'dart:typed_data'; +import 'package:ffi/ffi.dart'; + +Pointer> prepareListOfStrings(List values) { + final ptrArray = calloc>(values.length); + for (var i = 0; i < values.length; i++) { + ptrArray[i] = values[i].toNativeUtf8().cast(); + } + return ptrArray; +} + +Pointer prepareString(String val) => val.toNativeUtf8().cast(); + +String? toDartString(Pointer val) { + if (val == nullptr) return null; + return val.cast().toDartString(); +} + +List toDartStrings(Pointer> val, int length) { + final dartStrings = []; + int counter = 0; + while (counter < length) { + dartStrings.add(toDartString(val[counter])); + counter++; + } + return dartStrings; +} + +Pointer prepareUint8List(Uint8List ints) { + final Pointer ptr = calloc(ints.length); + ptr.asTypedList(ints.length).setAll(0, ints); + return ptr.cast(); +} + +Uint8List toUint8List(Pointer val, {int? length}) { + final codeUnits = val.cast(); + if (length != null) { + RangeError.checkNotNegative(length, 'length'); + } else { + length = _length(codeUnits); + } + return codeUnits.asTypedList(length); +} + +int _length(Pointer codeUnits) { + var length = 0; + while (codeUnits[length] != 0) { + length++; + } + return length; +} diff --git a/packages/mediapipe-core/lib/src/mediapipe_common_bindings.dart b/packages/mediapipe-core/lib/src/mediapipe_common_bindings.dart new file mode 100644 index 00000000..839b08ac --- /dev/null +++ b/packages/mediapipe-core/lib/src/mediapipe_common_bindings.dart @@ -0,0 +1,332 @@ +// AUTO GENERATED FILE, DO NOT EDIT. +// +// Generated by `package:ffigen`. +// ignore_for_file: type=lint +import 'dart:ffi' as ffi; + +final class BaseOptions extends ffi.Struct { + external ffi.Pointer model_asset_buffer; + + external ffi.Pointer model_asset_path; +} + +final class Category extends ffi.Struct { + @ffi.Int() + external int index; + + @ffi.Float() + external double score; + + external ffi.Pointer category_name; + + external ffi.Pointer display_name; +} + +final class __mbstate_t extends ffi.Union { + @ffi.Array.multi([128]) + external ffi.Array __mbstate8; + + @ffi.LongLong() + external int _mbstateL; +} + +final class __darwin_pthread_handler_rec extends ffi.Struct { + external ffi + .Pointer)>> + __routine; + + external ffi.Pointer __arg; + + external ffi.Pointer<__darwin_pthread_handler_rec> __next; +} + +final class _opaque_pthread_attr_t extends ffi.Struct { + @ffi.Long() + external int __sig; + + @ffi.Array.multi([56]) + external ffi.Array __opaque; +} + +final class _opaque_pthread_cond_t extends ffi.Struct { + @ffi.Long() + external int __sig; + + @ffi.Array.multi([40]) + external ffi.Array __opaque; +} + +final class _opaque_pthread_condattr_t extends ffi.Struct { + @ffi.Long() + external int __sig; + + @ffi.Array.multi([8]) + external ffi.Array __opaque; +} + +final class _opaque_pthread_mutex_t extends ffi.Struct { + @ffi.Long() + external int __sig; + + @ffi.Array.multi([56]) + external ffi.Array __opaque; +} + +final class _opaque_pthread_mutexattr_t extends ffi.Struct { + @ffi.Long() + external int __sig; + + @ffi.Array.multi([8]) + external ffi.Array __opaque; +} + +final class _opaque_pthread_once_t extends ffi.Struct { + @ffi.Long() + external int __sig; + + @ffi.Array.multi([8]) + external ffi.Array __opaque; +} + +final class _opaque_pthread_rwlock_t extends ffi.Struct { + @ffi.Long() + external int __sig; + + @ffi.Array.multi([192]) + external ffi.Array __opaque; +} + +final class _opaque_pthread_rwlockattr_t extends ffi.Struct { + @ffi.Long() + external int __sig; + + @ffi.Array.multi([16]) + external ffi.Array __opaque; +} + +final class _opaque_pthread_t extends ffi.Struct { + @ffi.Long() + external int __sig; + + external ffi.Pointer<__darwin_pthread_handler_rec> __cleanup_stack; + + @ffi.Array.multi([8176]) + external ffi.Array __opaque; +} + +final class Classifications extends ffi.Struct { + external ffi.Pointer categories; + + @ffi.Uint32() + external int categories_count; + + @ffi.Int() + external int head_index; + + external ffi.Pointer head_name; +} + +final class ClassificationResult extends ffi.Struct { + external ffi.Pointer classifications; + + @ffi.Uint32() + external int classifications_count; + + @ffi.Int64() + external int timestamp_ms; + + @ffi.Bool() + external bool has_timestamp_ms; +} + +final class ClassifierOptions extends ffi.Struct { + external ffi.Pointer display_names_locale; + + @ffi.Int() + external int max_results; + + @ffi.Float() + external double score_threshold; + + external ffi.Pointer> category_allowlist; + + @ffi.Uint32() + external int category_allowlist_count; + + external ffi.Pointer> category_denylist; + + @ffi.Uint32() + external int category_denylist_count; +} + +const int true1 = 1; + +const int false1 = 0; + +const int __bool_true_false_are_defined = 1; + +const int __WORDSIZE = 64; + +const int __DARWIN_ONLY_64_BIT_INO_T = 1; + +const int __DARWIN_ONLY_UNIX_CONFORMANCE = 1; + +const int __DARWIN_ONLY_VERS_1050 = 1; + +const int __DARWIN_UNIX03 = 1; + +const int __DARWIN_64_BIT_INO_T = 1; + +const int __DARWIN_VERS_1050 = 1; + +const int __DARWIN_NON_CANCELABLE = 0; + +const String __DARWIN_SUF_EXTSN = '\$DARWIN_EXTSN'; + +const int __DARWIN_C_ANSI = 4096; + +const int __DARWIN_C_FULL = 900000; + +const int __DARWIN_C_LEVEL = 900000; + +const int __STDC_WANT_LIB_EXT1__ = 1; + +const int __DARWIN_NO_LONG_LONG = 0; + +const int _DARWIN_FEATURE_64_BIT_INODE = 1; + +const int _DARWIN_FEATURE_ONLY_64_BIT_INODE = 1; + +const int _DARWIN_FEATURE_ONLY_VERS_1050 = 1; + +const int _DARWIN_FEATURE_ONLY_UNIX_CONFORMANCE = 1; + +const int _DARWIN_FEATURE_UNIX_CONFORMANCE = 3; + +const int __has_ptrcheck = 0; + +const int __DARWIN_NULL = 0; + +const int __PTHREAD_SIZE__ = 8176; + +const int __PTHREAD_ATTR_SIZE__ = 56; + +const int __PTHREAD_MUTEXATTR_SIZE__ = 8; + +const int __PTHREAD_MUTEX_SIZE__ = 56; + +const int __PTHREAD_CONDATTR_SIZE__ = 8; + +const int __PTHREAD_COND_SIZE__ = 40; + +const int __PTHREAD_ONCE_SIZE__ = 8; + +const int __PTHREAD_RWLOCK_SIZE__ = 192; + +const int __PTHREAD_RWLOCKATTR_SIZE__ = 16; + +const int USER_ADDR_NULL = 0; + +const int INT8_MAX = 127; + +const int INT16_MAX = 32767; + +const int INT32_MAX = 2147483647; + +const int INT64_MAX = 9223372036854775807; + +const int INT8_MIN = -128; + +const int INT16_MIN = -32768; + +const int INT32_MIN = -2147483648; + +const int INT64_MIN = -9223372036854775808; + +const int UINT8_MAX = 255; + +const int UINT16_MAX = 65535; + +const int UINT32_MAX = 4294967295; + +const int UINT64_MAX = -1; + +const int INT_LEAST8_MIN = -128; + +const int INT_LEAST16_MIN = -32768; + +const int INT_LEAST32_MIN = -2147483648; + +const int INT_LEAST64_MIN = -9223372036854775808; + +const int INT_LEAST8_MAX = 127; + +const int INT_LEAST16_MAX = 32767; + +const int INT_LEAST32_MAX = 2147483647; + +const int INT_LEAST64_MAX = 9223372036854775807; + +const int UINT_LEAST8_MAX = 255; + +const int UINT_LEAST16_MAX = 65535; + +const int UINT_LEAST32_MAX = 4294967295; + +const int UINT_LEAST64_MAX = -1; + +const int INT_FAST8_MIN = -128; + +const int INT_FAST16_MIN = -32768; + +const int INT_FAST32_MIN = -2147483648; + +const int INT_FAST64_MIN = -9223372036854775808; + +const int INT_FAST8_MAX = 127; + +const int INT_FAST16_MAX = 32767; + +const int INT_FAST32_MAX = 2147483647; + +const int INT_FAST64_MAX = 9223372036854775807; + +const int UINT_FAST8_MAX = 255; + +const int UINT_FAST16_MAX = 65535; + +const int UINT_FAST32_MAX = 4294967295; + +const int UINT_FAST64_MAX = -1; + +const int INTPTR_MAX = 9223372036854775807; + +const int INTPTR_MIN = -9223372036854775808; + +const int UINTPTR_MAX = -1; + +const int INTMAX_MAX = 9223372036854775807; + +const int UINTMAX_MAX = -1; + +const int INTMAX_MIN = -9223372036854775808; + +const int PTRDIFF_MIN = -9223372036854775808; + +const int PTRDIFF_MAX = 9223372036854775807; + +const int SIZE_MAX = -1; + +const int RSIZE_MAX = 9223372036854775807; + +const int WCHAR_MAX = 2147483647; + +const int WCHAR_MIN = -2147483648; + +const int WINT_MIN = -2147483648; + +const int WINT_MAX = 2147483647; + +const int SIG_ATOMIC_MIN = -2147483648; + +const int SIG_ATOMIC_MAX = 2147483647; diff --git a/packages/mediapipe-core/lib/src/task_options.dart b/packages/mediapipe-core/lib/src/task_options.dart new file mode 100644 index 00000000..2e1320c3 --- /dev/null +++ b/packages/mediapipe-core/lib/src/task_options.dart @@ -0,0 +1,120 @@ +import 'dart:ffi'; +import 'dart:typed_data'; +import 'package:equatable/equatable.dart'; +import 'package:ffi/ffi.dart'; +import 'mediapipe_common_bindings.dart' as bindings; +import 'ffi_utils.dart'; + +class BaseOptions extends Equatable { + const BaseOptions({this.modelAssetBuffer, this.modelAssetPath}) + : assert( + !(modelAssetBuffer == null && modelAssetPath == null), + 'You must supply either `modelAssetBuffer` or `modelAssetPath`', + ), + assert( + !(modelAssetBuffer != null && modelAssetPath != null), + 'You must only supply one of `modelAssetBuffer` and `modelAssetPath`', + ); + + /// The model asset file contents as bytes; + final Uint8List? modelAssetBuffer; + + /// Path to the model asset file. + final String? modelAssetPath; + + Pointer toStruct() { + final struct = calloc(); + + if (modelAssetPath != null) { + struct.ref.model_asset_path = prepareString(modelAssetPath!); + } + if (modelAssetBuffer != null) { + struct.ref.model_asset_buffer = prepareUint8List(modelAssetBuffer!); + } + return struct; + } + + @override + List get props => [modelAssetBuffer, modelAssetPath]; +} + +class ClassifierOptions extends Equatable { + const ClassifierOptions({ + this.displayNamesLocale, + this.maxResults, + this.scoreThreshold, + this.categoryAllowlist, + this.categoryDenylist, + }); + + /// The locale to use for display names specified through the TFLite Model + /// Metadata. + final String? displayNamesLocale; + + /// The maximum number of top-scored classification results to return. + final int? maxResults; + + /// Overrides the ones provided in the model metadata. Results below this + /// value are rejected. + final double? scoreThreshold; + + /// Allowlist of category names. If non-empty, classification results whose + /// category name is not in this set will be discarded. Duplicate or unknown + /// category names are ignored. Mutually exclusive with `categoryDenylist`. + final List? categoryAllowlist; + + /// Denylist of category names. If non-empty, classification results whose + /// category name is in this set will be discarded. Duplicate or unknown + /// category names are ignored. Mutually exclusive with `categoryAllowList`. + final List? categoryDenylist; + + Pointer toStruct() { + final struct = calloc(); + _setDisplayNamesLocale(struct.ref); + _setMaxResults(struct.ref); + _setScoreThreshold(struct.ref); + _setAllowlist(struct.ref); + _setDenylist(struct.ref); + return struct; + } + + void _setDisplayNamesLocale(bindings.ClassifierOptions struct) { + if (displayNamesLocale != null) { + struct.display_names_locale = prepareString(displayNamesLocale!); + } + } + + void _setMaxResults(bindings.ClassifierOptions struct) { + // This value must not be zero, and -1 implies no limit. + struct.max_results = maxResults ?? -1; + } + + void _setScoreThreshold(bindings.ClassifierOptions struct) { + if (scoreThreshold != null) { + struct.score_threshold = scoreThreshold!; + } + } + + void _setAllowlist(bindings.ClassifierOptions struct) { + if (categoryAllowlist != null) { + struct.category_allowlist = prepareListOfStrings(categoryAllowlist!); + struct.category_allowlist_count = categoryAllowlist!.length; + } + } + + void _setDenylist(bindings.ClassifierOptions struct) { + if (categoryDenylist != null) { + struct.category_denylist = prepareListOfStrings(categoryDenylist!); + struct.category_denylist_count = categoryDenylist!.length; + } + } + + @override + List get props => [ + displayNamesLocale, + maxResults, + scoreThreshold, + ...(categoryAllowlist ?? []), + ...(categoryDenylist ?? []), + ]; +} diff --git a/packages/mediapipe-core/pubspec.yaml b/packages/mediapipe-core/pubspec.yaml new file mode 100644 index 00000000..43923497 --- /dev/null +++ b/packages/mediapipe-core/pubspec.yaml @@ -0,0 +1,17 @@ +name: mediapipe_core +description: A starting point for Dart libraries or applications. +version: 1.0.0 + +repository: https://github.com/google/flutter-mediapipe/issues +environment: + sdk: ^3.1.0-333.0.dev + +# Add regular dependencies here. +dependencies: + equatable: ^2.0.5 + ffi: ^2.1.0 + +dev_dependencies: + ffigen: ^9.0.1 + lints: ^2.0.0 + test: ^1.21.0 diff --git a/packages/mediapipe-core/test/task_options_test.dart b/packages/mediapipe-core/test/task_options_test.dart new file mode 100644 index 00000000..7721f08a --- /dev/null +++ b/packages/mediapipe-core/test/task_options_test.dart @@ -0,0 +1,116 @@ +import 'dart:ffi'; +import 'dart:typed_data'; + +import 'package:test/test.dart'; +import 'package:mediapipe_core/mediapipe_core.dart'; + +void main() { + group('BaseOptions constructor should', () { + test('enforce exactly one of modelPath and modelBuffer', () { + expect( + () => BaseOptions( + modelAssetPath: 'abc', + modelAssetBuffer: Uint8List.fromList([1, 2, 3]), + ), + throwsA(TypeMatcher()), + ); + + expect(BaseOptions.new, throwsA(TypeMatcher())); + }); + }); + + group('BaseOptions.toStruct/fromStruct should', () { + test('allocate memory in C for a modelAssetPath', () { + final options = BaseOptions(modelAssetPath: 'abc'); + final struct = options.toStruct(); + expect(toDartString(struct.ref.model_asset_path), 'abc'); + + // TODO: Is this the most precise expectation we can make here? + expect(struct.ref.model_asset_buffer, isA>()); + }); + + test('allocate memory in C for a modelAssetBuffer', () { + final options = BaseOptions( + modelAssetBuffer: Uint8List.fromList([1, 2, 3]), + ); + final struct = options.toStruct(); + expect( + toUint8List(struct.ref.model_asset_buffer), + Uint8List.fromList([1, 2, 3]), + ); + + // TODO: Is this the most precise expectation we can make here? + expect(struct.ref.model_asset_path, isA>()); + }); + + test('allocate memory in C for a modelAssetBuffer containing 0', () { + final options = BaseOptions( + modelAssetBuffer: Uint8List.fromList([1, 2, 0, 3]), + ); + final struct = options.toStruct(); + expect( + toUint8List(struct.ref.model_asset_buffer), + Uint8List.fromList([1, 2]), + ); + + // TODO: Is this the most precise expectation we can make here? + expect(struct.ref.model_asset_path, isA>()); + }); + }); + + group('ClassOptions should', () { + test('allocate memory for empty fields', () { + final options = ClassifierOptions(); + + final struct = options.toStruct(); + + expect(struct.ref.max_results, -1); + expect(struct.ref.score_threshold, 0.0); + expect(struct.ref.category_allowlist, isA>()); + expect(struct.ref.category_allowlist_count, 0); + expect(struct.ref.category_denylist, isA>()); + expect(struct.ref.category_denylist_count, 0); + // TODO: Is this the most precise expectation we can make here? + expect(struct.ref.display_names_locale, isA>()); + + // TODO: Could we do something like this? + // (right now, this segfaults) + // expect(struct.ref.display_names_locale.cast().asTypedList(1), + // Uint8List.fromList([0])); + }); + + test('allocate memory for full fields', () { + final options = ClassifierOptions( + displayNamesLocale: 'en', + maxResults: 5, + scoreThreshold: 0.9, + categoryAllowlist: ['good', 'great', 'best'], + categoryDenylist: ['bad', 'terrible', 'worst', 'honestly come on'], + ); + + final struct = options.toStruct(); + + expect(toDartString(struct.ref.display_names_locale), 'en'); + expect(struct.ref.max_results, 5); + expect(struct.ref.score_threshold, greaterThan(0.8999)); + expect(struct.ref.score_threshold, lessThan(0.90001)); + expect( + toDartStrings( + struct.ref.category_allowlist, + struct.ref.category_allowlist_count, + ), + ['good', 'great', 'best'], + ); + expect(struct.ref.category_allowlist_count, 3); + + expect( + toDartStrings( + struct.ref.category_denylist, + struct.ref.category_denylist_count, + ), + ['bad', 'terrible', 'worst', 'honestly come on'], + ); + expect(struct.ref.category_denylist_count, 4); + }); + }); +} diff --git a/packages/mediapipe-core/third_party/mediapipe/tasks/c/components/containers/category.h b/packages/mediapipe-core/third_party/mediapipe/tasks/c/components/containers/category.h new file mode 100644 index 00000000..b6eede40 --- /dev/null +++ b/packages/mediapipe-core/third_party/mediapipe/tasks/c/components/containers/category.h @@ -0,0 +1,50 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef MEDIAPIPE_TASKS_C_COMPONENTS_CONTAINERS_CATEGORY_H_ +#define MEDIAPIPE_TASKS_C_COMPONENTS_CONTAINERS_CATEGORY_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +// Defines a single classification result. +// +// The label maps packed into the TFLite Model Metadata [1] are used to populate +// the 'category_name' and 'display_name' fields. +// +// [1]: https://www.tensorflow.org/lite/convert/metadata +struct Category { + // The index of the category in the classification model output. + int index; + + // The score for this category, e.g. (but not necessarily) a probability in + // [0,1]. + float score; + + // The optional ID for the category, read from the label map packed in the + // TFLite Model Metadata if present. Not necessarily human-readable. + const char* category_name; + + // The optional human-readable name for the category, read from the label map + // packed in the TFLite Model Metadata if present. + const char* display_name; +}; + +#ifdef __cplusplus +} // extern C +#endif + +#endif // MEDIAPIPE_TASKS_C_COMPONENTS_CONTAINERS_CATEGORY_H_ diff --git a/packages/mediapipe-core/third_party/mediapipe/tasks/c/components/containers/classification_result.h b/packages/mediapipe-core/third_party/mediapipe/tasks/c/components/containers/classification_result.h new file mode 100644 index 00000000..ef2914e5 --- /dev/null +++ b/packages/mediapipe-core/third_party/mediapipe/tasks/c/components/containers/classification_result.h @@ -0,0 +1,68 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef MEDIAPIPE_TASKS_C_COMPONENTS_CONTAINERS_CLASSIFICATION_RESULT_H_ +#define MEDIAPIPE_TASKS_C_COMPONENTS_CONTAINERS_CLASSIFICATION_RESULT_H_ + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +// Defines classification results for a given classifier head. +struct Classifications { + // The array of predicted categories, usually sorted by descending scores, + // e.g. from high to low probability. + struct Category* categories; + // The number of elements in the categories array. + uint32_t categories_count; + + // The index of the classifier head (i.e. output tensor) these categories + // refer to. This is useful for multi-head models. + int head_index; + + // The optional name of the classifier head, as provided in the TFLite Model + // Metadata [1] if present. This is useful for multi-head models. + // + // [1]: https://www.tensorflow.org/lite/convert/metadata + const char* head_name; +}; + +// Defines classification results of a model. +struct ClassificationResult { + // The classification results for each head of the model. + struct Classifications* classifications; + // The number of classifications in the classifications array. + uint32_t classifications_count; + + // The optional timestamp (in milliseconds) of the start of the chunk of data + // corresponding to these results. + // + // This is only used for classification on time series (e.g. audio + // classification). In these use cases, the amount of data to process might + // exceed the maximum size that the model can process: to solve this, the + // input data is split into multiple chunks starting at different timestamps. + int64_t timestamp_ms; + // Specifies whether the timestamp contains a valid value. + bool has_timestamp_ms; +}; + +#ifdef __cplusplus +} // extern C +#endif + +#endif // MEDIAPIPE_TASKS_C_COMPONENTS_CONTAINERS_CLASSIFICATION_RESULT_H_ diff --git a/packages/mediapipe-core/third_party/mediapipe/tasks/c/components/processors/classifier_options.h b/packages/mediapipe-core/third_party/mediapipe/tasks/c/components/processors/classifier_options.h new file mode 100644 index 00000000..4658fb42 --- /dev/null +++ b/packages/mediapipe-core/third_party/mediapipe/tasks/c/components/processors/classifier_options.h @@ -0,0 +1,59 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef MEDIAPIPE_TASKS_C_COMPONENTS_PROCESSORS_CLASSIFIER_OPTIONS_H_ +#define MEDIAPIPE_TASKS_C_COMPONENTS_PROCESSORS_CLASSIFIER_OPTIONS_H_ + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +// Classifier options for MediaPipe C classification Tasks. +struct ClassifierOptions { + // The locale to use for display names specified through the TFLite Model + // Metadata, if any. Defaults to English. + char* display_names_locale; + + // The maximum number of top-scored classification results to return. If < 0, + // all available results will be returned. If 0, an invalid argument error is + // returned. + int max_results; + + // Score threshold to override the one provided in the model metadata (if + // any). Results below this value are rejected. + float score_threshold; + + // The allowlist of category names. If non-empty, detection results whose + // category name is not in this set will be filtered out. Duplicate or unknown + // category names are ignored. Mutually exclusive with category_denylist. + char** category_allowlist; + // The number of elements in the category allowlist. + uint32_t category_allowlist_count; + + // The denylist of category names. If non-empty, detection results whose + // category name is in this set will be filtered out. Duplicate or unknown + // category names are ignored. Mutually exclusive with category_allowlist. + char** category_denylist; + // The number of elements in the category denylist. + uint32_t category_denylist_count; +}; + +#ifdef __cplusplus +} // extern C +#endif + +#endif // MEDIAPIPE_TASKS_C_COMPONENTS_PROCESSORS_CLASSIFIER_OPTIONS_H_ diff --git a/packages/mediapipe-core/third_party/mediapipe/tasks/c/core/base_options.h b/packages/mediapipe-core/third_party/mediapipe/tasks/c/core/base_options.h new file mode 100644 index 00000000..d23b6884 --- /dev/null +++ b/packages/mediapipe-core/third_party/mediapipe/tasks/c/core/base_options.h @@ -0,0 +1,36 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef MEDIAPIPE_TASKS_C_CORE_BASE_OPTIONS_H_ +#define MEDIAPIPE_TASKS_C_CORE_BASE_OPTIONS_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +// Base options for MediaPipe C Tasks. +struct BaseOptions { + // The model asset file contents as a string. + char* model_asset_buffer; + + // The path to the model asset to open and mmap in memory. + char* model_asset_path; +}; + +#ifdef __cplusplus +} // extern C +#endif + +#endif // MEDIAPIPE_TASKS_C_CORE_BASE_OPTIONS_H_ From 688371a53a105516f0da6ba7d3f60a31bfc95e81 Mon Sep 17 00:00:00 2001 From: Craig Labenz Date: Fri, 13 Oct 2023 10:50:22 -0700 Subject: [PATCH 02/17] adds makefile for all packages --- Makefile | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 Makefile diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..1b2e7cab --- /dev/null +++ b/Makefile @@ -0,0 +1,40 @@ +# Runs unit tests for `mediapipe_core` +test_core: + cd packages/mediapipe-core && dart test + +# Runs the utility to pull in all header files from `google/mediapipe` +headers: + cd packages/build && dart bin/main.dart headers + +# Runs `ffigen` for `mediapipe_core` +generate_core: + cd packages/mediapipe-core && dart run ffigen --config=ffigen.yaml + +# Runs `ffigen` for `mediapipe_text` +generate_text: + cd packages/mediapipe-task-text && dart run ffigen --config=ffigen.yaml + +# Runs `ffigen` for all packages +generate: generate_core generate_text + +# Compiles the faked C artifacts for testing +compile_fake_text: + # Builds standalone executable + cd packages/mediapipe-task-text/test/c && gcc fake_text_classifier.c -o fake_text_classifier + # Builds what Flutter needs + cd packages/mediapipe-task-text/test/c && gcc -static -c -fPIC *.c -o fake_text_classifier.o + cd packages/mediapipe-task-text/test/c && gcc -shared -o fake_text_classifier.dylib fake_text_classifier.o + +# Runs all text tests +test_text_only: + cd packages/mediapipe-task-text && flutter test + +# Runs `ffigen` for `mediapipe_text` and all text tests +test_text: compile_fake_text test_text_only + +# Runs `ffigen` for all packages, compiles the faked C artifacts, and runs all tests +test: generate_text test_text generate_core test_core + +# Runs `ffigen` for all packages and all tests for all packages +test_only: test_core test_text + From 4a2e4d88c45548e3808bc0bb2a6e210609a010c2 Mon Sep 17 00:00:00 2001 From: Craig Labenz Date: Mon, 16 Oct 2023 12:53:14 -0700 Subject: [PATCH 03/17] fixes typo in Makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 1b2e7cab..7d28174a 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ test_core: # Runs the utility to pull in all header files from `google/mediapipe` headers: - cd packages/build && dart bin/main.dart headers + cd packages/build_cmd && dart bin/main.dart headers # Runs `ffigen` for `mediapipe_core` generate_core: From 8f0c39c61ccad5f0fd4e4bd3cdde258e6790d138 Mon Sep 17 00:00:00 2001 From: Kate Lovett Date: Mon, 16 Oct 2023 18:11:46 -0500 Subject: [PATCH 04/17] Apply suggestions from code review --- packages/mediapipe-core/analysis_options.yaml | 19 +---------------- .../mediapipe-core/lib/src/task_options.dart | 21 ++++++++++++------- 2 files changed, 14 insertions(+), 26 deletions(-) diff --git a/packages/mediapipe-core/analysis_options.yaml b/packages/mediapipe-core/analysis_options.yaml index 0e9249f3..0b27c8b0 100644 --- a/packages/mediapipe-core/analysis_options.yaml +++ b/packages/mediapipe-core/analysis_options.yaml @@ -1,24 +1,7 @@ -# This file configures the static analysis results for your project (errors, -# warnings, and lints). -# -# This enables the 'recommended' set of lints from `package:lints`. -# This set helps identify many issues that may lead to problems when running -# or consuming Dart code, and enforces writing Dart using a single, idiomatic -# style and format. -# -# If you want a smaller set of lints you can change this to specify -# 'package:lints/core.yaml'. These are just the most critical lints -# (the recommended set includes the core lints). -# The core lints are also what is used by pub.dev for scoring packages. +# Specify analysis options. include: package:lints/recommended.yaml -# Uncomment the following section to specify additional rules. - -# linter: -# rules: -# - camel_case_types - analyzer: exclude: - "lib/src/mediapipe_common_bindings.dart" diff --git a/packages/mediapipe-core/lib/src/task_options.dart b/packages/mediapipe-core/lib/src/task_options.dart index 2e1320c3..cdcf780e 100644 --- a/packages/mediapipe-core/lib/src/task_options.dart +++ b/packages/mediapipe-core/lib/src/task_options.dart @@ -54,18 +54,23 @@ class ClassifierOptions extends Equatable { /// The maximum number of top-scored classification results to return. final int? maxResults; - /// Overrides the ones provided in the model metadata. Results below this - /// value are rejected. + /// Overrides the ones provided in the model metadata. + /// + /// Results below this value are rejected. final double? scoreThreshold; - /// Allowlist of category names. If non-empty, classification results whose - /// category name is not in this set will be discarded. Duplicate or unknown - /// category names are ignored. Mutually exclusive with `categoryDenylist`. + /// Allowlist of category names. + /// + /// If non-empty, classification results whose category name is not in + /// this set will be discarded. Duplicate or unknown category names + /// are ignored. Mutually exclusive with `categoryDenylist`. final List? categoryAllowlist; - /// Denylist of category names. If non-empty, classification results whose - /// category name is in this set will be discarded. Duplicate or unknown - /// category names are ignored. Mutually exclusive with `categoryAllowList`. + /// Denylist of category names. + /// + /// If non-empty, classification results whose category name is in this set + /// will be discarded. Duplicate or unknown category names are ignored. + /// Mutually exclusive with `categoryAllowList`. final List? categoryDenylist; Pointer toStruct() { From 7b1d5ad7b37ca4a4da24fcd5a957a819fcc7bd3e Mon Sep 17 00:00:00 2001 From: Kate Lovett Date: Mon, 16 Oct 2023 18:26:43 -0500 Subject: [PATCH 05/17] Update Makefile --- Makefile | 37 ++++++++++++++++++++----------------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/Makefile b/Makefile index 7d28174a..84f0a670 100644 --- a/Makefile +++ b/Makefile @@ -1,22 +1,32 @@ -# Runs unit tests for `mediapipe_core` -test_core: - cd packages/mediapipe-core && dart test - # Runs the utility to pull in all header files from `google/mediapipe` headers: cd packages/build_cmd && dart bin/main.dart headers +# Runs `ffigen` for all packages +generate: generate_core generate_text + +# Runs `ffigen` for all packages, compiles the faked C artifacts, and runs all tests +test: generate_text test_text generate_core test_core + +# Runs `ffigen` for all packages and all tests for all packages +test_only: test_core test_text + +# Core --- + # Runs `ffigen` for `mediapipe_core` generate_core: cd packages/mediapipe-core && dart run ffigen --config=ffigen.yaml +# Runs unit tests for `mediapipe_core` +test_core: + cd packages/mediapipe-core && dart test + +# Text --- + # Runs `ffigen` for `mediapipe_text` generate_text: cd packages/mediapipe-task-text && dart run ffigen --config=ffigen.yaml -# Runs `ffigen` for all packages -generate: generate_core generate_text - # Compiles the faked C artifacts for testing compile_fake_text: # Builds standalone executable @@ -25,16 +35,9 @@ compile_fake_text: cd packages/mediapipe-task-text/test/c && gcc -static -c -fPIC *.c -o fake_text_classifier.o cd packages/mediapipe-task-text/test/c && gcc -shared -o fake_text_classifier.dylib fake_text_classifier.o -# Runs all text tests -test_text_only: - cd packages/mediapipe-task-text && flutter test - # Runs `ffigen` for `mediapipe_text` and all text tests test_text: compile_fake_text test_text_only -# Runs `ffigen` for all packages, compiles the faked C artifacts, and runs all tests -test: generate_text test_text generate_core test_core - -# Runs `ffigen` for all packages and all tests for all packages -test_only: test_core test_text - +# Runs all text tests +test_text_only: + cd packages/mediapipe-task-text && flutter test From ff83c89f12f5cb5f093b688ead2f995126207e21 Mon Sep 17 00:00:00 2001 From: Kate Lovett Date: Mon, 16 Oct 2023 18:48:23 -0500 Subject: [PATCH 06/17] Apply suggestions from code review --- .../tasks/c/components/containers/classification_result.h | 2 ++ .../tasks/c/components/processors/classifier_options.h | 2 ++ 2 files changed, 4 insertions(+) diff --git a/packages/mediapipe-core/third_party/mediapipe/tasks/c/components/containers/classification_result.h b/packages/mediapipe-core/third_party/mediapipe/tasks/c/components/containers/classification_result.h index ef2914e5..07ba44fb 100644 --- a/packages/mediapipe-core/third_party/mediapipe/tasks/c/components/containers/classification_result.h +++ b/packages/mediapipe-core/third_party/mediapipe/tasks/c/components/containers/classification_result.h @@ -28,6 +28,7 @@ struct Classifications { // The array of predicted categories, usually sorted by descending scores, // e.g. from high to low probability. struct Category* categories; + // The number of elements in the categories array. uint32_t categories_count; @@ -57,6 +58,7 @@ struct ClassificationResult { // exceed the maximum size that the model can process: to solve this, the // input data is split into multiple chunks starting at different timestamps. int64_t timestamp_ms; + // Specifies whether the timestamp contains a valid value. bool has_timestamp_ms; }; diff --git a/packages/mediapipe-core/third_party/mediapipe/tasks/c/components/processors/classifier_options.h b/packages/mediapipe-core/third_party/mediapipe/tasks/c/components/processors/classifier_options.h index 4658fb42..1c153bdd 100644 --- a/packages/mediapipe-core/third_party/mediapipe/tasks/c/components/processors/classifier_options.h +++ b/packages/mediapipe-core/third_party/mediapipe/tasks/c/components/processors/classifier_options.h @@ -41,12 +41,14 @@ struct ClassifierOptions { // category name is not in this set will be filtered out. Duplicate or unknown // category names are ignored. Mutually exclusive with category_denylist. char** category_allowlist; + // The number of elements in the category allowlist. uint32_t category_allowlist_count; // The denylist of category names. If non-empty, detection results whose // category name is in this set will be filtered out. Duplicate or unknown // category names are ignored. Mutually exclusive with category_allowlist. + char** category_denylist; // The number of elements in the category denylist. uint32_t category_denylist_count; From a39e75bf150a04480445b4d6a557bd9089b08f1a Mon Sep 17 00:00:00 2001 From: Craig Labenz Date: Tue, 17 Oct 2023 12:55:50 -0700 Subject: [PATCH 07/17] updated generated code's location and license (for 3P status) --- packages/mediapipe-core/ffigen.yaml | 20 +++++++++++++++++-- .../lib/generated/core_symbols.yaml | 2 +- .../generated}/mediapipe_common_bindings.dart | 19 ++++++++++++++++-- 3 files changed, 36 insertions(+), 5 deletions(-) rename packages/mediapipe-core/lib/src/{ => third_party/mediapipe/generated}/mediapipe_common_bindings.dart (91%) diff --git a/packages/mediapipe-core/ffigen.yaml b/packages/mediapipe-core/ffigen.yaml index 10c8a880..3f42a171 100644 --- a/packages/mediapipe-core/ffigen.yaml +++ b/packages/mediapipe-core/ffigen.yaml @@ -1,13 +1,29 @@ name: "MediaPipeCommonBindings" description: "Bindings for shared MediaPipe structs common across many tasks" output: - bindings: "lib/src/mediapipe_common_bindings.dart" + bindings: "lib/src/third_party/mediapipe/generated/mediapipe_common_bindings.dart" symbol-file: output: "package:mediapipe_core/generated/core_symbols.yaml" - import-path: "package:mediapipe_core/src/mediapipe_common_bindings.dart" + import-path: "package:mediapipe_core/src/third_party/mediapipe/generated/mediapipe_common_bindings.dart" headers: entry-points: - "third_party/mediapipe/tasks/c/core/base_options.h" - "third_party/mediapipe/tasks/c/components/containers/category.h" - "third_party/mediapipe/tasks/c/components/containers/classification_result.h" - "third_party/mediapipe/tasks/c/components/processors/classifier_options.h" +preamble: | + /* Copyright 2023 The MediaPipe Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + ==============================================================================*/ +ffi-native: diff --git a/packages/mediapipe-core/lib/generated/core_symbols.yaml b/packages/mediapipe-core/lib/generated/core_symbols.yaml index 497d7c36..95f3c222 100644 --- a/packages/mediapipe-core/lib/generated/core_symbols.yaml +++ b/packages/mediapipe-core/lib/generated/core_symbols.yaml @@ -1,6 +1,6 @@ format_version: 1.0.0 files: - package:mediapipe_core/src/mediapipe_common_bindings.dart: + package:mediapipe_core/src/third_party/mediapipe/generated/mediapipe_common_bindings.dart: used-config: ffi-native: false symbols: diff --git a/packages/mediapipe-core/lib/src/mediapipe_common_bindings.dart b/packages/mediapipe-core/lib/src/third_party/mediapipe/generated/mediapipe_common_bindings.dart similarity index 91% rename from packages/mediapipe-core/lib/src/mediapipe_common_bindings.dart rename to packages/mediapipe-core/lib/src/third_party/mediapipe/generated/mediapipe_common_bindings.dart index 839b08ac..3f87b50f 100644 --- a/packages/mediapipe-core/lib/src/mediapipe_common_bindings.dart +++ b/packages/mediapipe-core/lib/src/third_party/mediapipe/generated/mediapipe_common_bindings.dart @@ -1,3 +1,18 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + // AUTO GENERATED FILE, DO NOT EDIT. // // Generated by `package:ffigen`. @@ -159,12 +174,12 @@ final class ClassifierOptions extends ffi.Struct { external int category_denylist_count; } +const int __bool_true_false_are_defined = 1; + const int true1 = 1; const int false1 = 0; -const int __bool_true_false_are_defined = 1; - const int __WORDSIZE = 64; const int __DARWIN_ONLY_64_BIT_INO_T = 1; From 006e8eff1f65a31461ef23e34400ffab2029cbde Mon Sep 17 00:00:00 2001 From: Craig Labenz Date: Tue, 17 Oct 2023 15:52:37 -0700 Subject: [PATCH 08/17] code review responses including: * comments * licensing * resolving testing nits --- packages/mediapipe-core/CHANGELOG.md | 2 +- packages/mediapipe-core/analysis_options.yaml | 7 +- packages/mediapipe-core/ffigen.yaml | 5 +- .../mediapipe-core/lib/mediapipe_core.dart | 6 ++ .../mediapipe-core/lib/src/containers.dart | 55 +++++++++++++- .../mediapipe-core/lib/src/ffi_utils.dart | 30 +++++++- .../mediapipe-core/lib/src/task_options.dart | 43 +++++++++-- .../generated/mediapipe_common_bindings.dart | 76 +++++++++---------- packages/mediapipe-core/pubspec.yaml | 8 +- .../test/task_options_test.dart | 21 +++-- 10 files changed, 186 insertions(+), 67 deletions(-) diff --git a/packages/mediapipe-core/CHANGELOG.md b/packages/mediapipe-core/CHANGELOG.md index effe43c8..b78d64c6 100644 --- a/packages/mediapipe-core/CHANGELOG.md +++ b/packages/mediapipe-core/CHANGELOG.md @@ -1,3 +1,3 @@ -## 1.0.0 +## 0.0.1 - Initial version. diff --git a/packages/mediapipe-core/analysis_options.yaml b/packages/mediapipe-core/analysis_options.yaml index 0b27c8b0..e3e01d6d 100644 --- a/packages/mediapipe-core/analysis_options.yaml +++ b/packages/mediapipe-core/analysis_options.yaml @@ -1,7 +1,10 @@ # Specify analysis options. - include: package:lints/recommended.yaml +linter: + rules: + - public_member_api_docs # see https://github.com/flutter/flutter/wiki/Style-guide-for-Flutter-repo#documentation-dartdocs-javadocs-etc + analyzer: exclude: - - "lib/src/mediapipe_common_bindings.dart" + - "**/mediapipe_common_bindings.dart" diff --git a/packages/mediapipe-core/ffigen.yaml b/packages/mediapipe-core/ffigen.yaml index 3f42a171..547b1aa2 100644 --- a/packages/mediapipe-core/ffigen.yaml +++ b/packages/mediapipe-core/ffigen.yaml @@ -7,10 +7,7 @@ output: import-path: "package:mediapipe_core/src/third_party/mediapipe/generated/mediapipe_common_bindings.dart" headers: entry-points: - - "third_party/mediapipe/tasks/c/core/base_options.h" - - "third_party/mediapipe/tasks/c/components/containers/category.h" - - "third_party/mediapipe/tasks/c/components/containers/classification_result.h" - - "third_party/mediapipe/tasks/c/components/processors/classifier_options.h" + - "third_party/mediapipe/tasks/c/**" preamble: | /* Copyright 2023 The MediaPipe Authors. diff --git a/packages/mediapipe-core/lib/mediapipe_core.dart b/packages/mediapipe-core/lib/mediapipe_core.dart index 2e62b181..7bb29ef8 100644 --- a/packages/mediapipe-core/lib/mediapipe_core.dart +++ b/packages/mediapipe-core/lib/mediapipe_core.dart @@ -1,3 +1,9 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +/// Package containing core dependencies for MediaPipe's text, vision, and +/// audio-based tasks. library; export 'src/containers.dart' show Category, Classifications; diff --git a/packages/mediapipe-core/lib/src/containers.dart b/packages/mediapipe-core/lib/src/containers.dart index f8f66e4d..fbd7e0ef 100644 --- a/packages/mediapipe-core/lib/src/containers.dart +++ b/packages/mediapipe-core/lib/src/containers.dart @@ -1,20 +1,45 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + import 'dart:ffi'; import 'package:ffi/ffi.dart'; import 'package:mediapipe_core/mediapipe_core.dart'; -import 'mediapipe_common_bindings.dart' as bindings; +import 'third_party/mediapipe/generated/mediapipe_common_bindings.dart' + as bindings; +/// Dart representation of MediaPipe's "Category" concept. +/// +/// Category is a util class, that contains a category name, its display name, +/// a float value as score, and the index of the label in the corresponding +/// label file. Typically it's used as result of classification or detection +/// tasks. +/// +/// See more: +/// * [MediaPipe's Category documentation](https://developers.google.com/mediapipe/api/solutions/java/com/google/mediapipe/tasks/components/containers/Category) class Category { + /// Generative constructor that creates a [Category] instance. const Category({ required this.index, required this.score, required this.categoryName, required this.displayName, }); + + /// The index of the label in the corresponding label file. final int index; + + /// The probability score of this label category. final double score; + + /// The label of this category object. final String? categoryName; + + /// The display name of the label, which may be translated for different locales. final String? displayName; + /// Accepts a pointer to a list of structs, and a count representing the length + /// of the list, and returns a list of pure-Dart [Category] instances. static List fromStructs( Pointer structs, int count, @@ -26,6 +51,7 @@ class Category { return categories; } + /// Accepts a pointer to a single struct and returns a pure-Dart [Category] instance. static Category fromStruct(bindings.Category struct) { return Category( index: struct.index, @@ -35,6 +61,9 @@ class Category { ); } + /// Releases all C memory associated with a list of [bindings.Category] pointers. + /// This method is important to call after calling [Category.fromStructs] to + /// convert that C memory into pure-Dart objects. static void freeStructs(Pointer structs, int count) { int index = 0; while (index < count) { @@ -51,16 +80,34 @@ class Category { 'categoryName=$categoryName, displayName=$displayName)'; } +/// Dart representation of MediaPipe's "Classifications" concept. +/// +/// Represents the list of classification for a given classifier head. +/// Typically used as a result for classification tasks. +/// +/// See also: +/// * [MediaPipe's Classifications documentation](https://developers.google.com/mediapipe/api/solutions/java/com/google/mediapipe/tasks/components/containers/Classifications) class Classifications { + /// Generative constructor that creates a [Classifications] instance. const Classifications({ required this.categories, required this.headIndex, required this.headName, }); + + /// A list of Category objects which contain the actual classification + /// information, including human-readable labels and probability scores. final List categories; + + /// The index of the classifier head these entries refer to. final int headIndex; + + /// The optional name of the classifier head, which is the corresponding + /// tensor metadata name. final String? headName; + /// Accepts a pointer to a list of structs, and a count representing the length + /// of the list, and returns a list of pure-Dart [Classifications] instances. static List fromStructs( Pointer structs, int count, @@ -72,6 +119,8 @@ class Classifications { return classifications; } + /// Accepts a pointer to a single struct and returns a pure-Dart [Classifications] + /// instance. static Classifications fromStruct(bindings.Classifications struct) { return Classifications( categories: Category.fromStructs( @@ -83,6 +132,9 @@ class Classifications { ); } + /// Releases all C memory associated with a list of [bindings.Classifications] + /// pointers. This method is important to call after calling [Classifications.fromStructs] + /// to convert that C memory into pure-Dart objects. static void freeStructs( Pointer structs, int count, @@ -97,6 +149,7 @@ class Classifications { calloc.free(structs); } + /// Convenience getter for the first [Category] out of the [categories] list. Category? get firstCategory => categories.isNotEmpty ? categories.first : null; diff --git a/packages/mediapipe-core/lib/src/ffi_utils.dart b/packages/mediapipe-core/lib/src/ffi_utils.dart index 99327d83..a5411c96 100644 --- a/packages/mediapipe-core/lib/src/ffi_utils.dart +++ b/packages/mediapipe-core/lib/src/ffi_utils.dart @@ -1,22 +1,44 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + import 'dart:ffi'; import 'dart:typed_data'; import 'package:ffi/ffi.dart'; +/// Converts a list of Dart strings into their C memory equivalent. +/// +/// See also: +/// * [prepareString] Pointer> prepareListOfStrings(List values) { final ptrArray = calloc>(values.length); for (var i = 0; i < values.length; i++) { - ptrArray[i] = values[i].toNativeUtf8().cast(); + ptrArray[i] = prepareString(values[i]); } return ptrArray; } +/// Converts a single Dart string into its C memory equivalent. +/// +/// See also: +/// * [prepareListOfStrings] Pointer prepareString(String val) => val.toNativeUtf8().cast(); +/// Converts the C memory representation of a string into a Dart [String]. If the +/// supplied pointer is a null pointer, this function returns `null`. +/// +/// See also: +/// * [toDartStrings] String? toDartString(Pointer val) { - if (val == nullptr) return null; + if (val.address == 0) return null; return val.cast().toDartString(); } +/// Converts a list of C memory representations of strings into a list of Dart +/// [String]s. +/// +/// See also: +/// * [toDartString] List toDartStrings(Pointer> val, int length) { final dartStrings = []; int counter = 0; @@ -27,12 +49,16 @@ List toDartStrings(Pointer> val, int length) { return dartStrings; } +/// Converts Dart's representation for binary data, a [Uint8List], into its C +/// memory representation. Pointer prepareUint8List(Uint8List ints) { final Pointer ptr = calloc(ints.length); ptr.asTypedList(ints.length).setAll(0, ints); return ptr.cast(); } +/// Converts a pointer to binary data in C memory to Dart's representation for +/// binary data, a [Uint8List]. Uint8List toUint8List(Pointer val, {int? length}) { final codeUnits = val.cast(); if (length != null) { diff --git a/packages/mediapipe-core/lib/src/task_options.dart b/packages/mediapipe-core/lib/src/task_options.dart index cdcf780e..18dd0281 100644 --- a/packages/mediapipe-core/lib/src/task_options.dart +++ b/packages/mediapipe-core/lib/src/task_options.dart @@ -1,11 +1,26 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + import 'dart:ffi'; import 'dart:typed_data'; import 'package:equatable/equatable.dart'; import 'package:ffi/ffi.dart'; -import 'mediapipe_common_bindings.dart' as bindings; import 'ffi_utils.dart'; - +import 'third_party/mediapipe/generated/mediapipe_common_bindings.dart' + as bindings; + +/// Dart representation of MediaPipe's "BaseOptions" concept. +/// +/// Used to configure various classifiers by specifying the model they will use +/// for computation. +/// +/// See also: +/// * [MediaPipe's BaseOptions documentation](https://developers.google.com/mediapipe/api/solutions/java/com/google/mediapipe/tasks/core/BaseOptions) +/// * [ClassifierOptions], which is often used in conjunction to specify a +/// classifier's desired behavior. class BaseOptions extends Equatable { + /// Generative constructor that creates a [BaseOptions] instance. const BaseOptions({this.modelAssetBuffer, this.modelAssetPath}) : assert( !(modelAssetBuffer == null && modelAssetPath == null), @@ -22,6 +37,8 @@ class BaseOptions extends Equatable { /// Path to the model asset file. final String? modelAssetPath; + /// Converts this pure-Dart representation into C-memory suitable for the + /// MediaPipe SDK to instantiate various classifiers. Pointer toStruct() { final struct = calloc(); @@ -38,7 +55,16 @@ class BaseOptions extends Equatable { List get props => [modelAssetBuffer, modelAssetPath]; } +/// Dart representation of MediaPipe's "ClassifierOptions" concept. +/// +/// Classifier options shared across MediaPipe classification tasks. +/// +/// See also: +/// * [MediaPipe's ClassifierOptions documentation](https://developers.google.com/mediapipe/api/solutions/java/com/google/mediapipe/tasks/components/processors/ClassifierOptions) +/// * [BaseOptions], which is often used in conjunction to specify a +/// classifier's desired behavior. class ClassifierOptions extends Equatable { + /// Generative constructor that creates a [ClassifierOptions] instance. const ClassifierOptions({ this.displayNamesLocale, this.maxResults, @@ -54,9 +80,8 @@ class ClassifierOptions extends Equatable { /// The maximum number of top-scored classification results to return. final int? maxResults; - /// Overrides the ones provided in the model metadata. - /// - /// Results below this value are rejected. + /// If set, establishes a minimum `score` and leads to the rejection of any + /// categories with lower `score` values. final double? scoreThreshold; /// Allowlist of category names. @@ -64,6 +89,9 @@ class ClassifierOptions extends Equatable { /// If non-empty, classification results whose category name is not in /// this set will be discarded. Duplicate or unknown category names /// are ignored. Mutually exclusive with `categoryDenylist`. + /// + /// See also: + /// * [Category.categoryName] final List? categoryAllowlist; /// Denylist of category names. @@ -71,8 +99,13 @@ class ClassifierOptions extends Equatable { /// If non-empty, classification results whose category name is in this set /// will be discarded. Duplicate or unknown category names are ignored. /// Mutually exclusive with `categoryAllowList`. + /// + /// See also: + /// * [Category.categoryName] final List? categoryDenylist; + /// Converts this pure-Dart representation into C-memory suitable for the + /// MediaPipe SDK to instantiate various classifiers. Pointer toStruct() { final struct = calloc(); _setDisplayNamesLocale(struct.ref); diff --git a/packages/mediapipe-core/lib/src/third_party/mediapipe/generated/mediapipe_common_bindings.dart b/packages/mediapipe-core/lib/src/third_party/mediapipe/generated/mediapipe_common_bindings.dart index 3f87b50f..13bf3d24 100644 --- a/packages/mediapipe-core/lib/src/third_party/mediapipe/generated/mediapipe_common_bindings.dart +++ b/packages/mediapipe-core/lib/src/third_party/mediapipe/generated/mediapipe_common_bindings.dart @@ -25,18 +25,6 @@ final class BaseOptions extends ffi.Struct { external ffi.Pointer model_asset_path; } -final class Category extends ffi.Struct { - @ffi.Int() - external int index; - - @ffi.Float() - external double score; - - external ffi.Pointer category_name; - - external ffi.Pointer display_name; -} - final class __mbstate_t extends ffi.Union { @ffi.Array.multi([128]) external ffi.Array __mbstate8; @@ -129,6 +117,38 @@ final class _opaque_pthread_t extends ffi.Struct { external ffi.Array __opaque; } +final class ClassifierOptions extends ffi.Struct { + external ffi.Pointer display_names_locale; + + @ffi.Int() + external int max_results; + + @ffi.Float() + external double score_threshold; + + external ffi.Pointer> category_allowlist; + + @ffi.Uint32() + external int category_allowlist_count; + + external ffi.Pointer> category_denylist; + + @ffi.Uint32() + external int category_denylist_count; +} + +final class Category extends ffi.Struct { + @ffi.Int() + external int index; + + @ffi.Float() + external double score; + + external ffi.Pointer category_name; + + external ffi.Pointer display_name; +} + final class Classifications extends ffi.Struct { external ffi.Pointer categories; @@ -154,32 +174,6 @@ final class ClassificationResult extends ffi.Struct { external bool has_timestamp_ms; } -final class ClassifierOptions extends ffi.Struct { - external ffi.Pointer display_names_locale; - - @ffi.Int() - external int max_results; - - @ffi.Float() - external double score_threshold; - - external ffi.Pointer> category_allowlist; - - @ffi.Uint32() - external int category_allowlist_count; - - external ffi.Pointer> category_denylist; - - @ffi.Uint32() - external int category_denylist_count; -} - -const int __bool_true_false_are_defined = 1; - -const int true1 = 1; - -const int false1 = 0; - const int __WORDSIZE = 64; const int __DARWIN_ONLY_64_BIT_INO_T = 1; @@ -345,3 +339,9 @@ const int WINT_MAX = 2147483647; const int SIG_ATOMIC_MIN = -2147483648; const int SIG_ATOMIC_MAX = 2147483647; + +const int __bool_true_false_are_defined = 1; + +const int true1 = 1; + +const int false1 = 0; diff --git a/packages/mediapipe-core/pubspec.yaml b/packages/mediapipe-core/pubspec.yaml index 43923497..fbbf44fb 100644 --- a/packages/mediapipe-core/pubspec.yaml +++ b/packages/mediapipe-core/pubspec.yaml @@ -1,8 +1,10 @@ name: mediapipe_core -description: A starting point for Dart libraries or applications. -version: 1.0.0 +description: Shared logic and utilities required by other MediaPipe Task packages. +version: 0.0.1-wip + +repository: https://github.com/google/flutter-mediapipe +issue_tracker: https://github.com/google/flutter-mediapipe/issues -repository: https://github.com/google/flutter-mediapipe/issues environment: sdk: ^3.1.0-333.0.dev diff --git a/packages/mediapipe-core/test/task_options_test.dart b/packages/mediapipe-core/test/task_options_test.dart index 7721f08a..61ad88f7 100644 --- a/packages/mediapipe-core/test/task_options_test.dart +++ b/packages/mediapipe-core/test/task_options_test.dart @@ -1,3 +1,7 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + import 'dart:ffi'; import 'dart:typed_data'; @@ -24,9 +28,7 @@ void main() { final options = BaseOptions(modelAssetPath: 'abc'); final struct = options.toStruct(); expect(toDartString(struct.ref.model_asset_path), 'abc'); - - // TODO: Is this the most precise expectation we can make here? - expect(struct.ref.model_asset_buffer, isA>()); + expectNullPtr(struct.ref.model_asset_buffer); }); test('allocate memory in C for a modelAssetBuffer', () { @@ -38,9 +40,7 @@ void main() { toUint8List(struct.ref.model_asset_buffer), Uint8List.fromList([1, 2, 3]), ); - - // TODO: Is this the most precise expectation we can make here? - expect(struct.ref.model_asset_path, isA>()); + expectNullPtr(struct.ref.model_asset_path); }); test('allocate memory in C for a modelAssetBuffer containing 0', () { @@ -52,9 +52,7 @@ void main() { toUint8List(struct.ref.model_asset_buffer), Uint8List.fromList([1, 2]), ); - - // TODO: Is this the most precise expectation we can make here? - expect(struct.ref.model_asset_path, isA>()); + expectNullPtr(struct.ref.model_asset_path); }); }); @@ -70,8 +68,7 @@ void main() { expect(struct.ref.category_allowlist_count, 0); expect(struct.ref.category_denylist, isA>()); expect(struct.ref.category_denylist_count, 0); - // TODO: Is this the most precise expectation we can make here? - expect(struct.ref.display_names_locale, isA>()); + expectNullPtr(struct.ref.display_names_locale); // TODO: Could we do something like this? // (right now, this segfaults) @@ -114,3 +111,5 @@ void main() { }); }); } + +void expectNullPtr(Pointer ptr) => expect(ptr.address, equals(0)); From 37ea84d8d91dc2ebfd2534728bfcd8b5aa8b6f35 Mon Sep 17 00:00:00 2001 From: Craig Labenz Date: Wed, 18 Oct 2023 08:26:37 -0700 Subject: [PATCH 09/17] updated README --- packages/mediapipe-core/README.md | 44 +++++++++++-------------------- 1 file changed, 16 insertions(+), 28 deletions(-) diff --git a/packages/mediapipe-core/README.md b/packages/mediapipe-core/README.md index 8b55e735..124460d0 100644 --- a/packages/mediapipe-core/README.md +++ b/packages/mediapipe-core/README.md @@ -1,39 +1,27 @@ - - -TODO: Put a short description of the package here that helps potential users -know whether this package might be useful for them. - -## Features +A Flutter plugin to use the MediaPipe Core API, which enables multiple Mediapipe tasks. -TODO: List what your package can do. Maybe include images, gifs, or videos. +To learn more about MediaPipe, please visit the [MediaPipe website](https://developers.google.com/mediapipe) -## Getting started +## Getting Started -TODO: List prerequisites and provide or point to information on how to -start using the package. +To get started with MediaPipe, please [see the documentation](https://developers.google.com/mediapipe/solutions/guide). + + +## Issues and feedback -```dart -const like = 'sample'; -``` +Please file Flutter-MediaPipe specific issues, bugs, or feature requests in our [issue tracker](https://github.com/google/flutter-mediapipe/issues/new). -## Additional information +Issues that are specific to Flutter can be filed in the [Flutter issue tracker](https://github.com/flutter/flutter/issues/new). -TODO: Tell users more about the package: where to find more information, how to -contribute to the package, how to file issues, what response they can expect -from the package authors, and more. +To contribute a change to this plugin, +please review our [contribution guide](https://github.com/google/flutter-mediapipe/blob/master/CONTRIBUTING.md) +and open a [pull request](https://github.com/google/flutter-mediapipe/pulls). \ No newline at end of file From 61dbdacb149323e4e64520343248e18cdf3247ff Mon Sep 17 00:00:00 2001 From: Craig Labenz Date: Wed, 18 Oct 2023 15:32:16 -0700 Subject: [PATCH 10/17] setup sharing of base analysis_options --- packages/analysis_options.yaml | 6 ++++++ packages/mediapipe-core/analysis_options.yaml | 7 +------ 2 files changed, 7 insertions(+), 6 deletions(-) create mode 100644 packages/analysis_options.yaml diff --git a/packages/analysis_options.yaml b/packages/analysis_options.yaml new file mode 100644 index 00000000..c4d43fe3 --- /dev/null +++ b/packages/analysis_options.yaml @@ -0,0 +1,6 @@ +include: package:lints/recommended.yaml + +linter: + rules: + - public_member_api_docs # see https://github.com/flutter/flutter/wiki/Style-guide-for-Flutter-repo#documentation-dartdocs-javadocs-etc + diff --git a/packages/mediapipe-core/analysis_options.yaml b/packages/mediapipe-core/analysis_options.yaml index e3e01d6d..f2aa915b 100644 --- a/packages/mediapipe-core/analysis_options.yaml +++ b/packages/mediapipe-core/analysis_options.yaml @@ -1,9 +1,4 @@ -# Specify analysis options. -include: package:lints/recommended.yaml - -linter: - rules: - - public_member_api_docs # see https://github.com/flutter/flutter/wiki/Style-guide-for-Flutter-repo#documentation-dartdocs-javadocs-etc +include: ../analysis_options.yaml analyzer: exclude: From b93054b9f4534a3473103b6061a336a12eeb1df7 Mon Sep 17 00:00:00 2001 From: Craig Labenz Date: Mon, 23 Oct 2023 14:59:10 -0700 Subject: [PATCH 11/17] Update packages/mediapipe-core/lib/src/containers.dart Co-authored-by: Kate Lovett --- packages/mediapipe-core/lib/src/containers.dart | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/mediapipe-core/lib/src/containers.dart b/packages/mediapipe-core/lib/src/containers.dart index fbd7e0ef..56142479 100644 --- a/packages/mediapipe-core/lib/src/containers.dart +++ b/packages/mediapipe-core/lib/src/containers.dart @@ -10,8 +10,8 @@ import 'third_party/mediapipe/generated/mediapipe_common_bindings.dart' /// Dart representation of MediaPipe's "Category" concept. /// -/// Category is a util class, that contains a category name, its display name, -/// a float value as score, and the index of the label in the corresponding +/// Category is a util class, that contains a [categoryName], its [displayName], +/// a float value as [score], and the [index] of the label in the corresponding /// label file. Typically it's used as result of classification or detection /// tasks. /// From 10a85891d5844b331cf7fe9185052ab19c8dad96 Mon Sep 17 00:00:00 2001 From: Craig Labenz Date: Mon, 23 Oct 2023 15:00:05 -0700 Subject: [PATCH 12/17] Update packages/mediapipe-core/lib/src/containers.dart Co-authored-by: Kate Lovett --- packages/mediapipe-core/lib/src/containers.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/mediapipe-core/lib/src/containers.dart b/packages/mediapipe-core/lib/src/containers.dart index 56142479..5111040b 100644 --- a/packages/mediapipe-core/lib/src/containers.dart +++ b/packages/mediapipe-core/lib/src/containers.dart @@ -95,7 +95,7 @@ class Classifications { required this.headName, }); - /// A list of Category objects which contain the actual classification + /// A list of [Category] objects which contain the actual classification /// information, including human-readable labels and probability scores. final List categories; From e02eece40e83571f4ad9cf08898bfe93b6f3782a Mon Sep 17 00:00:00 2001 From: Craig Labenz Date: Tue, 31 Oct 2023 09:09:02 -0700 Subject: [PATCH 13/17] add free methods to core structs --- .../mediapipe-core/lib/src/task_options.dart | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/packages/mediapipe-core/lib/src/task_options.dart b/packages/mediapipe-core/lib/src/task_options.dart index 18dd0281..0aab3ee5 100644 --- a/packages/mediapipe-core/lib/src/task_options.dart +++ b/packages/mediapipe-core/lib/src/task_options.dart @@ -53,6 +53,16 @@ class BaseOptions extends Equatable { @override List get props => [modelAssetBuffer, modelAssetPath]; + + /// Releases all C memory held by this [bindings.BaseOptions] struct. + static void freeStruct(bindings.BaseOptions struct) { + if (struct.model_asset_buffer.address != 0) { + calloc.free(struct.model_asset_buffer); + } + if (struct.model_asset_path.address != 0) { + calloc.free(struct.model_asset_path); + } + } } /// Dart representation of MediaPipe's "ClassifierOptions" concept. @@ -147,6 +157,19 @@ class ClassifierOptions extends Equatable { } } + /// Releases all C memory held by this [bindings.ClassifierOptions] struct. + static void freeStruct(bindings.ClassifierOptions struct) { + if (struct.display_names_locale.address != 0) { + calloc.free(struct.display_names_locale); + } + if (struct.category_allowlist.address != 0) { + calloc.free(struct.category_allowlist); + } + if (struct.category_denylist.address != 0) { + calloc.free(struct.category_denylist); + } + } + @override List get props => [ displayNamesLocale, From 7e11eef45e73bb2a15d3a0e9d42ad672b3d818d1 Mon Sep 17 00:00:00 2001 From: Craig Labenz Date: Wed, 8 Nov 2023 10:56:42 -0800 Subject: [PATCH 14/17] code review updates to options --- packages/mediapipe-core/lib/src/task_options.dart | 6 ------ packages/mediapipe-core/test/task_options_test.dart | 9 ++------- 2 files changed, 2 insertions(+), 13 deletions(-) diff --git a/packages/mediapipe-core/lib/src/task_options.dart b/packages/mediapipe-core/lib/src/task_options.dart index 0aab3ee5..c12ff615 100644 --- a/packages/mediapipe-core/lib/src/task_options.dart +++ b/packages/mediapipe-core/lib/src/task_options.dart @@ -99,9 +99,6 @@ class ClassifierOptions extends Equatable { /// If non-empty, classification results whose category name is not in /// this set will be discarded. Duplicate or unknown category names /// are ignored. Mutually exclusive with `categoryDenylist`. - /// - /// See also: - /// * [Category.categoryName] final List? categoryAllowlist; /// Denylist of category names. @@ -109,9 +106,6 @@ class ClassifierOptions extends Equatable { /// If non-empty, classification results whose category name is in this set /// will be discarded. Duplicate or unknown category names are ignored. /// Mutually exclusive with `categoryAllowList`. - /// - /// See also: - /// * [Category.categoryName] final List? categoryDenylist; /// Converts this pure-Dart representation into C-memory suitable for the diff --git a/packages/mediapipe-core/test/task_options_test.dart b/packages/mediapipe-core/test/task_options_test.dart index 61ad88f7..143a9ff4 100644 --- a/packages/mediapipe-core/test/task_options_test.dart +++ b/packages/mediapipe-core/test/task_options_test.dart @@ -64,16 +64,11 @@ void main() { expect(struct.ref.max_results, -1); expect(struct.ref.score_threshold, 0.0); - expect(struct.ref.category_allowlist, isA>()); + expectNullPtr(struct.ref.category_allowlist); expect(struct.ref.category_allowlist_count, 0); - expect(struct.ref.category_denylist, isA>()); + expectNullPtr(struct.ref.category_denylist); expect(struct.ref.category_denylist_count, 0); expectNullPtr(struct.ref.display_names_locale); - - // TODO: Could we do something like this? - // (right now, this segfaults) - // expect(struct.ref.display_names_locale.cast().asTypedList(1), - // Uint8List.fromList([0])); }); test('allocate memory for full fields', () { From 0aae904523620181f323964a1fea37cd3658b456 Mon Sep 17 00:00:00 2001 From: Craig Labenz Date: Wed, 8 Nov 2023 11:04:01 -0800 Subject: [PATCH 15/17] adds publish blocker --- packages/mediapipe-core/pubspec.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/mediapipe-core/pubspec.yaml b/packages/mediapipe-core/pubspec.yaml index fbbf44fb..501b7892 100644 --- a/packages/mediapipe-core/pubspec.yaml +++ b/packages/mediapipe-core/pubspec.yaml @@ -1,6 +1,7 @@ name: mediapipe_core description: Shared logic and utilities required by other MediaPipe Task packages. version: 0.0.1-wip +publish_to: none repository: https://github.com/google/flutter-mediapipe issue_tracker: https://github.com/google/flutter-mediapipe/issues From 48dbf2e4a2a5ef713a133e1ce3379e76d2247738 Mon Sep 17 00:00:00 2001 From: Craig Labenz Date: Wed, 8 Nov 2023 13:02:47 -0800 Subject: [PATCH 16/17] Add GitHub actions for CI/CD (#14) * adds CI/CD for mediapipe_core * add newlines * moves file into workflows dir * uncomment flutter doctor * testing PR config to run this now * added master and beta CI scripts * add executable permissions to CI scripts --- .github/workflows/main.yaml | 36 ++++++++++++++++++++++++++++++ tool/ci_script_shared.sh | 28 +++++++++++++++++++++++ tool/mediapipe_ci_script_beta.sh | 16 +++++++++++++ tool/mediapipe_ci_script_master.sh | 16 +++++++++++++ tool/mediapipe_ci_script_stable.sh | 16 +++++++++++++ 5 files changed, 112 insertions(+) create mode 100644 .github/workflows/main.yaml create mode 100644 tool/ci_script_shared.sh create mode 100755 tool/mediapipe_ci_script_beta.sh create mode 100755 tool/mediapipe_ci_script_master.sh create mode 100755 tool/mediapipe_ci_script_stable.sh diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml new file mode 100644 index 00000000..52908f82 --- /dev/null +++ b/.github/workflows/main.yaml @@ -0,0 +1,36 @@ +name: Main Branch CI + +# Declare default permissions as read only. +permissions: read-all + +on: + push: + branches: [main] + pull_request: + branches: [main, ffi-wrapper-core-pkg] + workflow_dispatch: + schedule: + - cron: "0 0 * * *" # Every day at midnight + +defaults: + run: + shell: bash + +jobs: + flutter-tests: + name: Test mediapipe_core against ${{ matrix.flutter_version }} + runs-on: ${{ matrix.os }} + # Skip running job on forks + if: github.repository == 'google/flutter-mediapipe' + strategy: + fail-fast: false + matrix: + flutter_version: [stable, beta, master] + # TODO(craiglabenz): Add `ubuntu-latest` and `windows-latest` when those artifacts exist + os: [macos-latest] + steps: + - uses: actions/checkout@v4 + - uses: subosito/flutter-action@v2 + with: + channel: ${{ matrix.flutter_version }} + - run: ./tool/mediapipe_ci_script_${{ matrix.flutter_version }}.sh diff --git a/tool/ci_script_shared.sh b/tool/ci_script_shared.sh new file mode 100644 index 00000000..183f71a5 --- /dev/null +++ b/tool/ci_script_shared.sh @@ -0,0 +1,28 @@ +function ci_package () { + local channel="$1" + + shift + local arr=("$@") + for PACKAGE_NAME in "${arr[@]}" + do + echo "== Testing '${PACKAGE_NAME}' on Flutter's $channel channel ==" + pushd "packages/${PACKAGE_NAME}" + + # Grab packages. + flutter pub get + + # Run the analyzer to find any static analysis issues. + dart analyze --fatal-infos + + # Run the formatter on all the dart files to make sure everything's linted. + dart format --output none --set-exit-if-changed . + + # Run the actual tests. + if [ -d "test" ] + then + flutter test + fi + + popd + done +} diff --git a/tool/mediapipe_ci_script_beta.sh b/tool/mediapipe_ci_script_beta.sh new file mode 100755 index 00000000..5f914f48 --- /dev/null +++ b/tool/mediapipe_ci_script_beta.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +set -e + +DIR="${BASH_SOURCE%/*}" +source "$DIR/ci_script_shared.sh" + +flutter doctor -v + +declare -ar PACKAGE_NAMES=( + "mediapipe-core" +) + +ci_package "beta" "${PACKAGE_NAMES[@]}" + +echo "-- Success --" diff --git a/tool/mediapipe_ci_script_master.sh b/tool/mediapipe_ci_script_master.sh new file mode 100755 index 00000000..48c5d65b --- /dev/null +++ b/tool/mediapipe_ci_script_master.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +set -e + +DIR="${BASH_SOURCE%/*}" +source "$DIR/ci_script_shared.sh" + +flutter doctor -v + +declare -ar PACKAGE_NAMES=( + "mediapipe-core" +) + +ci_package "master" "${PACKAGE_NAMES[@]}" + +echo "-- Success --" diff --git a/tool/mediapipe_ci_script_stable.sh b/tool/mediapipe_ci_script_stable.sh new file mode 100755 index 00000000..153dda2a --- /dev/null +++ b/tool/mediapipe_ci_script_stable.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +set -e + +DIR="${BASH_SOURCE%/*}" +source "$DIR/ci_script_shared.sh" + +flutter doctor -v + +declare -ar PACKAGE_NAMES=( + "mediapipe-core" +) + +ci_package "stable" "${PACKAGE_NAMES[@]}" + +echo "-- Success --" From bc212e4f93b9f8a0db46565b9885eba616d6f70f Mon Sep 17 00:00:00 2001 From: Craig Labenz Date: Wed, 8 Nov 2023 13:04:12 -0800 Subject: [PATCH 17/17] adds ffiwrapper to ci/cd --- .github/workflows/main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 52908f82..45325b8e 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -7,7 +7,7 @@ on: push: branches: [main] pull_request: - branches: [main, ffi-wrapper-core-pkg] + branches: [main, ffi-wrapper, ffi-wrapper-core-pkg] workflow_dispatch: schedule: - cron: "0 0 * * *" # Every day at midnight