From 34ea7b6090fa40196b865a867f85a09f2a35863d Mon Sep 17 00:00:00 2001 From: Cynthia J Date: Thu, 3 Apr 2025 11:45:33 -0700 Subject: [PATCH 1/9] Add a new response api --- .../firebase_vertexai/lib/src/live_api.dart | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/packages/firebase_vertexai/firebase_vertexai/lib/src/live_api.dart b/packages/firebase_vertexai/firebase_vertexai/lib/src/live_api.dart index 2727c2232045..5b5ad17b5eac 100644 --- a/packages/firebase_vertexai/firebase_vertexai/lib/src/live_api.dart +++ b/packages/firebase_vertexai/firebase_vertexai/lib/src/live_api.dart @@ -176,6 +176,49 @@ class LiveServerToolCallCancellation implements LiveServerMessage { final List? functionIds; } +/// The status of a live generation response. +enum LiveResponseStatus { + /// The live generation is proceeding normally, and more content might be streamed. + normal, + + /// A client message has interrupted current model generation. If the client + /// is playing out the content in realtime, this is a good signal to stop and + /// empty the current queue. + interrupted, + + /// The model is done generating. Generation will only start in response to + /// additional client messages. + turnComplete, +} + +/// A single response chunk received during a live content generation. +/// +/// It can contain generated content, function calls to be executed, or +/// instructions to cancel previous function calls, along with the status of the +/// ongoing generation. +class LiveContentResponse { + // ignore: public_member_api_docs + LiveContentResponse( + {this.modelTurn, + this.responseStatus = LiveResponseStatus.normal, + this.functionCalls, + this.functionIdsToCancel}); + + /// The content generated by the live model. + final Content? modelTurn; + + /// The status of the live generation at the point this response was received. + /// Indicates whether the generation is ongoing, interrupted, or complete for + /// the current turn. Defaults to [LiveResponseStatus.normal]. + final LiveResponseStatus responseStatus; + + /// The list of function calls to be executed. + final List? functionCalls; + + /// The list of [FunctionCall.id] to cancel. + final List? functionIdsToCancel; +} + /// Represents realtime input from the client in a live stream. class LiveClientRealtimeInput { /// Creates a [LiveClientRealtimeInput] instance. From d2cc7c343e782937cd7d7c58f7ab394d898c5245 Mon Sep 17 00:00:00 2001 From: Cynthia J Date: Fri, 4 Apr 2025 15:15:33 -0700 Subject: [PATCH 2/9] make vertex android app gradle working --- .../example/android/app/build.gradle | 25 ++++++++++--------- .../android/app/src/main/AndroidManifest.xml | 5 +++- .../example/android/build.gradle | 14 ----------- .../gradle/wrapper/gradle-wrapper.properties | 2 +- .../example/android/settings.gradle | 12 +++++---- .../firebase_vertexai/example/lib/main.dart | 6 ++--- .../xcshareddata/xcschemes/Runner.xcscheme | 1 + 7 files changed, 28 insertions(+), 37 deletions(-) diff --git a/packages/firebase_vertexai/firebase_vertexai/example/android/app/build.gradle b/packages/firebase_vertexai/firebase_vertexai/example/android/app/build.gradle index fc205b4e21cd..85dbaef6ee63 100644 --- a/packages/firebase_vertexai/firebase_vertexai/example/android/app/build.gradle +++ b/packages/firebase_vertexai/firebase_vertexai/example/android/app/build.gradle @@ -1,3 +1,12 @@ +plugins { + id "com.android.application" + // START: FlutterFire Configuration + id 'com.google.gms.google-services' + // END: FlutterFire Configuration + id "kotlin-android" + id "dev.flutter.flutter-gradle-plugin" +} + def localProperties = new Properties() def localPropertiesFile = rootProject.file('local.properties') if (localPropertiesFile.exists()) { @@ -6,11 +15,6 @@ if (localPropertiesFile.exists()) { } } -def flutterRoot = localProperties.getProperty('flutter.sdk') -if (flutterRoot == null) { - throw new GradleException("Flutter SDK not found. Define location with flutter.sdk in the local.properties file.") -} - def flutterVersionCode = localProperties.getProperty('flutter.versionCode') if (flutterVersionCode == null) { flutterVersionCode = '1' @@ -21,12 +25,6 @@ if (flutterVersionName == null) { flutterVersionName = '1.0' } -apply plugin: 'com.android.application' -// START: FlutterFire Configuration -apply plugin: 'com.google.gms.google-services' -// END: FlutterFire Configuration -apply from: "$flutterRoot/packages/flutter_tools/gradle/flutter.gradle" - android { namespace "com.example.example" @@ -34,7 +32,7 @@ android { defaultConfig { applicationId "com.example.example" - minSdk 21 + minSdk 23 targetSdk 33 versionCode flutterVersionCode.toInteger() versionName flutterVersionName @@ -51,6 +49,9 @@ android { signingConfig signingConfigs.debug } } + kotlinOptions { + jvmTarget = '1.8' // Or '11' + } } flutter { diff --git a/packages/firebase_vertexai/firebase_vertexai/example/android/app/src/main/AndroidManifest.xml b/packages/firebase_vertexai/firebase_vertexai/example/android/app/src/main/AndroidManifest.xml index 8b7413a6a397..3401fcfb42b9 100644 --- a/packages/firebase_vertexai/firebase_vertexai/example/android/app/src/main/AndroidManifest.xml +++ b/packages/firebase_vertexai/firebase_vertexai/example/android/app/src/main/AndroidManifest.xml @@ -2,7 +2,8 @@ + android:icon="@mipmap/ic_launcher" + android:usesCleartextTraffic="true"> + + diff --git a/packages/firebase_vertexai/firebase_vertexai/example/android/build.gradle b/packages/firebase_vertexai/firebase_vertexai/example/android/build.gradle index 97c6de922a3d..bc157bd1a12b 100644 --- a/packages/firebase_vertexai/firebase_vertexai/example/android/build.gradle +++ b/packages/firebase_vertexai/firebase_vertexai/example/android/build.gradle @@ -1,17 +1,3 @@ -buildscript { - repositories { - google() - mavenCentral() - } - - dependencies { - classpath 'com.android.tools.build:gradle:8.1.2' - // START: FlutterFire Configuration - classpath 'com.google.gms:google-services:4.4.0' - // END: FlutterFire Configuration - } -} - allprojects { repositories { google() diff --git a/packages/firebase_vertexai/firebase_vertexai/example/android/gradle/wrapper/gradle-wrapper.properties b/packages/firebase_vertexai/firebase_vertexai/example/android/gradle/wrapper/gradle-wrapper.properties index e1ca574ef017..aa49780cd59e 100644 --- a/packages/firebase_vertexai/firebase_vertexai/example/android/gradle/wrapper/gradle-wrapper.properties +++ b/packages/firebase_vertexai/firebase_vertexai/example/android/gradle/wrapper/gradle-wrapper.properties @@ -2,4 +2,4 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.6.3-all.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.5-all.zip diff --git a/packages/firebase_vertexai/firebase_vertexai/example/android/settings.gradle b/packages/firebase_vertexai/firebase_vertexai/example/android/settings.gradle index 1d6d19b7f8ec..960e486b36e3 100644 --- a/packages/firebase_vertexai/firebase_vertexai/example/android/settings.gradle +++ b/packages/firebase_vertexai/firebase_vertexai/example/android/settings.gradle @@ -5,10 +5,9 @@ pluginManagement { def flutterSdkPath = properties.getProperty("flutter.sdk") assert flutterSdkPath != null, "flutter.sdk not set in local.properties" return flutterSdkPath - } - settings.ext.flutterSdkPath = flutterSdkPath() + }() - includeBuild("${settings.ext.flutterSdkPath}/packages/flutter_tools/gradle") + includeBuild("$flutterSdkPath/packages/flutter_tools/gradle") repositories { google() @@ -19,8 +18,11 @@ pluginManagement { plugins { id "dev.flutter.flutter-plugin-loader" version "1.0.0" - id "com.android.application" version "7.3.0" apply false - id "org.jetbrains.kotlin.android" version "1.7.10" apply false + id "com.android.application" version "8.2.1" apply false + // START: FlutterFire Configuration + id "com.google.gms.google-services" version "4.3.15" apply false + // END: FlutterFire Configuration + id "org.jetbrains.kotlin.android" version "1.8.10" apply false } include ":app" diff --git a/packages/firebase_vertexai/firebase_vertexai/example/lib/main.dart b/packages/firebase_vertexai/firebase_vertexai/example/lib/main.dart index e4e2a8c56e6d..8d7ffa5b7f5b 100644 --- a/packages/firebase_vertexai/firebase_vertexai/example/lib/main.dart +++ b/packages/firebase_vertexai/firebase_vertexai/example/lib/main.dart @@ -16,6 +16,7 @@ import 'package:firebase_core/firebase_core.dart'; import 'package:firebase_auth/firebase_auth.dart'; import 'package:firebase_vertexai/firebase_vertexai.dart'; import 'package:flutter/material.dart'; +import 'package:vertex_ai_example/firebase_options.dart'; import 'pages/chat_page.dart'; import 'pages/audio_page.dart'; @@ -28,12 +29,9 @@ import 'pages/document.dart'; import 'pages/video_page.dart'; import 'pages/bidi_page.dart'; -// REQUIRED if you want to run on Web -const FirebaseOptions? options = null; - void main() async { WidgetsFlutterBinding.ensureInitialized(); - await Firebase.initializeApp(); + await Firebase.initializeApp(options: DefaultFirebaseOptions.currentPlatform); await FirebaseAuth.instance.signInAnonymously(); var vertexInstance = diff --git a/packages/firebase_vertexai/firebase_vertexai/example/macos/Runner.xcodeproj/xcshareddata/xcschemes/Runner.xcscheme b/packages/firebase_vertexai/firebase_vertexai/example/macos/Runner.xcodeproj/xcshareddata/xcschemes/Runner.xcscheme index b2775746f883..b0a82f087ad1 100644 --- a/packages/firebase_vertexai/firebase_vertexai/example/macos/Runner.xcodeproj/xcshareddata/xcschemes/Runner.xcscheme +++ b/packages/firebase_vertexai/firebase_vertexai/example/macos/Runner.xcodeproj/xcshareddata/xcschemes/Runner.xcscheme @@ -59,6 +59,7 @@ ignoresPersistentStateOnLaunch = "NO" debugDocumentVersioning = "YES" debugServiceExtension = "internal" + enableGPUValidationMode = "1" allowLocationSimulation = "YES"> From e76a5b22ea84d1349e21e9b5415dcc741d5774ca Mon Sep 17 00:00:00 2001 From: Cynthia J Date: Mon, 7 Apr 2025 14:52:27 -0700 Subject: [PATCH 3/9] Some more update post release --- .../example/lib/utils/audio_recorder.dart | 4 ++++ .../firebase_vertexai/lib/src/live_api.dart | 11 ++++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/packages/firebase_vertexai/firebase_vertexai/example/lib/utils/audio_recorder.dart b/packages/firebase_vertexai/firebase_vertexai/example/lib/utils/audio_recorder.dart index 0d3ca4c2fb06..1f3710cd0c8f 100644 --- a/packages/firebase_vertexai/firebase_vertexai/example/lib/utils/audio_recorder.dart +++ b/packages/firebase_vertexai/firebase_vertexai/example/lib/utils/audio_recorder.dart @@ -137,6 +137,10 @@ class InMemoryAudioRecorder { encoder: _encoder, sampleRate: 16000, numChannels: 1, + androidConfig: const AndroidRecordConfig( + muteAudio: true, + audioSource: AndroidAudioSource.mic, + ), ); final devs = await _recorder.listInputDevices(); debugPrint(devs.toString()); diff --git a/packages/firebase_vertexai/firebase_vertexai/lib/src/live_api.dart b/packages/firebase_vertexai/firebase_vertexai/lib/src/live_api.dart index 5b5ad17b5eac..585372dcde7a 100644 --- a/packages/firebase_vertexai/firebase_vertexai/lib/src/live_api.dart +++ b/packages/firebase_vertexai/firebase_vertexai/lib/src/live_api.dart @@ -30,7 +30,16 @@ enum Voice { kore('Kore'), // ignore: public_member_api_docs - puck('Puck'); + leda('Leda'), + + // ignore: public_member_api_docs + orus('Orus'), + + // ignore: public_member_api_docs + puck('Puck'), + + // ignore: public_member_api_docs + zephyr('Zephyr'); const Voice(this._jsonString); final String _jsonString; From 0cfc505461aaeab71b18042dd49a2d82d2b31c21 Mon Sep 17 00:00:00 2001 From: Cynthia J Date: Mon, 14 Apr 2025 20:29:04 -0700 Subject: [PATCH 4/9] remove unspecified modality --- .../firebase_vertexai/firebase_vertexai/lib/src/live_api.dart | 3 --- 1 file changed, 3 deletions(-) diff --git a/packages/firebase_vertexai/firebase_vertexai/lib/src/live_api.dart b/packages/firebase_vertexai/firebase_vertexai/lib/src/live_api.dart index 585372dcde7a..0e2061e9d7f1 100644 --- a/packages/firebase_vertexai/firebase_vertexai/lib/src/live_api.dart +++ b/packages/firebase_vertexai/firebase_vertexai/lib/src/live_api.dart @@ -68,9 +68,6 @@ class SpeechConfig { /// The available response modalities. enum ResponseModalities { - /// Unspecified response modality. - unspecified('MODALITY_UNSPECIFIED'), - /// Text response modality. text('TEXT'), From cf022341f37620cd41b2fb2128114a2c78f08efd Mon Sep 17 00:00:00 2001 From: Cynthia J Date: Tue, 22 Apr 2025 10:19:00 -0700 Subject: [PATCH 5/9] breaking api update --- .../example/lib/pages/bidi_page.dart | 34 +++-- .../lib/firebase_vertexai.dart | 4 +- .../firebase_vertexai/lib/src/live_api.dart | 119 ++++++++---------- .../lib/src/live_session.dart | 8 +- .../firebase_vertexai/test/live_test.dart | 40 +++--- 5 files changed, 86 insertions(+), 119 deletions(-) diff --git a/packages/firebase_vertexai/firebase_vertexai/example/lib/pages/bidi_page.dart b/packages/firebase_vertexai/firebase_vertexai/example/lib/pages/bidi_page.dart index 0192c02367d6..97ffc610ab05 100644 --- a/packages/firebase_vertexai/firebase_vertexai/example/lib/pages/bidi_page.dart +++ b/packages/firebase_vertexai/firebase_vertexai/example/lib/pages/bidi_page.dart @@ -59,7 +59,7 @@ class _BidiPageState extends State { super.initState(); final config = LiveGenerationConfig( - speechConfig: SpeechConfig(voice: Voice.fenrir), + speechConfig: SpeechConfig(voiceName: 'Fenrir'), responseModalities: [ ResponseModalities.audio, ], @@ -328,25 +328,21 @@ class _BidiPageState extends State { } } - Future _handleLiveServerMessage(LiveServerMessage response) async { - if (response is LiveServerContent && response.modelTurn != null) { - await _handleLiveServerContent(response); - } - - if (response is LiveServerContent && - response.turnComplete != null && - response.turnComplete!) { - await _handleTurnComplete(); - } - - if (response is LiveServerContent && - response.interrupted != null && - response.interrupted!) { - log('Interrupted: $response'); - } + Future _handleLiveServerMessage(LiveServerResponse response) async { + final message = response.message; - if (response is LiveServerToolCall && response.functionCalls != null) { - await _handleLiveServerToolCall(response); + if (message is LiveServerContent) { + if (message.modelTurn != null) { + await _handleLiveServerContent(message); + } + if (message.turnComplete!) { + await _handleTurnComplete(); + } + if (message.interrupted!) { + log('Interrupted: $response'); + } + } else if (message is LiveServerToolCall && message.functionCalls != null) { + await _handleLiveServerToolCall(message); } } diff --git a/packages/firebase_vertexai/firebase_vertexai/lib/firebase_vertexai.dart b/packages/firebase_vertexai/firebase_vertexai/lib/firebase_vertexai.dart index e58f3ef736f9..694470ceef23 100644 --- a/packages/firebase_vertexai/firebase_vertexai/lib/firebase_vertexai.dart +++ b/packages/firebase_vertexai/firebase_vertexai/lib/firebase_vertexai.dart @@ -72,11 +72,11 @@ export 'src/live_api.dart' show LiveGenerationConfig, SpeechConfig, - Voice, ResponseModalities, LiveServerMessage, LiveServerContent, LiveServerToolCall, - LiveServerToolCallCancellation; + LiveServerToolCallCancellation, + LiveServerResponse; export 'src/live_session.dart' show LiveSession; export 'src/schema.dart' show Schema, SchemaType; diff --git a/packages/firebase_vertexai/firebase_vertexai/lib/src/live_api.dart b/packages/firebase_vertexai/firebase_vertexai/lib/src/live_api.dart index 0e2061e9d7f1..50de63dcaea4 100644 --- a/packages/firebase_vertexai/firebase_vertexai/lib/src/live_api.dart +++ b/packages/firebase_vertexai/firebase_vertexai/lib/src/live_api.dart @@ -15,54 +15,58 @@ import 'api.dart'; import 'content.dart'; import 'error.dart'; -/// The available voice options for speech synthesis. -enum Voice { - // ignore: public_member_api_docs - aoede('Aoede'), - - // ignore: public_member_api_docs - charon('Charon'), - - // ignore: public_member_api_docs - fenrir('Fenrir'), - - // ignore: public_member_api_docs - kore('Kore'), - - // ignore: public_member_api_docs - leda('Leda'), - +/// Configuration for a prebuilt voice. +/// +/// This class allows specifying a voice by its name. +class PrebuiltVoiceConfig { // ignore: public_member_api_docs - orus('Orus'), + const PrebuiltVoiceConfig({this.voiceName}); + /// The voice name to use for speech synthesis. + /// + /// See https://cloud.google.com/text-to-speech/docs/chirp3-hd for names and + /// sound demos. + final String? voiceName; // ignore: public_member_api_docs - puck('Puck'), + Map toJson() => + {if (voiceName case final voiceName?) 'voice_name': voiceName}; +} +/// Configuration for the voice to be used in speech synthesis. +/// +/// This class currently supports using a prebuilt voice configuration. +class VoiceConfig { // ignore: public_member_api_docs - zephyr('Zephyr'); - - const Voice(this._jsonString); - final String _jsonString; + VoiceConfig({this.prebuiltVoiceConfig}); + final PrebuiltVoiceConfig? prebuiltVoiceConfig; // ignore: public_member_api_docs - String toJson() => _jsonString; + Map toJson() => { + if (prebuiltVoiceConfig case final prebuiltVoiceConfig?) + 'prebuilt_voice_config': prebuiltVoiceConfig.toJson() + }; } /// Configures speech synthesis settings. +/// +/// Allows specifying the desired voice for speech synthesis. class SpeechConfig { /// Creates a [SpeechConfig] instance. /// - /// [voice] (optional): The desired voice for speech synthesis. - SpeechConfig({this.voice}); - - /// The voice to use for speech synthesis. - final Voice? voice; + /// [voiceName] See https://cloud.google.com/text-to-speech/docs/chirp3-hd + /// for names and sound demos. + SpeechConfig({String? voiceName}) + : voiceConfig = voiceName != null + ? VoiceConfig( + prebuiltVoiceConfig: PrebuiltVoiceConfig(voiceName: voiceName)) + : null; + + /// The voice config to use for speech synthesis. + final VoiceConfig? voiceConfig; // ignore: public_member_api_docs Map toJson() => { - if (voice case final voice?) - 'voice_config': { - 'prebuilt_voice_config': {'voice_name': voice.toJson()} - } + if (voiceConfig case final voiceConfig?) + 'voice_config': voiceConfig.toJson() }; } @@ -182,47 +186,17 @@ class LiveServerToolCallCancellation implements LiveServerMessage { final List? functionIds; } -/// The status of a live generation response. -enum LiveResponseStatus { - /// The live generation is proceeding normally, and more content might be streamed. - normal, - - /// A client message has interrupted current model generation. If the client - /// is playing out the content in realtime, this is a good signal to stop and - /// empty the current queue. - interrupted, - - /// The model is done generating. Generation will only start in response to - /// additional client messages. - turnComplete, -} - /// A single response chunk received during a live content generation. /// /// It can contain generated content, function calls to be executed, or /// instructions to cancel previous function calls, along with the status of the /// ongoing generation. -class LiveContentResponse { +class LiveServerResponse { // ignore: public_member_api_docs - LiveContentResponse( - {this.modelTurn, - this.responseStatus = LiveResponseStatus.normal, - this.functionCalls, - this.functionIdsToCancel}); + LiveServerResponse({required this.message}); - /// The content generated by the live model. - final Content? modelTurn; - - /// The status of the live generation at the point this response was received. - /// Indicates whether the generation is ongoing, interrupted, or complete for - /// the current turn. Defaults to [LiveResponseStatus.normal]. - final LiveResponseStatus responseStatus; - - /// The list of function calls to be executed. - final List? functionCalls; - - /// The list of [FunctionCall.id] to cancel. - final List? functionIdsToCancel; + /// The server message generated by the live model. + final LiveServerMessage message; } /// Represents realtime input from the client in a live stream. @@ -286,7 +260,7 @@ class LiveClientToolResponse { }; } -/// Parses a JSON object received from the live server into a [LiveServerMessage]. +/// Parses a JSON object received from the live server into a [LiveServerResponse]. /// /// This function handles different types of server messages, including: /// - Error messages, which result in a [VertexAIException] being thrown. @@ -324,8 +298,13 @@ class LiveClientToolResponse { /// - [jsonObject]: The JSON object received from the live server. /// /// Returns: -/// - A [LiveServerMessage] object representing the parsed message. -LiveServerMessage parseServerMessage(Object jsonObject) { +/// - A [LiveServerResponse] object representing the parsed message. +LiveServerResponse parseServerResponse(Object jsonObject) { + LiveServerMessage message = _parseServerMessage(jsonObject); + return LiveServerResponse(message: message); +} + +LiveServerMessage _parseServerMessage(Object jsonObject) { if (jsonObject case {'error': final Object error}) { throw parseError(error); } diff --git a/packages/firebase_vertexai/firebase_vertexai/lib/src/live_session.dart b/packages/firebase_vertexai/firebase_vertexai/lib/src/live_session.dart index 6addf4ab8a19..a7b5a3108214 100644 --- a/packages/firebase_vertexai/firebase_vertexai/lib/src/live_session.dart +++ b/packages/firebase_vertexai/firebase_vertexai/lib/src/live_session.dart @@ -33,7 +33,7 @@ class LiveSession { var jsonString = utf8.decode(message); var response = json.decode(jsonString); - _messageController.add(parseServerMessage(response)); + _messageController.add(parseServerResponse(response)); } catch (e) { _messageController.addError(e); } @@ -45,7 +45,7 @@ class LiveSession { ); } final WebSocketChannel _ws; - final _messageController = StreamController.broadcast(); + final _messageController = StreamController.broadcast(); late StreamSubscription _wsSubscription; /// Sends content to the server. @@ -107,10 +107,10 @@ class LiveSession { /// Receives messages from the server. /// - /// Returns a [Stream] of [LiveServerMessage] objects representing the + /// Returns a [Stream] of [LiveServerResponse] objects representing the /// messages received from the server. The stream will stops once the server /// sends turn complete message. - Stream receive() async* { + Stream receive() async* { _checkWsStatus(); await for (final result in _messageController.stream) { diff --git a/packages/firebase_vertexai/firebase_vertexai/test/live_test.dart b/packages/firebase_vertexai/firebase_vertexai/test/live_test.dart index b49853d55ce7..460cbfa69fb6 100644 --- a/packages/firebase_vertexai/firebase_vertexai/test/live_test.dart +++ b/packages/firebase_vertexai/firebase_vertexai/test/live_test.dart @@ -20,16 +20,8 @@ import 'package:flutter_test/flutter_test.dart'; void main() { group('LiveAPI Tests', () { - test('Voices enum toJson() returns correct value', () { - expect(Voice.aoede.toJson(), 'Aoede'); - expect(Voice.charon.toJson(), 'Charon'); - expect(Voice.fenrir.toJson(), 'Fenrir'); - expect(Voice.kore.toJson(), 'Kore'); - expect(Voice.puck.toJson(), 'Puck'); - }); - test('SpeechConfig toJson() returns correct JSON', () { - final speechConfigWithVoice = SpeechConfig(voice: Voice.aoede); + final speechConfigWithVoice = SpeechConfig(voiceName: 'Aoede'); expect(speechConfigWithVoice.toJson(), { 'voice_config': { 'prebuilt_voice_config': {'voice_name': 'Aoede'} @@ -41,7 +33,6 @@ void main() { }); test('ResponseModalities enum toJson() returns correct value', () { - expect(ResponseModalities.unspecified.toJson(), 'MODALITY_UNSPECIFIED'); expect(ResponseModalities.text.toJson(), 'TEXT'); expect(ResponseModalities.image.toJson(), 'IMAGE'); expect(ResponseModalities.audio.toJson(), 'AUDIO'); @@ -49,7 +40,7 @@ void main() { test('LiveGenerationConfig toJson() returns correct JSON', () { final liveGenerationConfig = LiveGenerationConfig( - speechConfig: SpeechConfig(voice: Voice.charon), + speechConfig: SpeechConfig(voiceName: 'Charon'), responseModalities: [ResponseModalities.text, ResponseModalities.audio], candidateCount: 2, maxOutputTokens: 100, @@ -184,9 +175,9 @@ void main() { 'turnComplete': true, } }; - final message = parseServerMessage(jsonObject); - expect(message, isA()); - final contentMessage = message as LiveServerContent; + final response = parseServerResponse(jsonObject); + expect(response.message, isA()); + final contentMessage = response.message as LiveServerContent; expect(contentMessage.turnComplete, true); expect(contentMessage.modelTurn, isA()); }); @@ -206,9 +197,9 @@ void main() { ] } }; - final message = parseServerMessage(jsonObject); - expect(message, isA()); - final toolCallMessage = message as LiveServerToolCall; + final response = parseServerResponse(jsonObject); + expect(response.message, isA()); + final toolCallMessage = response.message as LiveServerToolCall; expect(toolCallMessage.functionCalls, isA>()); }); @@ -219,28 +210,29 @@ void main() { 'ids': ['1', '2'] } }; - final message = parseServerMessage(jsonObject); - expect(message, isA()); - final cancellationMessage = message as LiveServerToolCallCancellation; + final response = parseServerResponse(jsonObject); + expect(response.message, isA()); + final cancellationMessage = + response.message as LiveServerToolCallCancellation; expect(cancellationMessage.functionIds, ['1', '2']); }); test('parseServerMessage parses setupComplete message correctly', () { final jsonObject = {'setupComplete': {}}; - final message = parseServerMessage(jsonObject); - expect(message, isA()); + final response = parseServerResponse(jsonObject); + expect(response.message, isA()); }); test('parseServerMessage throws VertexAIException for error message', () { final jsonObject = {'error': {}}; - expect(() => parseServerMessage(jsonObject), + expect(() => parseServerResponse(jsonObject), throwsA(isA())); }); test('parseServerMessage throws VertexAISdkException for unhandled format', () { final jsonObject = {'unknown': {}}; - expect(() => parseServerMessage(jsonObject), + expect(() => parseServerResponse(jsonObject), throwsA(isA())); }); }); From 5f1e987fe055a6e06dfd21c37cf86b82539c3258 Mon Sep 17 00:00:00 2001 From: Cynthia J Date: Tue, 22 Apr 2025 10:21:53 -0700 Subject: [PATCH 6/9] add todo for server content --- .../firebase_vertexai/firebase_vertexai/lib/src/live_api.dart | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/firebase_vertexai/firebase_vertexai/lib/src/live_api.dart b/packages/firebase_vertexai/firebase_vertexai/lib/src/live_api.dart index 50de63dcaea4..4aa3d596e764 100644 --- a/packages/firebase_vertexai/firebase_vertexai/lib/src/live_api.dart +++ b/packages/firebase_vertexai/firebase_vertexai/lib/src/live_api.dart @@ -142,6 +142,7 @@ class LiveServerContent implements LiveServerMessage { /// [interrupted] (optional): Indicates if the generation was interrupted. LiveServerContent({this.modelTurn, this.turnComplete, this.interrupted}); + // TODO(cynthia): Add accessor for media content /// The content generated by the model. final Content? modelTurn; From 8226e6aca9832fa69181af6fbb6fcd1044a22159 Mon Sep 17 00:00:00 2001 From: Cynthia J Date: Tue, 22 Apr 2025 10:27:23 -0700 Subject: [PATCH 7/9] fix an error of turncomplete and interrupted null --- .../firebase_vertexai/example/lib/pages/bidi_page.dart | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/firebase_vertexai/firebase_vertexai/example/lib/pages/bidi_page.dart b/packages/firebase_vertexai/firebase_vertexai/example/lib/pages/bidi_page.dart index 97ffc610ab05..31b693c96208 100644 --- a/packages/firebase_vertexai/firebase_vertexai/example/lib/pages/bidi_page.dart +++ b/packages/firebase_vertexai/firebase_vertexai/example/lib/pages/bidi_page.dart @@ -335,10 +335,10 @@ class _BidiPageState extends State { if (message.modelTurn != null) { await _handleLiveServerContent(message); } - if (message.turnComplete!) { + if (message.turnComplete != null && message.turnComplete!) { await _handleTurnComplete(); } - if (message.interrupted!) { + if (message.interrupted != null && message.interrupted!) { log('Interrupted: $response'); } } else if (message is LiveServerToolCall && message.functionCalls != null) { From cafb5af6bc05b7d421d00ad02a69c2692fa99f9a Mon Sep 17 00:00:00 2001 From: Cynthia J Date: Tue, 22 Apr 2025 10:42:07 -0700 Subject: [PATCH 8/9] update to pass analyzer --- .../firebase_vertexai/example/lib/main.dart | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/packages/firebase_vertexai/firebase_vertexai/example/lib/main.dart b/packages/firebase_vertexai/firebase_vertexai/example/lib/main.dart index 8d7ffa5b7f5b..e6d880e15427 100644 --- a/packages/firebase_vertexai/firebase_vertexai/example/lib/main.dart +++ b/packages/firebase_vertexai/firebase_vertexai/example/lib/main.dart @@ -16,7 +16,9 @@ import 'package:firebase_core/firebase_core.dart'; import 'package:firebase_auth/firebase_auth.dart'; import 'package:firebase_vertexai/firebase_vertexai.dart'; import 'package:flutter/material.dart'; -import 'package:vertex_ai_example/firebase_options.dart'; + +// Import after file is generated through flutterfire_cli. +// import 'package:vertex_ai_example/firebase_options.dart'; import 'pages/chat_page.dart'; import 'pages/audio_page.dart'; @@ -31,7 +33,10 @@ import 'pages/bidi_page.dart'; void main() async { WidgetsFlutterBinding.ensureInitialized(); - await Firebase.initializeApp(options: DefaultFirebaseOptions.currentPlatform); + // Enable this line instead once have the firebase_options.dart generated and + // imported through flutterfire_cli. + // await Firebase.initializeApp(options: DefaultFirebaseOptions.currentPlatform); + await Firebase.initializeApp(); await FirebaseAuth.instance.signInAnonymously(); var vertexInstance = From 14b2e6d07cdba160af3dfbaf402c75209994e452 Mon Sep 17 00:00:00 2001 From: Cynthia J Date: Tue, 22 Apr 2025 11:04:01 -0700 Subject: [PATCH 9/9] ignore the documentation --- .../firebase_vertexai/firebase_vertexai/lib/src/live_api.dart | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/firebase_vertexai/firebase_vertexai/lib/src/live_api.dart b/packages/firebase_vertexai/firebase_vertexai/lib/src/live_api.dart index 4aa3d596e764..0454f7fe91b9 100644 --- a/packages/firebase_vertexai/firebase_vertexai/lib/src/live_api.dart +++ b/packages/firebase_vertexai/firebase_vertexai/lib/src/live_api.dart @@ -39,6 +39,7 @@ class VoiceConfig { // ignore: public_member_api_docs VoiceConfig({this.prebuiltVoiceConfig}); + // ignore: public_member_api_docs final PrebuiltVoiceConfig? prebuiltVoiceConfig; // ignore: public_member_api_docs Map toJson() => {