From af1ad3ad70e95844261524388d870d78be318b8b Mon Sep 17 00:00:00 2001 From: Cynthia J Date: Thu, 1 May 2025 23:19:42 -0700 Subject: [PATCH 1/2] Add responseModality --- .../firebase_vertexai/example/lib/main.dart | 2 +- .../example/lib/pages/chat_page.dart | 67 +++++++++++++++++++ .../lib/firebase_vertexai.dart | 2 +- .../firebase_vertexai/lib/src/api.dart | 38 +++++++++++ .../firebase_vertexai/lib/src/live_api.dart | 27 +------- .../firebase_vertexai/test/live_test.dart | 1 + 6 files changed, 111 insertions(+), 26 deletions(-) diff --git a/packages/firebase_vertexai/firebase_vertexai/example/lib/main.dart b/packages/firebase_vertexai/firebase_vertexai/example/lib/main.dart index e6d880e15427..7928b1aa1d87 100644 --- a/packages/firebase_vertexai/firebase_vertexai/example/lib/main.dart +++ b/packages/firebase_vertexai/firebase_vertexai/example/lib/main.dart @@ -41,7 +41,7 @@ void main() async { var vertexInstance = FirebaseVertexAI.instanceFor(auth: FirebaseAuth.instance); - final model = vertexInstance.generativeModel(model: 'gemini-1.5-flash'); + final model = vertexInstance.generativeModel(model: 'gemini-2.0-flash'); runApp(GenerativeAISample(model: model)); } diff --git a/packages/firebase_vertexai/firebase_vertexai/example/lib/pages/chat_page.dart b/packages/firebase_vertexai/firebase_vertexai/example/lib/pages/chat_page.dart index 33b6e1141b72..af85c8003988 100644 --- a/packages/firebase_vertexai/firebase_vertexai/example/lib/pages/chat_page.dart +++ b/packages/firebase_vertexai/firebase_vertexai/example/lib/pages/chat_page.dart @@ -95,6 +95,17 @@ class _ChatPageState extends State { const SizedBox.square( dimension: 15, ), + if (!_loading) + IconButton( + onPressed: () async { + await _multiModalityResponse(_textController.text); + }, + icon: Icon( + Icons.model_training, + color: Theme.of(context).colorScheme.primary, + ), + tooltip: 'MultiModality response', + ), if (!_loading) IconButton( onPressed: () async { @@ -152,6 +163,62 @@ class _ChatPageState extends State { } } + Future _multiModalityResponse(String message) async { + setState(() { + _loading = true; + }); + + try { + _messages.add(MessageData(text: message, fromUser: true)); + var response = await widget.model.generateContent( + [Content.text(message)], + generationConfig: GenerationConfig( + responseModalities: [ + ResponseModalities.text, + ResponseModalities.audio, + ], + ), + ); + var inlineDatas = response.inlineDatas.toList(); + + if (inlineDatas.isEmpty) { + _showError('No response from API.'); + return; + } else { + for (final inlineData in inlineDatas) { + if (inlineData.mimeType.contains('image')) { + _messages.add( + MessageData( + text: response.text, + image: Image.memory(inlineData.bytes), + fromUser: false, + ), + ); + } else { + if (inlineData.mimeType.contains('audio')) { + print('Got audio response'); + } + } + } + setState(() { + _loading = false; + _scrollDown(); + }); + } + } catch (e) { + _showError(e.toString()); + setState(() { + _loading = false; + }); + } finally { + _textController.clear(); + setState(() { + _loading = false; + }); + _textFieldFocus.requestFocus(); + } + } + void _showError(String message) { showDialog( context: context, diff --git a/packages/firebase_vertexai/firebase_vertexai/lib/firebase_vertexai.dart b/packages/firebase_vertexai/firebase_vertexai/lib/firebase_vertexai.dart index 694470ceef23..de96a38e8302 100644 --- a/packages/firebase_vertexai/firebase_vertexai/lib/firebase_vertexai.dart +++ b/packages/firebase_vertexai/firebase_vertexai/lib/firebase_vertexai.dart @@ -27,6 +27,7 @@ export 'src/api.dart' HarmProbability, HarmBlockMethod, PromptFeedback, + ResponseModalities, SafetyRating, SafetySetting, // TODO(cynthiajiang) remove in next breaking change. @@ -72,7 +73,6 @@ export 'src/live_api.dart' show LiveGenerationConfig, SpeechConfig, - ResponseModalities, LiveServerMessage, LiveServerContent, LiveServerToolCall, diff --git a/packages/firebase_vertexai/firebase_vertexai/lib/src/api.dart b/packages/firebase_vertexai/firebase_vertexai/lib/src/api.dart index 6697c972dc05..668e2ab6342f 100644 --- a/packages/firebase_vertexai/firebase_vertexai/lib/src/api.dart +++ b/packages/firebase_vertexai/firebase_vertexai/lib/src/api.dart @@ -108,6 +108,15 @@ final class GenerateContentResponse { Iterable get functionCalls => candidates.firstOrNull?.content.parts.whereType() ?? const []; + + /// The inline data parts of the first candidate in [candidates], if any. + /// + /// Returns an empty list if there are no candidates, or if the first + /// candidate has no [InlineDataPart] parts. There is no error thrown if the + /// prompt or response were blocked. + Iterable get inlineDatas => + candidates.firstOrNull?.content.parts.whereType() ?? + const []; } /// Feedback metadata of a prompt specified in a [GenerativeModel] request. @@ -650,6 +659,24 @@ enum HarmBlockMethod { Object toJson() => _jsonString; } +/// The available response modalities. +enum ResponseModalities { + /// Text response modality. + text('TEXT'), + + /// Image response modality. + image('IMAGE'), + + /// Audio response modality. + audio('AUDIO'); + + const ResponseModalities(this._jsonString); + final String _jsonString; + + /// Convert to json format + String toJson() => _jsonString; +} + /// Configuration options for model generation and outputs. abstract class BaseGenerationConfig { // ignore: public_member_api_docs @@ -661,6 +688,7 @@ abstract class BaseGenerationConfig { this.topK, this.presencePenalty, this.frequencyPenalty, + this.responseModalities, }); /// Number of generated responses to return. @@ -737,6 +765,9 @@ abstract class BaseGenerationConfig { /// for more details. final double? frequencyPenalty; + /// The list of desired response modalities. + final List? responseModalities; + // ignore: public_member_api_docs Map toJson() => { if (candidateCount case final candidateCount?) @@ -750,6 +781,9 @@ abstract class BaseGenerationConfig { 'presencePenalty': presencePenalty, if (frequencyPenalty case final frequencyPenalty?) 'frequencyPenalty': frequencyPenalty, + if (responseModalities case final responseModalities?) + 'responseModalities': + responseModalities.map((modality) => modality.toJson()).toList(), }; } @@ -765,6 +799,7 @@ final class GenerationConfig extends BaseGenerationConfig { super.topK, super.presencePenalty, super.frequencyPenalty, + super.responseModalities, this.responseMimeType, this.responseSchema, }); @@ -990,6 +1025,9 @@ SafetyRating _parseSafetyRating(Object? jsonObject) { if (jsonObject is! Map) { throw unhandledFormat('SafetyRating', jsonObject); } + if (jsonObject.isEmpty) { + return SafetyRating(HarmCategory.unknown, HarmProbability.unknown); + } return SafetyRating(HarmCategory._parseValue(jsonObject['category']), HarmProbability._parseValue(jsonObject['probability']), probabilityScore: jsonObject['probabilityScore'] as double?, diff --git a/packages/firebase_vertexai/firebase_vertexai/lib/src/live_api.dart b/packages/firebase_vertexai/firebase_vertexai/lib/src/live_api.dart index 0454f7fe91b9..87d0af7b2431 100644 --- a/packages/firebase_vertexai/firebase_vertexai/lib/src/live_api.dart +++ b/packages/firebase_vertexai/firebase_vertexai/lib/src/live_api.dart @@ -71,30 +71,12 @@ class SpeechConfig { }; } -/// The available response modalities. -enum ResponseModalities { - /// Text response modality. - text('TEXT'), - - /// Image response modality. - image('IMAGE'), - - /// Audio response modality. - audio('AUDIO'); - - const ResponseModalities(this._jsonString); - final String _jsonString; - - /// Convert to json format - String toJson() => _jsonString; -} - /// Configures live generation settings. final class LiveGenerationConfig extends BaseGenerationConfig { // ignore: public_member_api_docs LiveGenerationConfig({ this.speechConfig, - this.responseModalities, + super.responseModalities, super.candidateCount, super.maxOutputTokens, super.temperature, @@ -107,17 +89,14 @@ final class LiveGenerationConfig extends BaseGenerationConfig { /// The speech configuration. final SpeechConfig? speechConfig; - /// The list of desired response modalities. - final List? responseModalities; + // /// The list of desired response modalities. + // final List? responseModalities; @override Map toJson() => { ...super.toJson(), if (speechConfig case final speechConfig?) 'speechConfig': speechConfig.toJson(), - if (responseModalities case final responseModalities?) - 'responseModalities': - responseModalities.map((modality) => modality.toJson()).toList(), }; } diff --git a/packages/firebase_vertexai/firebase_vertexai/test/live_test.dart b/packages/firebase_vertexai/firebase_vertexai/test/live_test.dart index 460cbfa69fb6..b8b73d17e0ca 100644 --- a/packages/firebase_vertexai/firebase_vertexai/test/live_test.dart +++ b/packages/firebase_vertexai/firebase_vertexai/test/live_test.dart @@ -13,6 +13,7 @@ // limitations under the License. import 'dart:typed_data'; +import 'package:firebase_vertexai/src/api.dart'; import 'package:firebase_vertexai/src/content.dart'; import 'package:firebase_vertexai/src/error.dart'; import 'package:firebase_vertexai/src/live_api.dart'; From d8f8770bb8d0c75cd1a3499e3c738372db8fb73d Mon Sep 17 00:00:00 2001 From: Cynthia J Date: Sun, 4 May 2025 22:06:32 -0700 Subject: [PATCH 2/2] review comments --- .../example/lib/pages/chat_page.dart | 16 ++++++---------- .../firebase_vertexai/lib/src/api.dart | 2 +- .../firebase_vertexai/lib/src/live_api.dart | 3 --- 3 files changed, 7 insertions(+), 14 deletions(-) diff --git a/packages/firebase_vertexai/firebase_vertexai/example/lib/pages/chat_page.dart b/packages/firebase_vertexai/firebase_vertexai/example/lib/pages/chat_page.dart index af85c8003988..489fccafa577 100644 --- a/packages/firebase_vertexai/firebase_vertexai/example/lib/pages/chat_page.dart +++ b/packages/firebase_vertexai/firebase_vertexai/example/lib/pages/chat_page.dart @@ -98,13 +98,13 @@ class _ChatPageState extends State { if (!_loading) IconButton( onPressed: () async { - await _multiModalityResponse(_textController.text); + await _imageResponse(_textController.text); }, icon: Icon( - Icons.model_training, + Icons.image, color: Theme.of(context).colorScheme.primary, ), - tooltip: 'MultiModality response', + tooltip: 'Image response', ), if (!_loading) IconButton( @@ -163,7 +163,7 @@ class _ChatPageState extends State { } } - Future _multiModalityResponse(String message) async { + Future _imageResponse(String message) async { setState(() { _loading = true; }); @@ -175,11 +175,11 @@ class _ChatPageState extends State { generationConfig: GenerationConfig( responseModalities: [ ResponseModalities.text, - ResponseModalities.audio, + ResponseModalities.image, ], ), ); - var inlineDatas = response.inlineDatas.toList(); + var inlineDatas = response.inlineDataParts.toList(); if (inlineDatas.isEmpty) { _showError('No response from API.'); @@ -194,10 +194,6 @@ class _ChatPageState extends State { fromUser: false, ), ); - } else { - if (inlineData.mimeType.contains('audio')) { - print('Got audio response'); - } } } setState(() { diff --git a/packages/firebase_vertexai/firebase_vertexai/lib/src/api.dart b/packages/firebase_vertexai/firebase_vertexai/lib/src/api.dart index 668e2ab6342f..43a5374570a0 100644 --- a/packages/firebase_vertexai/firebase_vertexai/lib/src/api.dart +++ b/packages/firebase_vertexai/firebase_vertexai/lib/src/api.dart @@ -114,7 +114,7 @@ final class GenerateContentResponse { /// Returns an empty list if there are no candidates, or if the first /// candidate has no [InlineDataPart] parts. There is no error thrown if the /// prompt or response were blocked. - Iterable get inlineDatas => + Iterable get inlineDataParts => candidates.firstOrNull?.content.parts.whereType() ?? const []; } diff --git a/packages/firebase_vertexai/firebase_vertexai/lib/src/live_api.dart b/packages/firebase_vertexai/firebase_vertexai/lib/src/live_api.dart index 87d0af7b2431..075c69eeef41 100644 --- a/packages/firebase_vertexai/firebase_vertexai/lib/src/live_api.dart +++ b/packages/firebase_vertexai/firebase_vertexai/lib/src/live_api.dart @@ -89,9 +89,6 @@ final class LiveGenerationConfig extends BaseGenerationConfig { /// The speech configuration. final SpeechConfig? speechConfig; - // /// The list of desired response modalities. - // final List? responseModalities; - @override Map toJson() => { ...super.toJson(),