diff --git a/packages/firebase_vertexai/firebase_vertexai/example/lib/main.dart b/packages/firebase_vertexai/firebase_vertexai/example/lib/main.dart index e6d880e15427..7928b1aa1d87 100644 --- a/packages/firebase_vertexai/firebase_vertexai/example/lib/main.dart +++ b/packages/firebase_vertexai/firebase_vertexai/example/lib/main.dart @@ -41,7 +41,7 @@ void main() async { var vertexInstance = FirebaseVertexAI.instanceFor(auth: FirebaseAuth.instance); - final model = vertexInstance.generativeModel(model: 'gemini-1.5-flash'); + final model = vertexInstance.generativeModel(model: 'gemini-2.0-flash'); runApp(GenerativeAISample(model: model)); } diff --git a/packages/firebase_vertexai/firebase_vertexai/example/lib/pages/chat_page.dart b/packages/firebase_vertexai/firebase_vertexai/example/lib/pages/chat_page.dart index 33b6e1141b72..489fccafa577 100644 --- a/packages/firebase_vertexai/firebase_vertexai/example/lib/pages/chat_page.dart +++ b/packages/firebase_vertexai/firebase_vertexai/example/lib/pages/chat_page.dart @@ -95,6 +95,17 @@ class _ChatPageState extends State { const SizedBox.square( dimension: 15, ), + if (!_loading) + IconButton( + onPressed: () async { + await _imageResponse(_textController.text); + }, + icon: Icon( + Icons.image, + color: Theme.of(context).colorScheme.primary, + ), + tooltip: 'Image response', + ), if (!_loading) IconButton( onPressed: () async { @@ -152,6 +163,58 @@ class _ChatPageState extends State { } } + Future _imageResponse(String message) async { + setState(() { + _loading = true; + }); + + try { + _messages.add(MessageData(text: message, fromUser: true)); + var response = await widget.model.generateContent( + [Content.text(message)], + generationConfig: GenerationConfig( + responseModalities: [ + ResponseModalities.text, + ResponseModalities.image, + ], + ), + ); + var inlineDatas = response.inlineDataParts.toList(); + + if (inlineDatas.isEmpty) { + _showError('No response from API.'); + return; + } else { + for (final inlineData in inlineDatas) { + if (inlineData.mimeType.contains('image')) { + _messages.add( + MessageData( + text: response.text, + image: Image.memory(inlineData.bytes), + fromUser: false, + ), + ); + } + } + setState(() { + _loading = false; + _scrollDown(); + }); + } + } catch (e) { + _showError(e.toString()); + setState(() { + _loading = false; + }); + } finally { + _textController.clear(); + setState(() { + _loading = false; + }); + _textFieldFocus.requestFocus(); + } + } + void _showError(String message) { showDialog( context: context, diff --git a/packages/firebase_vertexai/firebase_vertexai/lib/firebase_vertexai.dart b/packages/firebase_vertexai/firebase_vertexai/lib/firebase_vertexai.dart index 694470ceef23..de96a38e8302 100644 --- a/packages/firebase_vertexai/firebase_vertexai/lib/firebase_vertexai.dart +++ b/packages/firebase_vertexai/firebase_vertexai/lib/firebase_vertexai.dart @@ -27,6 +27,7 @@ export 'src/api.dart' HarmProbability, HarmBlockMethod, PromptFeedback, + ResponseModalities, SafetyRating, SafetySetting, // TODO(cynthiajiang) remove in next breaking change. @@ -72,7 +73,6 @@ export 'src/live_api.dart' show LiveGenerationConfig, SpeechConfig, - ResponseModalities, LiveServerMessage, LiveServerContent, LiveServerToolCall, diff --git a/packages/firebase_vertexai/firebase_vertexai/lib/src/api.dart b/packages/firebase_vertexai/firebase_vertexai/lib/src/api.dart index 6697c972dc05..43a5374570a0 100644 --- a/packages/firebase_vertexai/firebase_vertexai/lib/src/api.dart +++ b/packages/firebase_vertexai/firebase_vertexai/lib/src/api.dart @@ -108,6 +108,15 @@ final class GenerateContentResponse { Iterable get functionCalls => candidates.firstOrNull?.content.parts.whereType() ?? const []; + + /// The inline data parts of the first candidate in [candidates], if any. + /// + /// Returns an empty list if there are no candidates, or if the first + /// candidate has no [InlineDataPart] parts. There is no error thrown if the + /// prompt or response were blocked. + Iterable get inlineDataParts => + candidates.firstOrNull?.content.parts.whereType() ?? + const []; } /// Feedback metadata of a prompt specified in a [GenerativeModel] request. @@ -650,6 +659,24 @@ enum HarmBlockMethod { Object toJson() => _jsonString; } +/// The available response modalities. +enum ResponseModalities { + /// Text response modality. + text('TEXT'), + + /// Image response modality. + image('IMAGE'), + + /// Audio response modality. + audio('AUDIO'); + + const ResponseModalities(this._jsonString); + final String _jsonString; + + /// Convert to json format + String toJson() => _jsonString; +} + /// Configuration options for model generation and outputs. abstract class BaseGenerationConfig { // ignore: public_member_api_docs @@ -661,6 +688,7 @@ abstract class BaseGenerationConfig { this.topK, this.presencePenalty, this.frequencyPenalty, + this.responseModalities, }); /// Number of generated responses to return. @@ -737,6 +765,9 @@ abstract class BaseGenerationConfig { /// for more details. final double? frequencyPenalty; + /// The list of desired response modalities. + final List? responseModalities; + // ignore: public_member_api_docs Map toJson() => { if (candidateCount case final candidateCount?) @@ -750,6 +781,9 @@ abstract class BaseGenerationConfig { 'presencePenalty': presencePenalty, if (frequencyPenalty case final frequencyPenalty?) 'frequencyPenalty': frequencyPenalty, + if (responseModalities case final responseModalities?) + 'responseModalities': + responseModalities.map((modality) => modality.toJson()).toList(), }; } @@ -765,6 +799,7 @@ final class GenerationConfig extends BaseGenerationConfig { super.topK, super.presencePenalty, super.frequencyPenalty, + super.responseModalities, this.responseMimeType, this.responseSchema, }); @@ -990,6 +1025,9 @@ SafetyRating _parseSafetyRating(Object? jsonObject) { if (jsonObject is! Map) { throw unhandledFormat('SafetyRating', jsonObject); } + if (jsonObject.isEmpty) { + return SafetyRating(HarmCategory.unknown, HarmProbability.unknown); + } return SafetyRating(HarmCategory._parseValue(jsonObject['category']), HarmProbability._parseValue(jsonObject['probability']), probabilityScore: jsonObject['probabilityScore'] as double?, diff --git a/packages/firebase_vertexai/firebase_vertexai/lib/src/live_api.dart b/packages/firebase_vertexai/firebase_vertexai/lib/src/live_api.dart index 0454f7fe91b9..075c69eeef41 100644 --- a/packages/firebase_vertexai/firebase_vertexai/lib/src/live_api.dart +++ b/packages/firebase_vertexai/firebase_vertexai/lib/src/live_api.dart @@ -71,30 +71,12 @@ class SpeechConfig { }; } -/// The available response modalities. -enum ResponseModalities { - /// Text response modality. - text('TEXT'), - - /// Image response modality. - image('IMAGE'), - - /// Audio response modality. - audio('AUDIO'); - - const ResponseModalities(this._jsonString); - final String _jsonString; - - /// Convert to json format - String toJson() => _jsonString; -} - /// Configures live generation settings. final class LiveGenerationConfig extends BaseGenerationConfig { // ignore: public_member_api_docs LiveGenerationConfig({ this.speechConfig, - this.responseModalities, + super.responseModalities, super.candidateCount, super.maxOutputTokens, super.temperature, @@ -107,17 +89,11 @@ final class LiveGenerationConfig extends BaseGenerationConfig { /// The speech configuration. final SpeechConfig? speechConfig; - /// The list of desired response modalities. - final List? responseModalities; - @override Map toJson() => { ...super.toJson(), if (speechConfig case final speechConfig?) 'speechConfig': speechConfig.toJson(), - if (responseModalities case final responseModalities?) - 'responseModalities': - responseModalities.map((modality) => modality.toJson()).toList(), }; } diff --git a/packages/firebase_vertexai/firebase_vertexai/test/live_test.dart b/packages/firebase_vertexai/firebase_vertexai/test/live_test.dart index 460cbfa69fb6..b8b73d17e0ca 100644 --- a/packages/firebase_vertexai/firebase_vertexai/test/live_test.dart +++ b/packages/firebase_vertexai/firebase_vertexai/test/live_test.dart @@ -13,6 +13,7 @@ // limitations under the License. import 'dart:typed_data'; +import 'package:firebase_vertexai/src/api.dart'; import 'package:firebase_vertexai/src/content.dart'; import 'package:firebase_vertexai/src/error.dart'; import 'package:firebase_vertexai/src/live_api.dart';