From 55c05a03925a219a2c91f67b8c371605c556e139 Mon Sep 17 00:00:00 2001 From: Daniel La Rocque Date: Wed, 23 Apr 2025 11:19:30 -0400 Subject: [PATCH 01/20] Hybrid inference code changes --- packages/ai/src/api.test.ts | 15 + packages/ai/src/api.ts | 29 +- .../ai/src/backwards-compatbility.test.ts | 7 +- packages/ai/src/methods/chat-session.test.ts | 19 +- packages/ai/src/methods/chat-session.ts | 4 + .../ai/src/methods/chrome-adapter.test.ts | 626 ++++++++++++++++++ packages/ai/src/methods/chrome-adapter.ts | 327 +++++++++ packages/ai/src/methods/count-tokens.test.ts | 44 +- packages/ai/src/methods/count-tokens.ts | 17 +- .../ai/src/methods/generate-content.test.ts | 66 +- packages/ai/src/methods/generate-content.ts | 53 +- .../ai/src/models/generative-model.test.ts | 191 ++++-- packages/ai/src/models/generative-model.ts | 16 +- packages/ai/src/types/language-model.ts | 83 +++ packages/ai/src/types/requests.ts | 38 ++ 15 files changed, 1430 insertions(+), 105 deletions(-) create mode 100644 packages/ai/src/methods/chrome-adapter.test.ts create mode 100644 packages/ai/src/methods/chrome-adapter.ts create mode 100644 packages/ai/src/types/language-model.ts diff --git a/packages/ai/src/api.test.ts b/packages/ai/src/api.test.ts index 27237b4edd3..6ce353107ac 100644 --- a/packages/ai/src/api.test.ts +++ b/packages/ai/src/api.test.ts @@ -102,6 +102,21 @@ describe('Top level API', () => { expect(genModel).to.be.an.instanceOf(GenerativeModel); expect(genModel.model).to.equal('publishers/google/models/my-model'); }); + it('getGenerativeModel with HybridParams sets a default model', () => { + const genModel = getGenerativeModel(fakeAI, { + mode: 'only_on_device' + }); + expect(genModel.model).to.equal( + `publishers/google/models/${GenerativeModel.DEFAULT_HYBRID_IN_CLOUD_MODEL}` + ); + }); + it('getGenerativeModel with HybridParams honors a model override', () => { + const genModel = getGenerativeModel(fakeAI, { + mode: 'prefer_on_device', + inCloudParams: { model: 'my-model' } + }); + expect(genModel.model).to.equal('publishers/google/models/my-model'); + }); it('getImagenModel throws if no model is provided', () => { try { getImagenModel(fakeAI, {} as ImagenModelParams); diff --git a/packages/ai/src/api.ts b/packages/ai/src/api.ts index d2229c067fc..4a27be8786f 100644 --- a/packages/ai/src/api.ts +++ b/packages/ai/src/api.ts @@ -23,6 +23,7 @@ import { AIService } from './service'; import { AI, AIOptions, VertexAI, VertexAIOptions } from './public-types'; import { ImagenModelParams, + HybridParams, ModelParams, RequestOptions, AIErrorCode @@ -31,6 +32,8 @@ import { AIError } from './errors'; import { AIModel, GenerativeModel, ImagenModel } from './models'; import { encodeInstanceIdentifier } from './helpers'; import { GoogleAIBackend, VertexAIBackend } from './backend'; +import { ChromeAdapter } from './methods/chrome-adapter'; +import { LanguageModel } from './types/language-model'; export { ChatSession } from './methods/chat-session'; export * from './requests/schema-builder'; @@ -147,16 +150,36 @@ export function getAI( */ export function getGenerativeModel( ai: AI, - modelParams: ModelParams, + modelParams: ModelParams | HybridParams, requestOptions?: RequestOptions ): GenerativeModel { - if (!modelParams.model) { + // Uses the existence of HybridParams.mode to clarify the type of the modelParams input. + const hybridParams = modelParams as HybridParams; + let inCloudParams: ModelParams; + if (hybridParams.mode) { + inCloudParams = hybridParams.inCloudParams || { + model: GenerativeModel.DEFAULT_HYBRID_IN_CLOUD_MODEL + }; + } else { + inCloudParams = modelParams as ModelParams; + } + + if (!inCloudParams.model) { throw new AIError( AIErrorCode.NO_MODEL, `Must provide a model name. Example: getGenerativeModel({ model: 'my-model-name' })` ); } - return new GenerativeModel(ai, modelParams, requestOptions); + return new GenerativeModel( + ai, + inCloudParams, + new ChromeAdapter( + window.LanguageModel as LanguageModel, + hybridParams.mode, + hybridParams.onDeviceParams + ), + requestOptions + ); } /** diff --git a/packages/ai/src/backwards-compatbility.test.ts b/packages/ai/src/backwards-compatbility.test.ts index 62463009b24..da0b613bf21 100644 --- a/packages/ai/src/backwards-compatbility.test.ts +++ b/packages/ai/src/backwards-compatbility.test.ts @@ -28,6 +28,7 @@ import { } from './api'; import { AI, VertexAI, AIErrorCode } from './public-types'; import { VertexAIBackend } from './backend'; +import { ChromeAdapter } from './methods/chrome-adapter'; function assertAssignable(): void {} @@ -65,7 +66,11 @@ describe('backwards-compatible types', () => { it('AIModel is backwards compatible with VertexAIModel', () => { assertAssignable(); - const model = new GenerativeModel(fakeAI, { model: 'model-name' }); + const model = new GenerativeModel( + fakeAI, + { model: 'model-name' }, + new ChromeAdapter() + ); expect(model).to.be.instanceOf(AIModel); expect(model).to.be.instanceOf(VertexAIModel); }); diff --git a/packages/ai/src/methods/chat-session.test.ts b/packages/ai/src/methods/chat-session.test.ts index 0564aa84ed6..ed0b4d4877f 100644 --- a/packages/ai/src/methods/chat-session.test.ts +++ b/packages/ai/src/methods/chat-session.test.ts @@ -24,6 +24,7 @@ import { GenerateContentStreamResult } from '../types'; import { ChatSession } from './chat-session'; import { ApiSettings } from '../types/internal'; import { VertexAIBackend } from '../backend'; +import { ChromeAdapter } from './chrome-adapter'; use(sinonChai); use(chaiAsPromised); @@ -46,7 +47,11 @@ describe('ChatSession', () => { generateContentMethods, 'generateContent' ).rejects('generateContent failed'); - const chatSession = new ChatSession(fakeApiSettings, 'a-model'); + const chatSession = new ChatSession( + fakeApiSettings, + 'a-model', + new ChromeAdapter() + ); await expect(chatSession.sendMessage('hello')).to.be.rejected; expect(generateContentStub).to.be.calledWith( fakeApiSettings, @@ -63,7 +68,11 @@ describe('ChatSession', () => { generateContentMethods, 'generateContentStream' ).rejects('generateContentStream failed'); - const chatSession = new ChatSession(fakeApiSettings, 'a-model'); + const chatSession = new ChatSession( + fakeApiSettings, + 'a-model', + new ChromeAdapter() + ); await expect(chatSession.sendMessageStream('hello')).to.be.rejected; expect(generateContentStreamStub).to.be.calledWith( fakeApiSettings, @@ -82,7 +91,11 @@ describe('ChatSession', () => { generateContentMethods, 'generateContentStream' ).resolves({} as unknown as GenerateContentStreamResult); - const chatSession = new ChatSession(fakeApiSettings, 'a-model'); + const chatSession = new ChatSession( + fakeApiSettings, + 'a-model', + new ChromeAdapter() + ); await chatSession.sendMessageStream('hello'); expect(generateContentStreamStub).to.be.calledWith( fakeApiSettings, diff --git a/packages/ai/src/methods/chat-session.ts b/packages/ai/src/methods/chat-session.ts index 60794001e37..112ddf5857e 100644 --- a/packages/ai/src/methods/chat-session.ts +++ b/packages/ai/src/methods/chat-session.ts @@ -30,6 +30,7 @@ import { validateChatHistory } from './chat-session-helpers'; import { generateContent, generateContentStream } from './generate-content'; import { ApiSettings } from '../types/internal'; import { logger } from '../logger'; +import { ChromeAdapter } from './chrome-adapter'; /** * Do not log a message for this error. @@ -50,6 +51,7 @@ export class ChatSession { constructor( apiSettings: ApiSettings, public model: string, + private chromeAdapter: ChromeAdapter, public params?: StartChatParams, public requestOptions?: RequestOptions ) { @@ -95,6 +97,7 @@ export class ChatSession { this._apiSettings, this.model, generateContentRequest, + this.chromeAdapter, this.requestOptions ) ) @@ -146,6 +149,7 @@ export class ChatSession { this._apiSettings, this.model, generateContentRequest, + this.chromeAdapter, this.requestOptions ); diff --git a/packages/ai/src/methods/chrome-adapter.test.ts b/packages/ai/src/methods/chrome-adapter.test.ts new file mode 100644 index 00000000000..fbe7ec1a5c5 --- /dev/null +++ b/packages/ai/src/methods/chrome-adapter.test.ts @@ -0,0 +1,626 @@ +/** + * @license + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { AIError } from '../errors'; +import { expect, use } from 'chai'; +import sinonChai from 'sinon-chai'; +import chaiAsPromised from 'chai-as-promised'; +import { ChromeAdapter } from './chrome-adapter'; +import { + Availability, + LanguageModel, + LanguageModelCreateOptions, + LanguageModelMessageContent +} from '../types/language-model'; +import { match, stub } from 'sinon'; +import { GenerateContentRequest, AIErrorCode } from '../types'; +import { Schema } from '../api'; + +use(sinonChai); +use(chaiAsPromised); + +/** + * Converts the ReadableStream from response.body to an array of strings. + */ +async function toStringArray( + stream: ReadableStream +): Promise { + const decoder = new TextDecoder(); + const actual = []; + const reader = stream.getReader(); + while (true) { + const { done, value } = await reader.read(); + if (done) { + break; + } + actual.push(decoder.decode(value)); + } + return actual; +} + +describe('ChromeAdapter', () => { + describe('constructor', () => { + it('sets image as expected input type by default', async () => { + const languageModelProvider = { + availability: () => Promise.resolve(Availability.available) + } as LanguageModel; + const availabilityStub = stub( + languageModelProvider, + 'availability' + ).resolves(Availability.available); + const adapter = new ChromeAdapter( + languageModelProvider, + 'prefer_on_device' + ); + await adapter.isAvailable({ + contents: [ + { + role: 'user', + parts: [{ text: 'hi' }] + } + ] + }); + expect(availabilityStub).to.have.been.calledWith({ + expectedInputs: [{ type: 'image' }] + }); + }); + it('honors explicitly set expected inputs', async () => { + const languageModelProvider = { + availability: () => Promise.resolve(Availability.available) + } as LanguageModel; + const availabilityStub = stub( + languageModelProvider, + 'availability' + ).resolves(Availability.available); + const createOptions = { + // Explicitly sets expected inputs. + expectedInputs: [{ type: 'text' }] + } as LanguageModelCreateOptions; + const adapter = new ChromeAdapter( + languageModelProvider, + 'prefer_on_device', + { + createOptions + } + ); + await adapter.isAvailable({ + contents: [ + { + role: 'user', + parts: [{ text: 'hi' }] + } + ] + }); + expect(availabilityStub).to.have.been.calledWith(createOptions); + }); + }); + describe('isAvailable', () => { + it('returns false if mode is only cloud', async () => { + const adapter = new ChromeAdapter(undefined, 'only_in_cloud'); + expect( + await adapter.isAvailable({ + contents: [] + }) + ).to.be.false; + }); + it('returns false if LanguageModel API is undefined', async () => { + const adapter = new ChromeAdapter(undefined, 'prefer_on_device'); + expect( + await adapter.isAvailable({ + contents: [] + }) + ).to.be.false; + }); + it('returns false if request contents empty', async () => { + const adapter = new ChromeAdapter( + { + availability: async () => Availability.available + } as LanguageModel, + 'prefer_on_device' + ); + expect( + await adapter.isAvailable({ + contents: [] + }) + ).to.be.false; + }); + it('returns false if request content has non-user role', async () => { + const adapter = new ChromeAdapter( + { + availability: async () => Availability.available + } as LanguageModel, + 'prefer_on_device' + ); + expect( + await adapter.isAvailable({ + contents: [ + { + role: 'model', + parts: [] + } + ] + }) + ).to.be.false; + }); + it('returns true if request has image with supported mime type', async () => { + const adapter = new ChromeAdapter( + { + availability: async () => Availability.available + } as LanguageModel, + 'prefer_on_device' + ); + for (const mimeType of ChromeAdapter.SUPPORTED_MIME_TYPES) { + expect( + await adapter.isAvailable({ + contents: [ + { + role: 'user', + parts: [ + { + inlineData: { + mimeType, + data: '' + } + } + ] + } + ] + }) + ).to.be.true; + } + }); + it('returns true if model is readily available', async () => { + const languageModelProvider = { + availability: () => Promise.resolve(Availability.available) + } as LanguageModel; + const adapter = new ChromeAdapter( + languageModelProvider, + 'prefer_on_device' + ); + expect( + await adapter.isAvailable({ + contents: [ + { + role: 'user', + parts: [ + { text: 'describe this image' }, + { inlineData: { mimeType: 'image/jpeg', data: 'asd' } } + ] + } + ] + }) + ).to.be.true; + }); + it('returns false and triggers download when model is available after download', async () => { + const languageModelProvider = { + availability: () => Promise.resolve(Availability.downloadable), + create: () => Promise.resolve({}) + } as LanguageModel; + const createStub = stub(languageModelProvider, 'create').resolves( + {} as LanguageModel + ); + const createOptions = { + expectedInputs: [{ type: 'image' }] + } as LanguageModelCreateOptions; + const adapter = new ChromeAdapter( + languageModelProvider, + 'prefer_on_device', + { createOptions } + ); + expect( + await adapter.isAvailable({ + contents: [{ role: 'user', parts: [{ text: 'hi' }] }] + }) + ).to.be.false; + expect(createStub).to.have.been.calledOnceWith(createOptions); + }); + it('avoids redundant downloads', async () => { + const languageModelProvider = { + availability: () => Promise.resolve(Availability.downloadable), + create: () => Promise.resolve({}) + } as LanguageModel; + const downloadPromise = new Promise(() => { + /* never resolves */ + }); + const createStub = stub(languageModelProvider, 'create').returns( + downloadPromise + ); + const adapter = new ChromeAdapter(languageModelProvider); + await adapter.isAvailable({ + contents: [{ role: 'user', parts: [{ text: 'hi' }] }] + }); + await adapter.isAvailable({ + contents: [{ role: 'user', parts: [{ text: 'hi' }] }] + }); + expect(createStub).to.have.been.calledOnce; + }); + it('clears state when download completes', async () => { + const languageModelProvider = { + availability: () => Promise.resolve(Availability.downloadable), + create: () => Promise.resolve({}) + } as LanguageModel; + let resolveDownload; + const downloadPromise = new Promise(resolveCallback => { + resolveDownload = resolveCallback; + }); + const createStub = stub(languageModelProvider, 'create').returns( + downloadPromise + ); + const adapter = new ChromeAdapter(languageModelProvider); + await adapter.isAvailable({ + contents: [{ role: 'user', parts: [{ text: 'hi' }] }] + }); + resolveDownload!(); + await adapter.isAvailable({ + contents: [{ role: 'user', parts: [{ text: 'hi' }] }] + }); + expect(createStub).to.have.been.calledTwice; + }); + it('returns false when model is never available', async () => { + const languageModelProvider = { + availability: () => Promise.resolve(Availability.unavailable), + create: () => Promise.resolve({}) + } as LanguageModel; + const adapter = new ChromeAdapter( + languageModelProvider, + 'prefer_on_device' + ); + expect( + await adapter.isAvailable({ + contents: [{ role: 'user', parts: [{ text: 'hi' }] }] + }) + ).to.be.false; + }); + }); + describe('generateContent', () => { + it('throws if Chrome API is undefined', async () => { + const adapter = new ChromeAdapter(undefined, 'only_on_device'); + await expect( + adapter.generateContent({ + contents: [] + }) + ) + .to.eventually.be.rejectedWith( + AIError, + 'Chrome AI requested for unsupported browser version.' + ) + .and.have.property('code', AIErrorCode.REQUEST_ERROR); + }); + it('generates content', async () => { + const languageModelProvider = { + create: () => Promise.resolve({}) + } as LanguageModel; + const languageModel = { + // eslint-disable-next-line @typescript-eslint/no-unused-vars + prompt: (p: LanguageModelMessageContent[]) => Promise.resolve('') + } as LanguageModel; + const createStub = stub(languageModelProvider, 'create').resolves( + languageModel + ); + const promptOutput = 'hi'; + const promptStub = stub(languageModel, 'prompt').resolves(promptOutput); + const createOptions = { + systemPrompt: 'be yourself', + expectedInputs: [{ type: 'image' }] + } as LanguageModelCreateOptions; + const adapter = new ChromeAdapter( + languageModelProvider, + 'prefer_on_device', + { createOptions } + ); + const request = { + contents: [{ role: 'user', parts: [{ text: 'anything' }] }] + } as GenerateContentRequest; + const response = await adapter.generateContent(request); + // Asserts initialization params are proxied. + expect(createStub).to.have.been.calledOnceWith(createOptions); + // Asserts Vertex input type is mapped to Chrome type. + expect(promptStub).to.have.been.calledOnceWith([ + { + type: 'text', + content: request.contents[0].parts[0].text + } + ]); + // Asserts expected output. + expect(await response.json()).to.deep.equal({ + candidates: [ + { + content: { + parts: [{ text: promptOutput }] + } + } + ] + }); + }); + it('generates content using image type input', async () => { + const languageModelProvider = { + create: () => Promise.resolve({}) + } as LanguageModel; + const languageModel = { + // eslint-disable-next-line @typescript-eslint/no-unused-vars + prompt: (p: LanguageModelMessageContent[]) => Promise.resolve('') + } as LanguageModel; + const createStub = stub(languageModelProvider, 'create').resolves( + languageModel + ); + const promptOutput = 'hi'; + const promptStub = stub(languageModel, 'prompt').resolves(promptOutput); + const createOptions = { + systemPrompt: 'be yourself', + expectedInputs: [{ type: 'image' }] + } as LanguageModelCreateOptions; + const adapter = new ChromeAdapter( + languageModelProvider, + 'prefer_on_device', + { createOptions } + ); + const request = { + contents: [ + { + role: 'user', + parts: [ + { text: 'anything' }, + { + inlineData: { + data: sampleBase64EncodedImage, + mimeType: 'image/jpeg' + } + } + ] + } + ] + } as GenerateContentRequest; + const response = await adapter.generateContent(request); + // Asserts initialization params are proxied. + expect(createStub).to.have.been.calledOnceWith(createOptions); + // Asserts Vertex input type is mapped to Chrome type. + expect(promptStub).to.have.been.calledOnceWith([ + { + type: 'text', + content: request.contents[0].parts[0].text + }, + { + type: 'image', + content: match.instanceOf(ImageBitmap) + } + ]); + // Asserts expected output. + expect(await response.json()).to.deep.equal({ + candidates: [ + { + content: { + parts: [{ text: promptOutput }] + } + } + ] + }); + }); + it('honors prompt options', async () => { + const languageModel = { + // eslint-disable-next-line @typescript-eslint/no-unused-vars + prompt: (p: LanguageModelMessageContent[]) => Promise.resolve('') + } as LanguageModel; + const languageModelProvider = { + create: () => Promise.resolve(languageModel) + } as LanguageModel; + const promptOutput = '{}'; + const promptStub = stub(languageModel, 'prompt').resolves(promptOutput); + const promptOptions = { + responseConstraint: Schema.object({ + properties: {} + }) + }; + const adapter = new ChromeAdapter( + languageModelProvider, + 'prefer_on_device', + { promptOptions } + ); + const request = { + contents: [{ role: 'user', parts: [{ text: 'anything' }] }] + } as GenerateContentRequest; + await adapter.generateContent(request); + expect(promptStub).to.have.been.calledOnceWith( + [ + { + type: 'text', + content: request.contents[0].parts[0].text + } + ], + promptOptions + ); + }); + }); + describe('countTokens', () => { + it('counts tokens is not yet available', async () => { + const inputText = 'first'; + // setting up stubs + const languageModelProvider = { + create: () => Promise.resolve({}) + } as LanguageModel; + const languageModel = { + measureInputUsage: _i => Promise.resolve(123) + } as LanguageModel; + const createStub = stub(languageModelProvider, 'create').resolves( + languageModel + ); + + const adapter = new ChromeAdapter( + languageModelProvider, + 'prefer_on_device' + ); + + const countTokenRequest = { + contents: [{ role: 'user', parts: [{ text: inputText }] }] + } as GenerateContentRequest; + + try { + await adapter.countTokens(countTokenRequest); + } catch (e) { + // the call to countToken should be rejected with Error + expect((e as AIError).code).to.equal(AIErrorCode.REQUEST_ERROR); + expect((e as AIError).message).includes('not yet available'); + } + + // Asserts that no language model was initialized + expect(createStub).not.called; + }); + }); + describe('generateContentStream', () => { + it('generates content stream', async () => { + const languageModelProvider = { + create: () => Promise.resolve({}) + } as LanguageModel; + const languageModel = { + promptStreaming: _i => new ReadableStream() + } as LanguageModel; + const createStub = stub(languageModelProvider, 'create').resolves( + languageModel + ); + const part = 'hi'; + const promptStub = stub(languageModel, 'promptStreaming').returns( + new ReadableStream({ + start(controller) { + controller.enqueue([part]); + controller.close(); + } + }) + ); + const createOptions = { + expectedInputs: [{ type: 'image' }] + } as LanguageModelCreateOptions; + const adapter = new ChromeAdapter( + languageModelProvider, + 'prefer_on_device', + { createOptions } + ); + const request = { + contents: [{ role: 'user', parts: [{ text: 'anything' }] }] + } as GenerateContentRequest; + const response = await adapter.generateContentStream(request); + expect(createStub).to.have.been.calledOnceWith(createOptions); + expect(promptStub).to.have.been.calledOnceWith([ + { + type: 'text', + content: request.contents[0].parts[0].text + } + ]); + const actual = await toStringArray(response.body!); + expect(actual).to.deep.equal([ + `data: {"candidates":[{"content":{"role":"model","parts":[{"text":["${part}"]}]}}]}\n\n` + ]); + }); + it('generates content stream with image input', async () => { + const languageModelProvider = { + create: () => Promise.resolve({}) + } as LanguageModel; + const languageModel = { + promptStreaming: _i => new ReadableStream() + } as LanguageModel; + const createStub = stub(languageModelProvider, 'create').resolves( + languageModel + ); + const part = 'hi'; + const promptStub = stub(languageModel, 'promptStreaming').returns( + new ReadableStream({ + start(controller) { + controller.enqueue([part]); + controller.close(); + } + }) + ); + const createOptions = { + expectedInputs: [{ type: 'image' }] + } as LanguageModelCreateOptions; + const adapter = new ChromeAdapter( + languageModelProvider, + 'prefer_on_device', + { createOptions } + ); + const request = { + contents: [ + { + role: 'user', + parts: [ + { text: 'anything' }, + { + inlineData: { + data: sampleBase64EncodedImage, + mimeType: 'image/jpeg' + } + } + ] + } + ] + } as GenerateContentRequest; + const response = await adapter.generateContentStream(request); + expect(createStub).to.have.been.calledOnceWith(createOptions); + expect(promptStub).to.have.been.calledOnceWith([ + { + type: 'text', + content: request.contents[0].parts[0].text + }, + { + type: 'image', + content: match.instanceOf(ImageBitmap) + } + ]); + const actual = await toStringArray(response.body!); + expect(actual).to.deep.equal([ + `data: {"candidates":[{"content":{"role":"model","parts":[{"text":["${part}"]}]}}]}\n\n` + ]); + }); + it('honors prompt options', async () => { + const languageModel = { + // eslint-disable-next-line @typescript-eslint/no-unused-vars + promptStreaming: p => new ReadableStream() + } as LanguageModel; + const languageModelProvider = { + create: () => Promise.resolve(languageModel) + } as LanguageModel; + const promptStub = stub(languageModel, 'promptStreaming').returns( + new ReadableStream() + ); + const promptOptions = { + responseConstraint: Schema.object({ + properties: {} + }) + }; + const adapter = new ChromeAdapter( + languageModelProvider, + 'prefer_on_device', + { promptOptions } + ); + const request = { + contents: [{ role: 'user', parts: [{ text: 'anything' }] }] + } as GenerateContentRequest; + await adapter.generateContentStream(request); + expect(promptStub).to.have.been.calledOnceWith( + [ + { + type: 'text', + content: request.contents[0].parts[0].text + } + ], + promptOptions + ); + }); + }); +}); + +// TODO: Move to using image from test-utils. +const sampleBase64EncodedImage = + '/9j/4QDeRXhpZgAASUkqAAgAAAAGABIBAwABAAAAAQAAABoBBQABAAAAVgAAABsBBQABAAAAXgAAACgBAwABAAAAAgAAABMCAwABAAAAAQAAAGmHBAABAAAAZgAAAAAAAABIAAAAAQAAAEgAAAABAAAABwAAkAcABAAAADAyMTABkQcABAAAAAECAwCGkgcAFgAAAMAAAAAAoAcABAAAADAxMDABoAMAAQAAAP//AAACoAQAAQAAAMgAAAADoAQAAQAAACwBAAAAAAAAQVNDSUkAAABQaWNzdW0gSUQ6IDM5MP/bAEMACAYGBwYFCAcHBwkJCAoMFA0MCwsMGRITDxQdGh8eHRocHCAkLicgIiwjHBwoNyksMDE0NDQfJzk9ODI8LjM0Mv/bAEMBCQkJDAsMGA0NGDIhHCEyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMv/CABEIASwAyAMBIgACEQEDEQH/xAAbAAABBQEBAAAAAAAAAAAAAAAAAQIDBAUGB//EABgBAQEBAQEAAAAAAAAAAAAAAAABAgME/9oADAMBAAIQAxAAAAHfA7ZFFgBQAAUUBQFBFABSUBQBQBZQUiqC7wAoigooQKACgCigKIoAosIKSigABWBdZAUAUAUQUUUAFIBQAWAFAUVFABSKoLqAKAKAKJVt4BvrFLAqKooArHgoQAoKiqDyKKoaiqhSqhCqgLFKHKdBiZmbodX5n2MbWHkdZS2kWhUBQIVUBwgUucv8Oad7nUzey3vPO5q4UrlOEWjzT0vhssDpea9Gy03BsqooKhCgCgCgHIcd0fN5DnuWHseY0Ureh+ZelLIqFq+f+gQJ5f6V5r6pE4i2ioDhCFVAVWrCiBxvJdlzFzVc56GjFoy4/a8d2q2TmpN3V1OF2MWp1/NrL0hzinRnO5Sdwc+L0Jz5HQLzyy9AYQYmDrZfXkyxVs5m4yVt3F0/M7l1YotpQnScdumqsFSb0yElm4zf5hjvV56bOtteViXq3ecRMbJgG+L4tzGqNyTDJNqMx5rfSHGRdpAcidPqLyFbuBeWrdmyONg7TJTBTrqZg3b6GGzbSzILYW8uSuF2hPG9l6uFdbPQRxzU8M2Lc62fpUJZNGC5TXAseNuVc2abO0pSKUsjdI+OdNoTzYc3fIANzF1LVTalK9KU72e1coa1TOqe3naA8inKGZ0QV5ZGzSywKWVrSAUROTjuno8lSLQbFq5kNrXsYAvQu5xmW9y18l0tjmrFu8ZM66C0nLabEsPGrT3xOlnIyXjkzC8tSxh2zRbWlsVNZtY6a9SKq1ZCd0rLHS17SPlgUtvpvatrVetlYJJZRpNcOOfmRaEN+s3Vctl0qCWs+PLljs19iWw+RdZEcU1VBFVUR6Kr5a6rplEzvnH5krF9Y33LnNFkqWIynAqZ3Zno3U03xO1mVY1HrGDxgOREpURkjiMXDUXOlsVpjRIJ0RXhix3KbUuzn6DLla6nK1RwFAKKK+GNsuigXReXW6mpRS2yWu6Zgr64Rq90abqclllYVJiJxIrAkI1JXRvJZoJJqUcY1yzmrvLnMLJX1QngWQrF9hTW01IZmwlt1F5bWtMTPruLc+fYltSVo83SKpnX/8QALRAAAQQCAQMDBAIBBQAAAAAAAQACAwQREgUQExQgITAVIjEyI0AkJTM0QXD/2gAIAQEAAQUC/wDH5Z2wu/scrHmBjg+P0hzXf0pGCSPjpnwT2bDa0LOWe6dEgCW06yYIWwRf0uVrbNdf79Grg2ZeUrxkMsco+CFleP4uRuyQvPITOjdyLzS4yy+Znqts7dtcbSZOgAB8V6Yw1nlziCE39obclR8EzZ4YrUM7vRy2PLVBpbT+Plv+Nn0RPZU42jJpc9HIwOhtqk8yU/j5dxMq+1YbrVaH2eUd/lsDpJG516zRMnjLSHRt0i+PlYss613Fli5OLBhOkwv1ShNG4PlDIqdzyunjd/l/k5NwFWu0dw/gMLlXhfFyHLD+SpGZbTq8GIR3Y7NCGKvRrd9fT5F4VgLxboXZ5ALXkgs8mFZt3I5vIvLzLYXnzL6lhfVYwvq9dfVqy5IEpzTG93618me0P9S5T96GPNQDWm+f8HifZuVlZWVlZXJnPILKysoytXsuUe0y27LHxzS92Y/ca72xzmWOW1cMcklSSKIMkbIzzYNrs8b6dO1HXYLsBaHAqS0yOTKyvLb37crZOQm5Bkcw5GFykuyqZ81iJ0mru9JgJ8bmHoGly1ds+KSNMikkXZsAduVo+5HKBwmW5mFzy5z70r43WJXEyuKz9ywjs8wzSQPdkuwUAcch/u9InavA0s2maqnMYpC1rmtjAV1zvHpVi1hiiQghz4cC8SsnUqxX0+svDrix9KgzLxeHHiiG/SX4+lyI8ZMFLVmgFz9nY2UELioNnqSRz5KEa/6AUpe0Miyrf8Dadnug6uQwOjgSyKye+WyIbAEgLuRoSxORwVLU2tTyOfJj2QlkY3ua8dGN0MhO2LmkK3bkgn7Ykjk4+KQ14BXj67YNkydqtE/VahagLVqwFo3f0PHlwe4NOSWRrh7agqxUEyZmGF9+IKG/G53Q7YPfaou9amEzV+wAI9BkY0k5PWtHOwy1d3V4zC38oKaq6WQfiw+FrIIqxXutiPRlfatWLVi0YvZTU4bDnVV4zkKpRrvUbS1F3tG4hbhbhbhS2WxtmmM0nHt0gysrZZWfR7rPXKysrZbFblblbruFZ990Nc7BCYpsxXdXcWy2WyysrPXuxrvMK7sa1ytF212120RqMZGFhY6BAoFArZZWVlZWfTC1zi+0c15y9+q1WgT4F33KOUl+0a7jMtfl2PTn4K+S0xPDoIe2srKyrE2vSGPuP7LF22/EEFq5dtybDlMAYMrZbLdOsgJ7t3KJj4xn4crK2QkKDgfTnpMThmNU1jXMbNogc/DlZWVno1+FsAvz6H5x0/KhZ7/GR0wgPd7tjD1x0f8Auoxs/wCHCwtemOuUx4ag8FZHV8bcqu33+LKysArt5WpWq1WOmShIQnSZBTBs4eyz1z8AKygvZaharC1RYsdQcESLcL8rJWVn0Z6gdG9MrKys9CAUWLtuWvUEhCRbDp7rZbLKCCygvx6s9AUCisBYRCPTKyUPQ0ooOKBK/8QAIhEAAwACAgIBBQAAAAAAAAAAAAEREBIgIQIwURMiMUBQ/9oACAEDAQE/Af5k9E9yWITC9S7RCCIQhCEGuyEcPFMTYrCYsxTrDYmVQTKhPouPJ9GyNj6iG7mEIRkZGPxZGR8aTofiRkZGM6OjY/OahNFp38lZWX5NkXxPtxuzZlNjZm5ubmxc01RqakIak4XhSl9NJxf6cJxvNCxCelMp/8QAIhEAAwACAgIBBQAAAAAAAAAAAAERECASMAIhIjFAQVBx/9oACAECAQE/Af1d6LumXZs5MTLhn51pR5WlKUulz5JLFLrR/XH8ITEIQhCCHld3IbRUesez2Px0jI8PERxIz5HyPZxRxWkIQmvI5FLil6Z137C9NJ2XFL0MhD//xAA2EAABAwEFBQcDBAEFAAAAAAABAAIRIQMQEjFBEyAiMlEEMDNSYXGRQIGhIzRCklAUQ1Nwcv/aAAgBAQAGPwL/AKfYHfyMfUttf+M1TXNyIpvHCQY+icw5OEI9ktdKBbR3sAmjZDZkxnW6TQI2HZK+a00CDG/Ri3Zm3mjonWNtGMZOTJgCdTCIaS8+ixOOCyCDLMU7sWVnQxJKaHEyMy2kqWyLSYxJwtHS5u/atiOK5z7USGmIQAHdktMONAsTnEn1WQKnojgjCdE21FAUW2b5I3aHStzZ1r3jP/d5uDbV1XyWgKzrAy3Xn+L+IXWTj5e8s2aRN2SOhVm1woXLDo1oQazmOSGLOK7hY9shYdckxvQDvGWvQxuMeBiIOSbNjs36kpjvKZXihSHhOfnhE0TuDDHrdaECGMdLu9w6khYncrBiKlBozJhWTHiHAqyd6Qms+VJsmfCwhh9k97C8EDqn/quZHlVO2Wi4e2OVO2KnamrxbIr/AGimi0OA9GL9qFXsZVeyPVezWirY2qq20H2Wbv6qy+E5hzFEFZgecKwI1Vh91bOGmV1B6K1Vr9t9vsN3mCqAm7N7SOjdE0NqQZTrTrc1ztCrJ4PC3VWDcQnF+FbvLhzfhYmmicMfKuF04skQ+eI6LFtBms0xhNXH4v2MVWIHhELCDiGvoqHWE6rWwadUHTJb5dQuE16ojaEjOt0OEX0ErDBk6IF7YnqjgYTGcLw3wpwOj2WqqFTNE4qnOViJWCaR0VXnKKKr/wAKTfJMlTEjVsolZXNoAIzRuBmEHWwaGnJzRRbTZ8PnCLZaGn0WS5KrCLM1WK0xD0OS8Jhn0RH+nZ/VeC1eC1eEFyflYHWsTkAuZ/yoZaf2Xij7hTtW/YLnb+Vzs+VLsvRybaEV6SjhENu2kNwN8yfbFoMcrf4p1o9pwikTQIl1nXQkXVXCGhYiYJ8rl+4tGTlAR5nR/IthQVS4j4WztHEnQlgVLX5YtFUwvFHyqWjflcy2r3WZZ5SjifiAyXpdha8hvRCGzwprA0kzWEABT3XCQPcKpCwsIy6IY/xRTjeD7ysAM+u5ov07LaHoVithx9JyvoB8LIfCyU7Ie+60sPG3MXHEeEZIVr7qoaUDQP6obR0x0CptPhBhDhN9Ci9xDoya0IutHusmt/iFBIXDakey8QlZ31c0fdTuY2wAeqxC0OI5yoxk+l+MWpb6XfrAV0WOyAprcOAn23ch8LLcxPxfK4XfKzCqVkhxqhquMrNZrNTzegWM0U6uP00rJThF2ar3WfdSPo5mAFDcuqwu3JYYN3EQAuZRKw4e+e3QhYYWI825hGt0aLJZd5kslxKBu5IuN2hnvc+4gIzdzQVhNfX6CqpuZX0VR39d83D6ckG7F/kafT0/xf8A/8QAKhABAAIBAwMDBAIDAQAAAAAAAQARITFBURBhcSCBkTChscHR8EBQ4fH/2gAIAQEAAT8h/wAiv8iof60/24fSvm0naH+R2aUdppQR8PVerRTWafXUA+lrvlRRsJt2f+xcK5o6rMHN0LZb9Fagaq0EyEPYezzAGwavL67l+jb1sex1ucH2lNKQvo1+4DXUq1qO8JQuOPmZPNWNPbllNUa93l+m+Nx3niXqZkfLEtIvwwS75Bt1qXL9H43mjIKjs5hxLIxhtWEwAKAMH07uBuNpYwtVXCGs7xLQcmZjdZmpBJoLnaFJ1hXpOcFSE2YaxxFP5/qcz+iXToFmTpK7yt+RC1GWVyrPaHXZjILVX8kNe0A+l+w+psg/PfTViLG0CD8QCO8wRgYDiC7aYcs8evd6Brtt3jBCFweZUJVb7fUI7W74YEcS8LFVhJzjk4dy8SodQh3BdmyEXRzd7TFspRGYByYeUzF14jPPEuXLly5cuX1voJWze2sQ9Q9zg+amaprCQ2IEoCSuY63Ir4MUahd+BmIVIZuUJECnsXWXLxBDX26+XmU6Xz/7B6iXK05n8hGGqPmbfyP/ACbwnQ2SxsPmU6p4Z+gVlGn8XL6L7f8AJtJ7Q/KUi17sMo5YxypaCW4JWPpGGnmOw2v8iFmYsfKLYjkdZeDFDDg0nxh+YLPL+3rAovb+8vPUvzA65saxNfuiJo4RLXF13F2lmFXuvaKkPabIc4ZYEFrumMtNnH9E5U7Xd/MEFXvNB7FuMe0c02mB3mVhstCBhU0/pNAtCaNTXRMJW6svWpfUs6vbSB84N+NZSDuiCsttdle72mPNFBy4gHLLvAbbzAzStbf3M1+rqfeaZZioic9GqZcBKxw6mYehtWyxgJ6A0l8UrYI2w+TpmbVfCc8e01A7G4Am8NmW9XzxHqqqOF68w02AWwwaR0UXXYymRduZhOHzFc3L8ydyHa660DiXiJbc7qbQ68TJeQN5lUp3IxjxlldJXAGhvzGQDjQla/mO1nlbX8SpaWtplxI3wfuMXhYM1gea6UwzwhqIoFb6IX3dfboerh4s/c7Ku7jYbcZBKfAP4hEIvg/xCqWcYJrnusF0L2ilrPtY/UeCdwsCgzQq1kzPaNZXE8vB0QuFCtP2R/SzWKmP5lZq66aINj8zdH3JY2L3b/EUWNVZT7SgKpYEv6iCaNkipsd5QBFfMK7/ADLhKuriEWio7PmWrwcAzdF4xALHlbKs4Z1wsK+kLuRnGtlWvBMmobbEsBvLa4Ra2bGWPmIdgfeWyhbQxMealG6ViFVJbmACj/e8MOBdG1M5KoWzlPfQP2TdqXYgVMbhBCOIfJjqCjWwEDunsDxEaxiLGc+YGofiC6/tph0fEbq08FzOOphG5asjVVFSkYRPapngwWxcu0vBdTFabfWF2AxjqRcMdpCHIuhjHRaq1shjR+YLyRaBfeDFw3B95hI3XGcc98n5iGQXeCM9ykB5sGtyXMwjvSacC9j0UgA0epLcxoY1vwIuGsVEyJgECgfuUxBo3SqX0bqmOle5Fwz9XSSp7y5TclPW+DjyysaQ2D7yoIZQUVASNWtGaMDyJZG1bMueKBkF4emONKdQe8fmlpZKmGwDaCjdRVzyl+r5RZctlwODPeW5l5eWnej0a07kyste7Cuz4iOp+IbRXiF0fvmcLfaBgGB59RCuYRi1grWpmq3zACxuMsW4ipmHSFCF5eEAxPoFO6HfPOX6g+h0Hr241UgcciUSu9EJR2iYsUkpMCjTWLHiCiA7Cd0TDl5ljaUzMJfQMGEBfQvMZ3mqnuQnZf4ej09wdMswMrA4BbDfiY6VK6VAgQ6e2d5Ei4qWqn5s+itCbuWLqhlWkq2LKEXLOty5cvqlICFMPQZcHouVl00QXXQwuRGdtTZDAmnruX12bcwwxnnJGlohhFSuj0Ybtvo6KU/mKNxw06XL6X6UuLMxjxEbIUS+eOldNT7zpWodT1r8S0So9Fsy1mBrWLawbfpjeawPRVbNOteu6hB2RJpKbpkjKiWOgWj0pKSXuUpKCg6bJfRcuX1GX0CxLzOdyKnhMtou0sa9L5JmoXcg2sE0PQOcoy+lstCp7dIO81QWXhJAJh0Zhme2lG0EaxxLeickGmHRljeW3gYGMiJWUqDT0rLS24nU3GkrAgLhBQ5orOopHhhHWKMs/9oADAMBAAIAAwAAABASIMVBgAVIggAJsGy6fNBiyj4Y5ptsnyTbFtvCz9pNNPGuqMCNo42YQIEExL6CRYMEGT8YCBzUGdVEHKQHraFgCRaW/wDNpnycuGNdceiyLtY4mcgOiOu29EEGuHlAnRrvBwEb0uqOJE43dRwqzkz2egbGwwUOslkwzPIcsSwSNhRUkWEw1v62L+JMcNPr2AmjywACL2YgqfCuq0/Cz+/jqnaGEcefx1OE4WV4cia8oyMQ8U8lMsIgsWO//8QAHREAAwACAwEBAAAAAAAAAAAAAAERECEgMVFBMP/aAAgBAwEBPxBc1+a/BIhCcITMI8QhCYQhCEJkvMQmYQhMwSNeZGhNUhCEIQb2JLs6VO48HoK5+AEVawVlRxOosomXwd8GnZFXhBRoo6jcWhEUOTSFpEsbUKcC6hquh+Q9qiTHo2Gy+i7hlYQVKEyMkG6xMadEsQVNWsKSdaxKa3svsSIaTUmSLsaJEyxoR7dxN2w294KG1dcCJhIQvQkXwVG3IpKLNtFFEf038E3ME6JsbQ4LKEhtzEIQgmkJBlpkEt46D4xkZcREF0PMJiix8T5k1yH+A//EAB4RAAMBAQADAQEBAAAAAAAAAAABERAhIDFBMFFh/9oACAECAQE/EPwf5PaPLlKXwo8u0pSlHxtGUpcdGmMo/RWlC6rOhZS5zhwLrp0UmC+CpFGXTp0aFzo0Khvgvd8QpR+8Uo8UY3hhO7WUKvQfs9qhB/Q1cMLofRRZwoyLzYIjmNwtyoqx5BNoX9YkbbejnwfUEgxiqXWPwCf4cfBQoKFzOCBKesbMOHCLwvBFnCFFE4bIRBUylKUqIyEEGxKimUpcjwmijeLKUuVFHlekUospdpk/Fii0nkmn/8QAJhABAAICAgICAgIDAQAAAAAAAQARITFBURBhcYGRobHBINHw4f/aAAgBAQABPxDweDX+J4P8jfk14NeVQJUNf4G/J4NeKleKh4JQyvDDwHipXivFQJUJUrxUrxUDuVK8ceArxUJUqVA8HioeK8VAzKglSoVUqVDLKhiV4rzUCoFwxKlSpXgPBAuVK8VKrwF+K8VApm5UCV4rxmVCVA81KlngPAY8V4qV1L8DfCB7N8RCCVTnDfgMeK8G5UJXgPJhh5NeefBszFrbCQytzUeUao/D74+vBr/AgAyf4TDfk8BC0HvMPJrzz5Du/sDX4afqAmGh09Z6tZ8y6HhnL0DxVZuAzNHW4FtX6iIo7J/LlggsaQei6lY9npH/AFNo2ptfvweTUuoeUhnWfias6ur9zmvJvwbOtJ6ixUpjK35UfuXT0sbc6a5cGnnUL5mcCXrzLchY3eC3HuH3Uh0/D9mofTOTtN9iw35PBr/Ac8U7vqA+qD5uBejEvV1kHSBKE5R22G1rFxXpUFJYPmYeA58heEtci8c45jURYWjAr6YsPtTBr6p1QtXvZiUhnAA9EqG/BL8GvF+HPAhZtt/Ep6IEFjWWXZEyZxhjcAsIVY6kJuM7G4jJYFaxpL6xBJXdgs7L3DZCXPuskrndJk1KfdVNat1CRLa/LF/QQxLhuX4PA/4VRxeHLBSZcWf99S27qvcugnIGo2dXu2sS82b2g/GU/MunLN0XKR9RXnZipcJeTeMnCR4FO+1/In8VEYLeinvEoIwVXoGXnxcJcGpfi/Fy21LB7I/QfuXRjHXqK8gK5zKKcge5qpOkLtH81MXGMwG1V9/qBRMNPJuMY1SJ6Zg5lwzDEepTJTCOyvUSXhBnJM/khigpQ1Qv9+L8DDEuGZcuXLmJy595j8JEMc8nuC1NlOYZQwYgoYo0vrHxDJYqMeAChgzKA1gouBzr1iKCjyip+TcPydMB03LYrV5B7uOogpwsP/EaDsTkPzzK6RwxgYYzbLC2ZleUPuA7/crA3mse/AtMIMvwuKgIR/JSndEl3GvmUJdIWrx7blVdY7bq36i1x4YU2iJHJpkW20V/ZNdWx0Fv1REywUgayt8QlCxGmUPVal73duXYUnWY+VQ5Vkvp1Ag0hWzxDsCsXKtreYa0/wDbifph/wDkpH0qKek5slT+CIaofwlXT1a/9MP+GH5h/wB0PqaXb0oftGVjP1D/ALmeGP0e9zIIYbq2kjuNCnKUn9MAvw3aQZgIXxSv8XKN2Iv0f+yWSW7IOyCu8DX+CATBIHSMWMyI3ofUAs5L8mJc6D+IMN6h7ePz/cKYvEpSSoVxhPc7rmPMHW38zcW1eWqOWAiW1MVH4jixHSNPq63CEMEwbVAtddYleJbjRl+6qUt1UOMD8x6hdbNH3OdTEKNn3uYnWIotw22VL6i1l282Y3BCipGSWhRzahznsOD76iAbC4lVV25rqG3MRWFkeviCur66Mct/MICcbEf7V7ghVYEpzTpqFMewB7H7lg2lxHBUByqDApdpbLOHlsg7m7CgEPbvqc3VboZs7UcmYEolD8gcGV/UE4ubQVrDspUiXl23DrBwRa6lX2IrB2HTqLvOkKi3pemJetOKgvvC7GOIgruagHj22wp4akoviWsDVT8BmYYyWD9LnBBXAfoYpCBtFdrgibPAo/mGxbGKaEFBQIhVs1BrbVCoYrPUGI40OBqpS3BgF9lwUjdg5be4fSpbgAbN6lmQ2Jw5hzC5q1qIuyH3/uYsKtqcFEDqLQa8BadkDjGVt7gxY52EBmfsodOLYW6TiLZmtcnpllt3zKfRULQeUNkDIQVQ9Ff5lSnC/dWRunxDrAWE/T/CKLUlTl81iG04NeTdNFhBjiqVjdUX+Suos14DB3m7/UOlfVaPshiMBuGIXw1mWaer/wCkSLT+T/2Jf936ilV+I/7iREraYdFtsuA2+RGbJMKx8lJYIdJ/YV/UCVpV0n+iYILiy/qU5FqApirNIF6v1dxZbfwGYPzAryVXA85iHAPqGrsbZbeqMsKUJysHNv7I/FtkKAdFZwOIWOYw1Zsbz+IgC2um/lhhRL7yfqGKZ7xXaBmJzVNxbsY+KgZZbSfOFX3AboByDpRcx0HPYk/gIWAGjp9wJXC+oGmdIVbhE/uPyjmUfUb9WRDCBz+3CRAtrtSX6iStHACJ00uQJG30oN/zKAObBH5ghoDQbNAZh0hYGwesRpxTYNn3M8XUvGTdAbhRDqWQ5RfxLD8hS2NZ0IWX0ypT1Yqgdo3KBm0HyWMsIkDDQv7QutMrDgjS9trKAWqfiVhQ0OEdVHLE4pVKutai4IfbcRaHwVMBT9kIKi7Mv43KuOoPkbgk66BXXANRgEnuq/qUdpdmQ/1HgPoCBsd/B+poNfRSMQzT7Vxof3CgoFBxqV1DBEmURG919Ra5zFyNa+O4EC9qA4O+YLAIWyXNPMVlScBr5qcc8llH2wMABLUvYO/cGGRtbVwVnqYQBQ1/lg49ExPtDEHJvqC8nyxGE4ZV9wS4xFo6tbFUaFKj1/b+ojAGFMH1RhzbxQv7shIe6Av4JyvmEsVZAvISkembc1pl36c0Hmqz+5VygUUjd0R6OEhZTwJxHTZzQpPUpWRUKrftCMsCANFcymG0C8uqmp7kBXsgC3pZW4zFwW+kJkYmEfZbK8MpBpD8za0H5LYpgE5HmLL4S6a/E4AHRiLberLAAIU3doNi6JaY16Kl3gMYQQpHqXCTGK7iiHAEfctwAMl1ACDZGZIjAHhP9gmxYd0uZuDgbf8AyJllcAPVzMwCAqjBDDZgm385nymeL8C93FMbMMoyZIXZLu/zBTUZr2mXdxLcTNsaNvzO1Ms51/cA1T5ifvUIfUIUCO6GYMBDWH8SyIsutf4gQfGEPKHVDNpOYIr0gO7gJRge4B5I+k+5R4RBU1OiEBXdSdBaaYgwASymJ0xOmNu0DxLy8HMxgR5IdcC4IhiA9koep6SYdwzbCrCJ8qWgo3cHRiW6i1t8uplil/Gm+EDlhl7+IQriMAIlZgIkN1wwlhiFNqmbEbag5Z+WVoNtRWRiYR/HxADMInphBTljsbtmU1Z/gbzMPSuJWSeADDBlpK9R844ZlatMdyuLdW9S1tSrb3KFEVL9Eq0s0bgUsaYAOAPipUv1LmagX4Lwxu4kjlTQJqPVKbt6jpQ8BuZKUtrtcE6f3BHMwzcvFNF7iaBOiwmzwsOjqWBytSlBIVYSImoGtQTiAMqnDiEA6geoV4hhglzidqIWLEpFPq4I5H7lBiHJntZbuDhMI21AlSVV7uN2K5gwnXtqV7OxsqN3aLINwxATklvqX8RQiHuNdXFDzHOdDEsiibDDMuKdysqyYxKoqwgiWhZDUs7auJaGZbGLNcNRmwMZ4mIAqoKcwvLy3uWlstiyyDpAe40mHDcNKMM4mrBo9Rql+0o0V4q6xLhQY9w1j6eBRspuziNNtwcwblPH35CF9ZnqSnZHWZbiUjAm7j7cIfkQo4s4nLrTcUFojCAm0WJlBumAvA0YCENztcMQS5Y+BCDbCzczZgiXYl6wgbC/MM1MTBZNUS1kgJOBItSqTRheZaluO2c2/Ex/A6gOYM4Z8LlvH4wctYPgKMrrNz0kaSFfBcQMbTjNkVebSsAZEYVpqUXFUIMTOEVEzSZaSS9QXSoEwwdZSWPNSnWYcxGiy1hd7QEtxE6VC8oBhFOZbOXuCXgQz1JRZhEsa8GAimGoqB4BcGhixA8DEQc3Fc1LW7gsweg3Lo024ah5Q0wDmHMZ3IicQl3RmGShHATpwWJEjhZUcytCWLOYRDCktgtnuAFhmYO5vRP/2Q=='; diff --git a/packages/ai/src/methods/chrome-adapter.ts b/packages/ai/src/methods/chrome-adapter.ts new file mode 100644 index 00000000000..aa3709048a2 --- /dev/null +++ b/packages/ai/src/methods/chrome-adapter.ts @@ -0,0 +1,327 @@ +/** + * @license + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { AIError } from '../errors'; +import { logger } from '../logger'; +import { + CountTokensRequest, + GenerateContentRequest, + InferenceMode, + Part, + AIErrorCode, + OnDeviceParams +} from '../types'; +import { + Availability, + LanguageModel, + LanguageModelMessageContent +} from '../types/language-model'; + +/** + * Defines an inference "backend" that uses Chrome's on-device model, + * and encapsulates logic for detecting when on-device is possible. + */ +export class ChromeAdapter { + // Visible for testing + static SUPPORTED_MIME_TYPES = ['image/jpeg', 'image/png']; + private isDownloading = false; + private downloadPromise: Promise | undefined; + private oldSession: LanguageModel | undefined; + constructor( + private languageModelProvider?: LanguageModel, + private mode?: InferenceMode, + private onDeviceParams: OnDeviceParams = { + createOptions: { + // Defaults to support image inputs for convenience. + expectedInputs: [{ type: 'image' }] + } + } + ) {} + + /** + * Checks if a given request can be made on-device. + * + *
    Encapsulates a few concerns: + *
  1. the mode
    2. + *
  2. API existence
    3. + *
  3. prompt formatting
    4. + *
  4. model availability, including triggering download if necessary
    5. + *
+ * + *

Pros: callers needn't be concerned with details of on-device availability.

+ *

Cons: this method spans a few concerns and splits request validation from usage. + * If instance variables weren't already part of the API, we could consider a better + * separation of concerns.

+ */ + async isAvailable(request: GenerateContentRequest): Promise { + if (this.mode === 'only_in_cloud') { + logger.debug( + `On-device inference unavailable because mode is "only_in_cloud".` + ); + return false; + } + + // Triggers out-of-band download so model will eventually become available. + const availability = await this.downloadIfAvailable(); + + if (this.mode === 'only_on_device') { + return true; + } + + // Applies prefer_on_device logic. + if (availability !== Availability.available) { + logger.debug( + `On-device inference unavailable because availability is "${availability}".` + ); + return false; + } + if (!ChromeAdapter.isOnDeviceRequest(request)) { + logger.debug( + `On-device inference unavailable because request is incompatible.` + ); + return false; + } + + return true; + } + + /** + * Generates content on device. + * + *

This is comparable to {@link GenerativeModel.generateContent} for generating content in + * Cloud.

+ * @param request a standard Vertex {@link GenerateContentRequest} + * @returns {@link Response}, so we can reuse common response formatting. + */ + async generateContent(request: GenerateContentRequest): Promise { + const session = await this.createSession(); + // TODO: support multiple content objects when Chrome supports + // sequence + const contents = await Promise.all( + request.contents[0].parts.map(ChromeAdapter.toLanguageModelMessageContent) + ); + const text = await session.prompt( + contents, + this.onDeviceParams.promptOptions + ); + return ChromeAdapter.toResponse(text); + } + + /** + * Generates content stream on device. + * + *

This is comparable to {@link GenerativeModel.generateContentStream} for generating content in + * Cloud.

+ * @param request a standard Vertex {@link GenerateContentRequest} + * @returns {@link Response}, so we can reuse common response formatting. + */ + async generateContentStream( + request: GenerateContentRequest + ): Promise { + const session = await this.createSession(); + // TODO: support multiple content objects when Chrome supports + // sequence + const contents = await Promise.all( + request.contents[0].parts.map(ChromeAdapter.toLanguageModelMessageContent) + ); + const stream = await session.promptStreaming( + contents, + this.onDeviceParams.promptOptions + ); + return ChromeAdapter.toStreamResponse(stream); + } + + async countTokens(_request: CountTokensRequest): Promise { + throw new AIError( + AIErrorCode.REQUEST_ERROR, + 'Count Tokens is not yet available for on-device model.' + ); + } + + /** + * Asserts inference for the given request can be performed by an on-device model. + */ + private static isOnDeviceRequest(request: GenerateContentRequest): boolean { + // Returns false if the prompt is empty. + if (request.contents.length === 0) { + logger.debug('Empty prompt rejected for on-device inference.'); + return false; + } + + for (const content of request.contents) { + // Returns false if the request contains multiple roles, eg a chat history. + // TODO: remove this guard once LanguageModelMessage is supported. + if (content.role !== 'user') { + logger.debug( + `Non-user role "${content.role}" rejected for on-device inference.` + ); + return false; + } + + // Returns false if request contains an image with an unsupported mime type. + for (const part of content.parts) { + if ( + part.inlineData && + ChromeAdapter.SUPPORTED_MIME_TYPES.indexOf( + part.inlineData.mimeType + ) === -1 + ) { + logger.debug( + `Unsupported mime type "${part.inlineData.mimeType}" rejected for on-device inference.` + ); + return false; + } + } + } + + return true; + } + + /** + * Encapsulates logic to get availability and download a model if one is downloadable. + */ + private async downloadIfAvailable(): Promise { + const availability = await this.languageModelProvider?.availability( + this.onDeviceParams.createOptions + ); + + if (availability === Availability.downloadable) { + this.download(); + } + + return availability; + } + + /** + * Triggers out-of-band download of an on-device model. + * + *

Chrome only downloads models as needed. Chrome knows a model is needed when code calls + * LanguageModel.create.

+ * + *

Since Chrome manages the download, the SDK can only avoid redundant download requests by + * tracking if a download has previously been requested.

+ */ + private download(): void { + if (this.isDownloading) { + return; + } + this.isDownloading = true; + this.downloadPromise = this.languageModelProvider + ?.create(this.onDeviceParams.createOptions) + .then(() => { + this.isDownloading = false; + }); + } + + /** + * Converts a Vertex Part object to a Chrome LanguageModelMessageContent object. + */ + private static async toLanguageModelMessageContent( + part: Part + ): Promise { + if (part.text) { + return { + type: 'text', + content: part.text + }; + } else if (part.inlineData) { + const formattedImageContent = await fetch( + `data:${part.inlineData.mimeType};base64,${part.inlineData.data}` + ); + const imageBlob = await formattedImageContent.blob(); + const imageBitmap = await createImageBitmap(imageBlob); + return { + type: 'image', + content: imageBitmap + }; + } + // Assumes contents have been verified to contain only a single TextPart. + // TODO: support other input types + throw new Error('Not yet implemented'); + } + + /** + * Abstracts Chrome session creation. + * + *

Chrome uses a multi-turn session for all inference. Vertex uses single-turn for all + * inference. To map the Vertex API to Chrome's API, the SDK creates a new session for all + * inference.

+ * + *

Chrome will remove a model from memory if it's no longer in use, so this method ensures a + * new session is created before an old session is destroyed.

+ */ + private async createSession(): Promise { + if (!this.languageModelProvider) { + throw new AIError( + AIErrorCode.REQUEST_ERROR, + 'Chrome AI requested for unsupported browser version.' + ); + } + const newSession = await this.languageModelProvider.create( + this.onDeviceParams.createOptions + ); + if (this.oldSession) { + this.oldSession.destroy(); + } + // Holds session reference, so model isn't unloaded from memory. + this.oldSession = newSession; + return newSession; + } + + /** + * Formats string returned by Chrome as a {@link Response} returned by Vertex. + */ + private static toResponse(text: string): Response { + return { + json: async () => ({ + candidates: [ + { + content: { + parts: [{ text }] + } + } + ] + }) + } as Response; + } + + /** + * Formats string stream returned by Chrome as SSE returned by Vertex. + */ + private static toStreamResponse(stream: ReadableStream): Response { + const encoder = new TextEncoder(); + return { + body: stream.pipeThrough( + new TransformStream({ + transform(chunk, controller) { + const json = JSON.stringify({ + candidates: [ + { + content: { + role: 'model', + parts: [{ text: chunk }] + } + } + ] + }); + controller.enqueue(encoder.encode(`data: ${json}\n\n`)); + } + }) + ) + } as Response; + } +} diff --git a/packages/ai/src/methods/count-tokens.test.ts b/packages/ai/src/methods/count-tokens.test.ts index 7e04ddb3561..78c51d3f5b7 100644 --- a/packages/ai/src/methods/count-tokens.test.ts +++ b/packages/ai/src/methods/count-tokens.test.ts @@ -27,6 +27,7 @@ import { ApiSettings } from '../types/internal'; import { Task } from '../requests/request'; import { mapCountTokensRequest } from '../googleai-mappers'; import { GoogleAIBackend, VertexAIBackend } from '../backend'; +import { ChromeAdapter } from './chrome-adapter'; use(sinonChai); use(chaiAsPromised); @@ -66,7 +67,8 @@ describe('countTokens()', () => { const result = await countTokens( fakeApiSettings, 'model', - fakeRequestParams + fakeRequestParams, + new ChromeAdapter() ); expect(result.totalTokens).to.equal(6); expect(result.totalBillableCharacters).to.equal(16); @@ -92,7 +94,8 @@ describe('countTokens()', () => { const result = await countTokens( fakeApiSettings, 'model', - fakeRequestParams + fakeRequestParams, + new ChromeAdapter() ); expect(result.totalTokens).to.equal(1837); expect(result.totalBillableCharacters).to.equal(117); @@ -120,7 +123,8 @@ describe('countTokens()', () => { const result = await countTokens( fakeApiSettings, 'model', - fakeRequestParams + fakeRequestParams, + new ChromeAdapter() ); expect(result.totalTokens).to.equal(258); expect(result).to.not.have.property('totalBillableCharacters'); @@ -146,7 +150,12 @@ describe('countTokens()', () => { json: mockResponse.json } as Response); await expect( - countTokens(fakeApiSettings, 'model', fakeRequestParams) + countTokens( + fakeApiSettings, + 'model', + fakeRequestParams, + new ChromeAdapter() + ) ).to.be.rejectedWith(/404.*not found/); expect(mockFetch).to.be.called; }); @@ -164,7 +173,12 @@ describe('countTokens()', () => { it('maps request to GoogleAI format', async () => { makeRequestStub.resolves({ ok: true, json: () => {} } as Response); // Unused - await countTokens(fakeGoogleAIApiSettings, 'model', fakeRequestParams); + await countTokens( + fakeGoogleAIApiSettings, + 'model', + fakeRequestParams, + new ChromeAdapter() + ); expect(makeRequestStub).to.be.calledWith( 'model', @@ -176,4 +190,24 @@ describe('countTokens()', () => { ); }); }); + it('on-device', async () => { + const chromeAdapter = new ChromeAdapter(); + const isAvailableStub = stub(chromeAdapter, 'isAvailable').resolves(true); + const mockResponse = getMockResponse( + 'vertexAI', + 'unary-success-total-tokens.json' + ); + const countTokensStub = stub(chromeAdapter, 'countTokens').resolves( + mockResponse as Response + ); + const result = await countTokens( + fakeApiSettings, + 'model', + fakeRequestParams, + chromeAdapter + ); + expect(result.totalTokens).eq(6); + expect(isAvailableStub).to.be.called; + expect(countTokensStub).to.be.calledWith(fakeRequestParams); + }); }); diff --git a/packages/ai/src/methods/count-tokens.ts b/packages/ai/src/methods/count-tokens.ts index b1e60e3a182..81fb3ad061d 100644 --- a/packages/ai/src/methods/count-tokens.ts +++ b/packages/ai/src/methods/count-tokens.ts @@ -24,8 +24,9 @@ import { Task, makeRequest } from '../requests/request'; import { ApiSettings } from '../types/internal'; import * as GoogleAIMapper from '../googleai-mappers'; import { BackendType } from '../public-types'; +import { ChromeAdapter } from './chrome-adapter'; -export async function countTokens( +export async function countTokensOnCloud( apiSettings: ApiSettings, model: string, params: CountTokensRequest, @@ -48,3 +49,17 @@ export async function countTokens( ); return response.json(); } + +export async function countTokens( + apiSettings: ApiSettings, + model: string, + params: CountTokensRequest, + chromeAdapter: ChromeAdapter, + requestOptions?: RequestOptions +): Promise { + if (await chromeAdapter.isAvailable(params)) { + return (await chromeAdapter.countTokens(params)).json(); + } + + return countTokensOnCloud(apiSettings, model, params, requestOptions); +} diff --git a/packages/ai/src/methods/generate-content.test.ts b/packages/ai/src/methods/generate-content.test.ts index 13250fd83dd..16a48f473ad 100644 --- a/packages/ai/src/methods/generate-content.test.ts +++ b/packages/ai/src/methods/generate-content.test.ts @@ -34,6 +34,7 @@ import { Task } from '../requests/request'; import { AIError } from '../api'; import { mapGenerateContentRequest } from '../googleai-mappers'; import { GoogleAIBackend, VertexAIBackend } from '../backend'; +import { ChromeAdapter } from './chrome-adapter'; use(sinonChai); use(chaiAsPromised); @@ -96,7 +97,8 @@ describe('generateContent()', () => { const result = await generateContent( fakeApiSettings, 'model', - fakeRequestParams + fakeRequestParams, + new ChromeAdapter() ); expect(result.response.text()).to.include('Mountain View, California'); expect(makeRequestStub).to.be.calledWith( @@ -119,7 +121,8 @@ describe('generateContent()', () => { const result = await generateContent( fakeApiSettings, 'model', - fakeRequestParams + fakeRequestParams, + new ChromeAdapter() ); expect(result.response.text()).to.include('Use Freshly Ground Coffee'); expect(result.response.text()).to.include('30 minutes of brewing'); @@ -142,7 +145,8 @@ describe('generateContent()', () => { const result = await generateContent( fakeApiSettings, 'model', - fakeRequestParams + fakeRequestParams, + new ChromeAdapter() ); expect(result.response.usageMetadata?.totalTokenCount).to.equal(1913); expect(result.response.usageMetadata?.candidatesTokenCount).to.equal(76); @@ -177,7 +181,8 @@ describe('generateContent()', () => { const result = await generateContent( fakeApiSettings, 'model', - fakeRequestParams + fakeRequestParams, + new ChromeAdapter() ); expect(result.response.text()).to.include( 'Some information cited from an external source' @@ -204,7 +209,8 @@ describe('generateContent()', () => { const result = await generateContent( fakeApiSettings, 'model', - fakeRequestParams + fakeRequestParams, + new ChromeAdapter() ); expect(result.response.text).to.throw('SAFETY'); expect(makeRequestStub).to.be.calledWith( @@ -226,7 +232,8 @@ describe('generateContent()', () => { const result = await generateContent( fakeApiSettings, 'model', - fakeRequestParams + fakeRequestParams, + new ChromeAdapter() ); expect(result.response.text).to.throw('SAFETY'); expect(makeRequestStub).to.be.calledWith( @@ -248,7 +255,8 @@ describe('generateContent()', () => { const result = await generateContent( fakeApiSettings, 'model', - fakeRequestParams + fakeRequestParams, + new ChromeAdapter() ); expect(result.response.text()).to.equal(''); expect(makeRequestStub).to.be.calledWith( @@ -270,7 +278,8 @@ describe('generateContent()', () => { const result = await generateContent( fakeApiSettings, 'model', - fakeRequestParams + fakeRequestParams, + new ChromeAdapter() ); expect(result.response.text()).to.include('Some text'); expect(makeRequestStub).to.be.calledWith( @@ -292,7 +301,12 @@ describe('generateContent()', () => { json: mockResponse.json } as Response); await expect( - generateContent(fakeApiSettings, 'model', fakeRequestParams) + generateContent( + fakeApiSettings, + 'model', + fakeRequestParams, + new ChromeAdapter() + ) ).to.be.rejectedWith(/400.*invalid argument/); expect(mockFetch).to.be.called; }); @@ -307,7 +321,12 @@ describe('generateContent()', () => { json: mockResponse.json } as Response); await expect( - generateContent(fakeApiSettings, 'model', fakeRequestParams) + generateContent( + fakeApiSettings, + 'model', + fakeRequestParams, + new ChromeAdapter() + ) ).to.be.rejectedWith( /firebasevertexai\.googleapis[\s\S]*my-project[\s\S]*api-not-enabled/ ); @@ -347,7 +366,8 @@ describe('generateContent()', () => { generateContent( fakeGoogleAIApiSettings, 'model', - requestParamsWithMethod + requestParamsWithMethod, + new ChromeAdapter() ) ).to.be.rejectedWith(AIError, AIErrorCode.UNSUPPORTED); expect(makeRequestStub).to.not.be.called; @@ -362,7 +382,8 @@ describe('generateContent()', () => { await generateContent( fakeGoogleAIApiSettings, 'model', - fakeGoogleAIRequestParams + fakeGoogleAIRequestParams, + new ChromeAdapter() ); expect(makeRequestStub).to.be.calledWith( @@ -375,4 +396,25 @@ describe('generateContent()', () => { ); }); }); + // TODO: define a similar test for generateContentStream + it('on-device', async () => { + const chromeAdapter = new ChromeAdapter(); + const isAvailableStub = stub(chromeAdapter, 'isAvailable').resolves(true); + const mockResponse = getMockResponse( + 'vertexAI', + 'unary-success-basic-reply-short.json' + ); + const generateContentStub = stub(chromeAdapter, 'generateContent').resolves( + mockResponse as Response + ); + const result = await generateContent( + fakeApiSettings, + 'model', + fakeRequestParams, + chromeAdapter + ); + expect(result.response.text()).to.include('Mountain View, California'); + expect(isAvailableStub).to.be.called; + expect(generateContentStub).to.be.calledWith(fakeRequestParams); + }); }); diff --git a/packages/ai/src/methods/generate-content.ts b/packages/ai/src/methods/generate-content.ts index 5f7902f5954..ff99b306855 100644 --- a/packages/ai/src/methods/generate-content.ts +++ b/packages/ai/src/methods/generate-content.ts @@ -28,17 +28,18 @@ import { processStream } from '../requests/stream-reader'; import { ApiSettings } from '../types/internal'; import * as GoogleAIMapper from '../googleai-mappers'; import { BackendType } from '../public-types'; +import { ChromeAdapter } from './chrome-adapter'; -export async function generateContentStream( +async function generateContentStreamOnCloud( apiSettings: ApiSettings, model: string, params: GenerateContentRequest, requestOptions?: RequestOptions -): Promise { +): Promise { if (apiSettings.backend.backendType === BackendType.GOOGLE_AI) { params = GoogleAIMapper.mapGenerateContentRequest(params); } - const response = await makeRequest( + return makeRequest( model, Task.STREAM_GENERATE_CONTENT, apiSettings, @@ -46,19 +47,39 @@ export async function generateContentStream( JSON.stringify(params), requestOptions ); +} + +export async function generateContentStream( + apiSettings: ApiSettings, + model: string, + params: GenerateContentRequest, + chromeAdapter: ChromeAdapter, + requestOptions?: RequestOptions +): Promise { + let response; + if (await chromeAdapter.isAvailable(params)) { + response = await chromeAdapter.generateContentStream(params); + } else { + response = await generateContentStreamOnCloud( + apiSettings, + model, + params, + requestOptions + ); + } return processStream(response, apiSettings); // TODO: Map streaming responses } -export async function generateContent( +async function generateContentOnCloud( apiSettings: ApiSettings, model: string, params: GenerateContentRequest, requestOptions?: RequestOptions -): Promise { +): Promise { if (apiSettings.backend.backendType === BackendType.GOOGLE_AI) { params = GoogleAIMapper.mapGenerateContentRequest(params); } - const response = await makeRequest( + return makeRequest( model, Task.GENERATE_CONTENT, apiSettings, @@ -66,6 +87,26 @@ export async function generateContent( JSON.stringify(params), requestOptions ); +} + +export async function generateContent( + apiSettings: ApiSettings, + model: string, + params: GenerateContentRequest, + chromeAdapter: ChromeAdapter, + requestOptions?: RequestOptions +): Promise { + let response; + if (await chromeAdapter.isAvailable(params)) { + response = await chromeAdapter.generateContent(params); + } else { + response = await generateContentOnCloud( + apiSettings, + model, + params, + requestOptions + ); + } const generateContentResponse = await processGenerateContentResponse( response, apiSettings diff --git a/packages/ai/src/models/generative-model.test.ts b/packages/ai/src/models/generative-model.test.ts index d055b82b1be..e3d8f7fe011 100644 --- a/packages/ai/src/models/generative-model.test.ts +++ b/packages/ai/src/models/generative-model.test.ts @@ -22,6 +22,7 @@ import { match, restore, stub } from 'sinon'; import { getMockResponse } from '../../test-utils/mock-response'; import sinonChai from 'sinon-chai'; import { VertexAIBackend } from '../backend'; +import { ChromeAdapter } from '../methods/chrome-adapter'; use(sinonChai); @@ -41,21 +42,27 @@ const fakeAI: AI = { describe('GenerativeModel', () => { it('passes params through to generateContent', async () => { - const genModel = new GenerativeModel(fakeAI, { - model: 'my-model', - tools: [ - { - functionDeclarations: [ - { - name: 'myfunc', - description: 'mydesc' - } - ] - } - ], - toolConfig: { functionCallingConfig: { mode: FunctionCallingMode.NONE } }, - systemInstruction: { role: 'system', parts: [{ text: 'be friendly' }] } - }); + const genModel = new GenerativeModel( + fakeAI, + { + model: 'my-model', + tools: [ + { + functionDeclarations: [ + { + name: 'myfunc', + description: 'mydesc' + } + ] + } + ], + toolConfig: { + functionCallingConfig: { mode: FunctionCallingMode.NONE } + }, + systemInstruction: { role: 'system', parts: [{ text: 'be friendly' }] } + }, + new ChromeAdapter() + ); expect(genModel.tools?.length).to.equal(1); expect(genModel.toolConfig?.functionCallingConfig?.mode).to.equal( FunctionCallingMode.NONE @@ -86,10 +93,14 @@ describe('GenerativeModel', () => { restore(); }); it('passes text-only systemInstruction through to generateContent', async () => { - const genModel = new GenerativeModel(fakeAI, { - model: 'my-model', - systemInstruction: 'be friendly' - }); + const genModel = new GenerativeModel( + fakeAI, + { + model: 'my-model', + systemInstruction: 'be friendly' + }, + new ChromeAdapter() + ); expect(genModel.systemInstruction?.parts[0].text).to.equal('be friendly'); const mockResponse = getMockResponse( 'vertexAI', @@ -112,21 +123,27 @@ describe('GenerativeModel', () => { restore(); }); it('generateContent overrides model values', async () => { - const genModel = new GenerativeModel(fakeAI, { - model: 'my-model', - tools: [ - { - functionDeclarations: [ - { - name: 'myfunc', - description: 'mydesc' - } - ] - } - ], - toolConfig: { functionCallingConfig: { mode: FunctionCallingMode.NONE } }, - systemInstruction: { role: 'system', parts: [{ text: 'be friendly' }] } - }); + const genModel = new GenerativeModel( + fakeAI, + { + model: 'my-model', + tools: [ + { + functionDeclarations: [ + { + name: 'myfunc', + description: 'mydesc' + } + ] + } + ], + toolConfig: { + functionCallingConfig: { mode: FunctionCallingMode.NONE } + }, + systemInstruction: { role: 'system', parts: [{ text: 'be friendly' }] } + }, + new ChromeAdapter() + ); expect(genModel.tools?.length).to.equal(1); expect(genModel.toolConfig?.functionCallingConfig?.mode).to.equal( FunctionCallingMode.NONE @@ -168,12 +185,16 @@ describe('GenerativeModel', () => { restore(); }); it('passes base model params through to ChatSession when there are no startChatParams', async () => { - const genModel = new GenerativeModel(fakeAI, { - model: 'my-model', - generationConfig: { - topK: 1 - } - }); + const genModel = new GenerativeModel( + fakeAI, + { + model: 'my-model', + generationConfig: { + topK: 1 + } + }, + new ChromeAdapter() + ); const chatSession = genModel.startChat(); expect(chatSession.params?.generationConfig).to.deep.equal({ topK: 1 @@ -181,12 +202,16 @@ describe('GenerativeModel', () => { restore(); }); it('overrides base model params with startChatParams', () => { - const genModel = new GenerativeModel(fakeAI, { - model: 'my-model', - generationConfig: { - topK: 1 - } - }); + const genModel = new GenerativeModel( + fakeAI, + { + model: 'my-model', + generationConfig: { + topK: 1 + } + }, + new ChromeAdapter() + ); const chatSession = genModel.startChat({ generationConfig: { topK: 2 @@ -197,17 +222,23 @@ describe('GenerativeModel', () => { }); }); it('passes params through to chat.sendMessage', async () => { - const genModel = new GenerativeModel(fakeAI, { - model: 'my-model', - tools: [ - { functionDeclarations: [{ name: 'myfunc', description: 'mydesc' }] } - ], - toolConfig: { functionCallingConfig: { mode: FunctionCallingMode.NONE } }, - systemInstruction: { role: 'system', parts: [{ text: 'be friendly' }] }, - generationConfig: { - topK: 1 - } - }); + const genModel = new GenerativeModel( + fakeAI, + { + model: 'my-model', + tools: [ + { functionDeclarations: [{ name: 'myfunc', description: 'mydesc' }] } + ], + toolConfig: { + functionCallingConfig: { mode: FunctionCallingMode.NONE } + }, + systemInstruction: { role: 'system', parts: [{ text: 'be friendly' }] }, + generationConfig: { + topK: 1 + } + }, + new ChromeAdapter() + ); expect(genModel.tools?.length).to.equal(1); expect(genModel.toolConfig?.functionCallingConfig?.mode).to.equal( FunctionCallingMode.NONE @@ -239,10 +270,14 @@ describe('GenerativeModel', () => { restore(); }); it('passes text-only systemInstruction through to chat.sendMessage', async () => { - const genModel = new GenerativeModel(fakeAI, { - model: 'my-model', - systemInstruction: 'be friendly' - }); + const genModel = new GenerativeModel( + fakeAI, + { + model: 'my-model', + systemInstruction: 'be friendly' + }, + new ChromeAdapter() + ); expect(genModel.systemInstruction?.parts[0].text).to.equal('be friendly'); const mockResponse = getMockResponse( 'vertexAI', @@ -265,17 +300,23 @@ describe('GenerativeModel', () => { restore(); }); it('startChat overrides model values', async () => { - const genModel = new GenerativeModel(fakeAI, { - model: 'my-model', - tools: [ - { functionDeclarations: [{ name: 'myfunc', description: 'mydesc' }] } - ], - toolConfig: { functionCallingConfig: { mode: FunctionCallingMode.NONE } }, - systemInstruction: { role: 'system', parts: [{ text: 'be friendly' }] }, - generationConfig: { - responseMimeType: 'image/jpeg' - } - }); + const genModel = new GenerativeModel( + fakeAI, + { + model: 'my-model', + tools: [ + { functionDeclarations: [{ name: 'myfunc', description: 'mydesc' }] } + ], + toolConfig: { + functionCallingConfig: { mode: FunctionCallingMode.NONE } + }, + systemInstruction: { role: 'system', parts: [{ text: 'be friendly' }] }, + generationConfig: { + responseMimeType: 'image/jpeg' + } + }, + new ChromeAdapter() + ); expect(genModel.tools?.length).to.equal(1); expect(genModel.toolConfig?.functionCallingConfig?.mode).to.equal( FunctionCallingMode.NONE @@ -325,7 +366,11 @@ describe('GenerativeModel', () => { restore(); }); it('calls countTokens', async () => { - const genModel = new GenerativeModel(fakeAI, { model: 'my-model' }); + const genModel = new GenerativeModel( + fakeAI, + { model: 'my-model' }, + new ChromeAdapter() + ); const mockResponse = getMockResponse( 'vertexAI', 'unary-success-total-tokens.json' diff --git a/packages/ai/src/models/generative-model.ts b/packages/ai/src/models/generative-model.ts index b09a9290aa4..98b662ebdb9 100644 --- a/packages/ai/src/models/generative-model.ts +++ b/packages/ai/src/models/generative-model.ts @@ -43,12 +43,17 @@ import { } from '../requests/request-helpers'; import { AI } from '../public-types'; import { AIModel } from './ai-model'; +import { ChromeAdapter } from '../methods/chrome-adapter'; /** * Class for generative model APIs. * @public */ export class GenerativeModel extends AIModel { + /** + * Defines the name of the default in-cloud model to use for hybrid inference. + */ + static DEFAULT_HYBRID_IN_CLOUD_MODEL = 'gemini-2.0-flash-lite'; generationConfig: GenerationConfig; safetySettings: SafetySetting[]; requestOptions?: RequestOptions; @@ -59,6 +64,7 @@ export class GenerativeModel extends AIModel { constructor( ai: AI, modelParams: ModelParams, + private chromeAdapter: ChromeAdapter, requestOptions?: RequestOptions ) { super(ai, modelParams.model); @@ -91,6 +97,7 @@ export class GenerativeModel extends AIModel { systemInstruction: this.systemInstruction, ...formattedParams }, + this.chromeAdapter, this.requestOptions ); } @@ -116,6 +123,7 @@ export class GenerativeModel extends AIModel { systemInstruction: this.systemInstruction, ...formattedParams }, + this.chromeAdapter, this.requestOptions ); } @@ -128,6 +136,7 @@ export class GenerativeModel extends AIModel { return new ChatSession( this._apiSettings, this.model, + this.chromeAdapter, { tools: this.tools, toolConfig: this.toolConfig, @@ -152,6 +161,11 @@ export class GenerativeModel extends AIModel { request: CountTokensRequest | string | Array ): Promise { const formattedParams = formatGenerateContentInput(request); - return countTokens(this._apiSettings, this.model, formattedParams); + return countTokens( + this._apiSettings, + this.model, + formattedParams, + this.chromeAdapter + ); } } diff --git a/packages/ai/src/types/language-model.ts b/packages/ai/src/types/language-model.ts new file mode 100644 index 00000000000..22916e7ff96 --- /dev/null +++ b/packages/ai/src/types/language-model.ts @@ -0,0 +1,83 @@ +/** + * @license + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export interface LanguageModel extends EventTarget { + create(options?: LanguageModelCreateOptions): Promise; + availability(options?: LanguageModelCreateCoreOptions): Promise; + prompt( + input: LanguageModelPrompt, + options?: LanguageModelPromptOptions + ): Promise; + promptStreaming( + input: LanguageModelPrompt, + options?: LanguageModelPromptOptions + ): ReadableStream; + measureInputUsage( + input: LanguageModelPrompt, + options?: LanguageModelPromptOptions + ): Promise; + destroy(): undefined; +} +export enum Availability { + 'unavailable' = 'unavailable', + 'downloadable' = 'downloadable', + 'downloading' = 'downloading', + 'available' = 'available' +} +export interface LanguageModelCreateCoreOptions { + topK?: number; + temperature?: number; + expectedInputs?: LanguageModelExpectedInput[]; +} +export interface LanguageModelCreateOptions + extends LanguageModelCreateCoreOptions { + signal?: AbortSignal; + systemPrompt?: string; + initialPrompts?: LanguageModelInitialPrompts; +} +export interface LanguageModelPromptOptions { + responseConstraint?: object; + // TODO: Restore AbortSignal once the API is defined. +} +interface LanguageModelExpectedInput { + type: LanguageModelMessageType; + languages?: string[]; +} +// TODO: revert to type from Prompt API explainer once it's supported. +export type LanguageModelPrompt = LanguageModelMessageContent[]; +type LanguageModelInitialPrompts = + | LanguageModelMessage[] + | LanguageModelMessageShorthand[]; +interface LanguageModelMessage { + role: LanguageModelMessageRole; + content: LanguageModelMessageContent[]; +} +interface LanguageModelMessageShorthand { + role: LanguageModelMessageRole; + content: string; +} +export interface LanguageModelMessageContent { + type: LanguageModelMessageType; + content: LanguageModelMessageContentValue; +} +type LanguageModelMessageRole = 'system' | 'user' | 'assistant'; +type LanguageModelMessageType = 'text' | 'image' | 'audio'; +type LanguageModelMessageContentValue = + | ImageBitmapSource + | AudioBuffer + | BufferSource + | string; diff --git a/packages/ai/src/types/requests.ts b/packages/ai/src/types/requests.ts index 67f45095c2a..f7d0cc558b9 100644 --- a/packages/ai/src/types/requests.ts +++ b/packages/ai/src/types/requests.ts @@ -17,6 +17,10 @@ import { TypedSchema } from '../requests/schema-builder'; import { Content, Part } from './content'; +import { + LanguageModelCreateOptions, + LanguageModelPromptOptions +} from './language-model'; import { FunctionCallingMode, HarmBlockMethod, @@ -231,3 +235,37 @@ export interface FunctionCallingConfig { mode?: FunctionCallingMode; allowedFunctionNames?: string[]; } + +/** + * Encapsulates configuration for on-device inference. + */ +export interface OnDeviceParams { + createOptions?: LanguageModelCreateOptions; + promptOptions?: LanguageModelPromptOptions; +} + +/** + * Toggles hybrid inference. + */ +export interface HybridParams { + /** + * Specifies on-device or in-cloud inference. Defaults to prefer on-device. + */ + mode: InferenceMode; + /** + * Optional. Specifies advanced params for on-device inference. + */ + onDeviceParams?: OnDeviceParams; + /** + * Optional. Specifies advanced params for in-cloud inference. + */ + inCloudParams?: ModelParams; +} + +/** + * Determines whether inference happens on-device or in-cloud. + */ +export type InferenceMode = + | 'prefer_on_device' + | 'only_on_device' + | 'only_in_cloud'; From c16cbf1a31416216c6067a39d4bfc6770c731fa0 Mon Sep 17 00:00:00 2001 From: Siddharth Gupta Date: Tue, 13 May 2025 09:15:08 -0700 Subject: [PATCH 02/20] Adding docs --- common/api-review/ai.api.md | 30 ++++++++++++++-- docs-devsite/_toc.yaml | 4 +++ docs-devsite/ai.chatsession.md | 5 +-- docs-devsite/ai.generativemodel.md | 16 +++++++-- docs-devsite/ai.hybridparams.md | 57 ++++++++++++++++++++++++++++++ docs-devsite/ai.md | 25 +++++++++---- docs-devsite/ai.modelparams.md | 2 +- docs-devsite/ai.ondeviceparams.md | 42 ++++++++++++++++++++++ docs-devsite/ai.requestoptions.md | 2 +- 9 files changed, 168 insertions(+), 15 deletions(-) create mode 100644 docs-devsite/ai.hybridparams.md create mode 100644 docs-devsite/ai.ondeviceparams.md diff --git a/common/api-review/ai.api.md b/common/api-review/ai.api.md index d096d4c27f6..a603a531358 100644 --- a/common/api-review/ai.api.md +++ b/common/api-review/ai.api.md @@ -112,7 +112,8 @@ export class BooleanSchema extends Schema { // @public export class ChatSession { - constructor(apiSettings: ApiSettings, model: string, params?: StartChatParams | undefined, requestOptions?: RequestOptions | undefined); + // Warning: (ae-forgotten-export) The symbol "ChromeAdapter" needs to be exported by the entry point index.d.ts + constructor(apiSettings: ApiSettings, model: string, chromeAdapter: ChromeAdapter, params?: StartChatParams | undefined, requestOptions?: RequestOptions | undefined); getHistory(): Promise; // (undocumented) model: string; @@ -395,8 +396,9 @@ export interface GenerativeContentBlob { // @public export class GenerativeModel extends AIModel { - constructor(ai: AI, modelParams: ModelParams, requestOptions?: RequestOptions); + constructor(ai: AI, modelParams: ModelParams, chromeAdapter: ChromeAdapter, requestOptions?: RequestOptions); countTokens(request: CountTokensRequest | string | Array): Promise; + static DEFAULT_HYBRID_IN_CLOUD_MODEL: string; generateContent(request: GenerateContentRequest | string | Array): Promise; generateContentStream(request: GenerateContentRequest | string | Array): Promise; // (undocumented) @@ -418,7 +420,7 @@ export class GenerativeModel extends AIModel { export function getAI(app?: FirebaseApp, options?: AIOptions): AI; // @public -export function getGenerativeModel(ai: AI, modelParams: ModelParams, requestOptions?: RequestOptions): GenerativeModel; +export function getGenerativeModel(ai: AI, modelParams: ModelParams | HybridParams, requestOptions?: RequestOptions): GenerativeModel; // @beta export function getImagenModel(ai: AI, modelParams: ImagenModelParams, requestOptions?: RequestOptions): ImagenModel; @@ -550,6 +552,13 @@ export enum HarmSeverity { HARM_SEVERITY_UNSUPPORTED = "HARM_SEVERITY_UNSUPPORTED" } +// @public +export interface HybridParams { + inCloudParams?: ModelParams; + mode: InferenceMode; + onDeviceParams?: OnDeviceParams; +} + // @beta export enum ImagenAspectRatio { LANDSCAPE_16x9 = "16:9", @@ -634,6 +643,9 @@ export interface ImagenSafetySettings { safetyFilterLevel?: ImagenSafetyFilterLevel; } +// @public +export type InferenceMode = 'prefer_on_device' | 'only_on_device' | 'only_in_cloud'; + // @public export interface InlineDataPart { // (undocumented) @@ -708,6 +720,18 @@ export interface ObjectSchemaInterface extends SchemaInterface { type: SchemaType.OBJECT; } +// @public +export interface OnDeviceParams { + // Warning: (ae-forgotten-export) The symbol "LanguageModelCreateOptions" needs to be exported by the entry point index.d.ts + // + // (undocumented) + createOptions?: LanguageModelCreateOptions; + // Warning: (ae-forgotten-export) The symbol "LanguageModelPromptOptions" needs to be exported by the entry point index.d.ts + // + // (undocumented) + promptOptions?: LanguageModelPromptOptions; +} + // @public export type Part = TextPart | InlineDataPart | FunctionCallPart | FunctionResponsePart | FileDataPart; diff --git a/docs-devsite/_toc.yaml b/docs-devsite/_toc.yaml index b77a6b5910e..4076c443ddc 100644 --- a/docs-devsite/_toc.yaml +++ b/docs-devsite/_toc.yaml @@ -80,6 +80,8 @@ toc: path: /docs/reference/js/ai.groundingattribution.md - title: GroundingMetadata path: /docs/reference/js/ai.groundingmetadata.md + - title: HybridParams + path: /docs/reference/js/ai.hybridparams.md - title: ImagenGCSImage path: /docs/reference/js/ai.imagengcsimage.md - title: ImagenGenerationConfig @@ -110,6 +112,8 @@ toc: path: /docs/reference/js/ai.objectschema.md - title: ObjectSchemaInterface path: /docs/reference/js/ai.objectschemainterface.md + - title: OnDeviceParams + path: /docs/reference/js/ai.ondeviceparams.md - title: PromptFeedback path: /docs/reference/js/ai.promptfeedback.md - title: RequestOptions diff --git a/docs-devsite/ai.chatsession.md b/docs-devsite/ai.chatsession.md index 1d6e403b6a8..610fb2274dd 100644 --- a/docs-devsite/ai.chatsession.md +++ b/docs-devsite/ai.chatsession.md @@ -22,7 +22,7 @@ export declare class ChatSession | Constructor | Modifiers | Description | | --- | --- | --- | -| [(constructor)(apiSettings, model, params, requestOptions)](./ai.chatsession.md#chatsessionconstructor) | | Constructs a new instance of the ChatSession class | +| [(constructor)(apiSettings, model, chromeAdapter, params, requestOptions)](./ai.chatsession.md#chatsessionconstructor) | | Constructs a new instance of the ChatSession class | ## Properties @@ -47,7 +47,7 @@ Constructs a new instance of the `ChatSession` class Signature: ```typescript -constructor(apiSettings: ApiSettings, model: string, params?: StartChatParams | undefined, requestOptions?: RequestOptions | undefined); +constructor(apiSettings: ApiSettings, model: string, chromeAdapter: ChromeAdapter, params?: StartChatParams | undefined, requestOptions?: RequestOptions | undefined); ``` #### Parameters @@ -56,6 +56,7 @@ constructor(apiSettings: ApiSettings, model: string, params?: StartChatParams | | --- | --- | --- | | apiSettings | ApiSettings | | | model | string | | +| chromeAdapter | ChromeAdapter | | | params | [StartChatParams](./ai.startchatparams.md#startchatparams_interface) \| undefined | | | requestOptions | [RequestOptions](./ai.requestoptions.md#requestoptions_interface) \| undefined | | diff --git a/docs-devsite/ai.generativemodel.md b/docs-devsite/ai.generativemodel.md index d91cf80e881..17c9d3c0863 100644 --- a/docs-devsite/ai.generativemodel.md +++ b/docs-devsite/ai.generativemodel.md @@ -23,12 +23,13 @@ export declare class GenerativeModel extends AIModel | Constructor | Modifiers | Description | | --- | --- | --- | -| [(constructor)(ai, modelParams, requestOptions)](./ai.generativemodel.md#generativemodelconstructor) | | Constructs a new instance of the GenerativeModel class | +| [(constructor)(ai, modelParams, chromeAdapter, requestOptions)](./ai.generativemodel.md#generativemodelconstructor) | | Constructs a new instance of the GenerativeModel class | ## Properties | Property | Modifiers | Type | Description | | --- | --- | --- | --- | +| [DEFAULT\_HYBRID\_IN\_CLOUD\_MODEL](./ai.generativemodel.md#generativemodeldefault_hybrid_in_cloud_model) | static | string | Defines the name of the default in-cloud model to use for hybrid inference. | | [generationConfig](./ai.generativemodel.md#generativemodelgenerationconfig) | | [GenerationConfig](./ai.generationconfig.md#generationconfig_interface) | | | [requestOptions](./ai.generativemodel.md#generativemodelrequestoptions) | | [RequestOptions](./ai.requestoptions.md#requestoptions_interface) | | | [safetySettings](./ai.generativemodel.md#generativemodelsafetysettings) | | [SafetySetting](./ai.safetysetting.md#safetysetting_interface)\[\] | | @@ -52,7 +53,7 @@ Constructs a new instance of the `GenerativeModel` class Signature: ```typescript -constructor(ai: AI, modelParams: ModelParams, requestOptions?: RequestOptions); +constructor(ai: AI, modelParams: ModelParams, chromeAdapter: ChromeAdapter, requestOptions?: RequestOptions); ``` #### Parameters @@ -61,8 +62,19 @@ constructor(ai: AI, modelParams: ModelParams, requestOptions?: RequestOptions); | --- | --- | --- | | ai | [AI](./ai.ai.md#ai_interface) | | | modelParams | [ModelParams](./ai.modelparams.md#modelparams_interface) | | +| chromeAdapter | ChromeAdapter | | | requestOptions | [RequestOptions](./ai.requestoptions.md#requestoptions_interface) | | +## GenerativeModel.DEFAULT\_HYBRID\_IN\_CLOUD\_MODEL + +Defines the name of the default in-cloud model to use for hybrid inference. + +Signature: + +```typescript +static DEFAULT_HYBRID_IN_CLOUD_MODEL: string; +``` + ## GenerativeModel.generationConfig Signature: diff --git a/docs-devsite/ai.hybridparams.md b/docs-devsite/ai.hybridparams.md new file mode 100644 index 00000000000..b2b3b1030fe --- /dev/null +++ b/docs-devsite/ai.hybridparams.md @@ -0,0 +1,57 @@ +Project: /docs/reference/js/_project.yaml +Book: /docs/reference/_book.yaml +page_type: reference + +{% comment %} +DO NOT EDIT THIS FILE! +This is generated by the JS SDK team, and any local changes will be +overwritten. Changes should be made in the source code at +https://github.com/firebase/firebase-js-sdk +{% endcomment %} + +# HybridParams interface +Toggles hybrid inference. + +Signature: + +```typescript +export interface HybridParams +``` + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [inCloudParams](./ai.hybridparams.md#hybridparamsincloudparams) | [ModelParams](./ai.modelparams.md#modelparams_interface) | Optional. Specifies advanced params for in-cloud inference. | +| [mode](./ai.hybridparams.md#hybridparamsmode) | [InferenceMode](./ai.md#inferencemode) | Specifies on-device or in-cloud inference. Defaults to prefer on-device. | +| [onDeviceParams](./ai.hybridparams.md#hybridparamsondeviceparams) | [OnDeviceParams](./ai.ondeviceparams.md#ondeviceparams_interface) | Optional. Specifies advanced params for on-device inference. | + +## HybridParams.inCloudParams + +Optional. Specifies advanced params for in-cloud inference. + +Signature: + +```typescript +inCloudParams?: ModelParams; +``` + +## HybridParams.mode + +Specifies on-device or in-cloud inference. Defaults to prefer on-device. + +Signature: + +```typescript +mode: InferenceMode; +``` + +## HybridParams.onDeviceParams + +Optional. Specifies advanced params for on-device inference. + +Signature: + +```typescript +onDeviceParams?: OnDeviceParams; +``` diff --git a/docs-devsite/ai.md b/docs-devsite/ai.md index c43c0391ba4..01b3a455682 100644 --- a/docs-devsite/ai.md +++ b/docs-devsite/ai.md @@ -20,7 +20,7 @@ The Firebase AI Web SDK. | [getAI(app, options)](./ai.md#getai_a94a413) | Returns the default [AI](./ai.ai.md#ai_interface) instance that is associated with the provided [FirebaseApp](./app.firebaseapp.md#firebaseapp_interface). If no instance exists, initializes a new instance with the default settings. | | [getVertexAI(app, options)](./ai.md#getvertexai_04094cf) | | | function(ai, ...) | -| [getGenerativeModel(ai, modelParams, requestOptions)](./ai.md#getgenerativemodel_80bd839) | Returns a [GenerativeModel](./ai.generativemodel.md#generativemodel_class) class with methods for inference and other functionality. | +| [getGenerativeModel(ai, modelParams, requestOptions)](./ai.md#getgenerativemodel_c63f46a) | Returns a [GenerativeModel](./ai.generativemodel.md#generativemodel_class) class with methods for inference and other functionality. | | [getImagenModel(ai, modelParams, requestOptions)](./ai.md#getimagenmodel_e1f6645) | (Public Preview) Returns an [ImagenModel](./ai.imagenmodel.md#imagenmodel_class) class with methods for using Imagen.Only Imagen 3 models (named imagen-3.0-*) are supported. | ## Classes @@ -97,6 +97,7 @@ The Firebase AI Web SDK. | [GenerativeContentBlob](./ai.generativecontentblob.md#generativecontentblob_interface) | Interface for sending an image. | | [GroundingAttribution](./ai.groundingattribution.md#groundingattribution_interface) | | | [GroundingMetadata](./ai.groundingmetadata.md#groundingmetadata_interface) | Metadata returned to client when grounding is enabled. | +| [HybridParams](./ai.hybridparams.md#hybridparams_interface) | Toggles hybrid inference. | | [ImagenGCSImage](./ai.imagengcsimage.md#imagengcsimage_interface) | An image generated by Imagen, stored in a Cloud Storage for Firebase bucket.This feature is not available yet. | | [ImagenGenerationConfig](./ai.imagengenerationconfig.md#imagengenerationconfig_interface) | (Public Preview) Configuration options for generating images with Imagen.See the [documentation](http://firebase.google.com/docs/vertex-ai/generate-images-imagen) for more details. | | [ImagenGenerationResponse](./ai.imagengenerationresponse.md#imagengenerationresponse_interface) | (Public Preview) The response from a request to generate images with Imagen. | @@ -105,10 +106,11 @@ The Firebase AI Web SDK. | [ImagenSafetySettings](./ai.imagensafetysettings.md#imagensafetysettings_interface) | (Public Preview) Settings for controlling the aggressiveness of filtering out sensitive content.See the [documentation](http://firebase.google.com/docs/vertex-ai/generate-images) for more details. | | [InlineDataPart](./ai.inlinedatapart.md#inlinedatapart_interface) | Content part interface if the part represents an image. | | [ModalityTokenCount](./ai.modalitytokencount.md#modalitytokencount_interface) | Represents token counting info for a single modality. | -| [ModelParams](./ai.modelparams.md#modelparams_interface) | Params passed to [getGenerativeModel()](./ai.md#getgenerativemodel_80bd839). | +| [ModelParams](./ai.modelparams.md#modelparams_interface) | Params passed to [getGenerativeModel()](./ai.md#getgenerativemodel_c63f46a). | | [ObjectSchemaInterface](./ai.objectschemainterface.md#objectschemainterface_interface) | Interface for [ObjectSchema](./ai.objectschema.md#objectschema_class) class. | +| [OnDeviceParams](./ai.ondeviceparams.md#ondeviceparams_interface) | Encapsulates configuration for on-device inference. | | [PromptFeedback](./ai.promptfeedback.md#promptfeedback_interface) | If the prompt was blocked, this will be populated with blockReason and the relevant safetyRatings. | -| [RequestOptions](./ai.requestoptions.md#requestoptions_interface) | Params passed to [getGenerativeModel()](./ai.md#getgenerativemodel_80bd839). | +| [RequestOptions](./ai.requestoptions.md#requestoptions_interface) | Params passed to [getGenerativeModel()](./ai.md#getgenerativemodel_c63f46a). | | [RetrievedContextAttribution](./ai.retrievedcontextattribution.md#retrievedcontextattribution_interface) | | | [SafetyRating](./ai.safetyrating.md#safetyrating_interface) | A safety rating associated with a [GenerateContentCandidate](./ai.generatecontentcandidate.md#generatecontentcandidate_interface) | | [SafetySetting](./ai.safetysetting.md#safetysetting_interface) | Safety setting that can be sent as part of request parameters. | @@ -140,6 +142,7 @@ The Firebase AI Web SDK. | Type Alias | Description | | --- | --- | | [BackendType](./ai.md#backendtype) | Type alias representing valid backend types. It can be either 'VERTEX_AI' or 'GOOGLE_AI'. | +| [InferenceMode](./ai.md#inferencemode) | Determines whether inference happens on-device or in-cloud. | | [Part](./ai.md#part) | Content part - includes text, image/video, or function call/response part types. | | [ResponseModality](./ai.md#responsemodality) | (Public Preview) Generation modalities to be returned in generation responses. | | [Role](./ai.md#role) | Role is the producer of the content. | @@ -226,14 +229,14 @@ export declare function getVertexAI(app?: FirebaseApp, options?: VertexAIOptions ## function(ai, ...) -### getGenerativeModel(ai, modelParams, requestOptions) {:#getgenerativemodel_80bd839} +### getGenerativeModel(ai, modelParams, requestOptions) {:#getgenerativemodel_c63f46a} Returns a [GenerativeModel](./ai.generativemodel.md#generativemodel_class) class with methods for inference and other functionality. Signature: ```typescript -export declare function getGenerativeModel(ai: AI, modelParams: ModelParams, requestOptions?: RequestOptions): GenerativeModel; +export declare function getGenerativeModel(ai: AI, modelParams: ModelParams | HybridParams, requestOptions?: RequestOptions): GenerativeModel; ``` #### Parameters @@ -241,7 +244,7 @@ export declare function getGenerativeModel(ai: AI, modelParams: ModelParams, req | Parameter | Type | Description | | --- | --- | --- | | ai | [AI](./ai.ai.md#ai_interface) | | -| modelParams | [ModelParams](./ai.modelparams.md#modelparams_interface) | | +| modelParams | [ModelParams](./ai.modelparams.md#modelparams_interface) \| [HybridParams](./ai.hybridparams.md#hybridparams_interface) | | | requestOptions | [RequestOptions](./ai.requestoptions.md#requestoptions_interface) | | Returns: @@ -360,6 +363,16 @@ Type alias representing valid backend types. It can be either `'VERTEX_AI'` or ` export type BackendType = (typeof BackendType)[keyof typeof BackendType]; ``` +## InferenceMode + +Determines whether inference happens on-device or in-cloud. + +Signature: + +```typescript +export type InferenceMode = 'prefer_on_device' | 'only_on_device' | 'only_in_cloud'; +``` + ## Part Content part - includes text, image/video, or function call/response part types. diff --git a/docs-devsite/ai.modelparams.md b/docs-devsite/ai.modelparams.md index a92b2e9035d..a5722e7d69d 100644 --- a/docs-devsite/ai.modelparams.md +++ b/docs-devsite/ai.modelparams.md @@ -10,7 +10,7 @@ https://github.com/firebase/firebase-js-sdk {% endcomment %} # ModelParams interface -Params passed to [getGenerativeModel()](./ai.md#getgenerativemodel_80bd839). +Params passed to [getGenerativeModel()](./ai.md#getgenerativemodel_c63f46a). Signature: diff --git a/docs-devsite/ai.ondeviceparams.md b/docs-devsite/ai.ondeviceparams.md new file mode 100644 index 00000000000..f4bfcbb5cff --- /dev/null +++ b/docs-devsite/ai.ondeviceparams.md @@ -0,0 +1,42 @@ +Project: /docs/reference/js/_project.yaml +Book: /docs/reference/_book.yaml +page_type: reference + +{% comment %} +DO NOT EDIT THIS FILE! +This is generated by the JS SDK team, and any local changes will be +overwritten. Changes should be made in the source code at +https://github.com/firebase/firebase-js-sdk +{% endcomment %} + +# OnDeviceParams interface +Encapsulates configuration for on-device inference. + +Signature: + +```typescript +export interface OnDeviceParams +``` + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [createOptions](./ai.ondeviceparams.md#ondeviceparamscreateoptions) | LanguageModelCreateOptions | | +| [promptOptions](./ai.ondeviceparams.md#ondeviceparamspromptoptions) | LanguageModelPromptOptions | | + +## OnDeviceParams.createOptions + +Signature: + +```typescript +createOptions?: LanguageModelCreateOptions; +``` + +## OnDeviceParams.promptOptions + +Signature: + +```typescript +promptOptions?: LanguageModelPromptOptions; +``` diff --git a/docs-devsite/ai.requestoptions.md b/docs-devsite/ai.requestoptions.md index 73aa03c1d25..8178ef5b696 100644 --- a/docs-devsite/ai.requestoptions.md +++ b/docs-devsite/ai.requestoptions.md @@ -10,7 +10,7 @@ https://github.com/firebase/firebase-js-sdk {% endcomment %} # RequestOptions interface -Params passed to [getGenerativeModel()](./ai.md#getgenerativemodel_80bd839). +Params passed to [getGenerativeModel()](./ai.md#getgenerativemodel_c63f46a). Signature: From 50f142a9805d316c9606f0520351e910aadefd83 Mon Sep 17 00:00:00 2001 From: Erik Eldridge Date: Wed, 14 May 2025 13:32:50 -0700 Subject: [PATCH 03/20] VinF Hybrid Inference: Document exported LanguageModel types (#9035) --- common/api-review/ai.api.md | 66 ++++++++++++++++++- docs-devsite/_toc.yaml | 12 ++++ .../ai.languagemodelcreatecoreoptions.md | 49 ++++++++++++++ docs-devsite/ai.languagemodelcreateoptions.md | 50 ++++++++++++++ docs-devsite/ai.languagemodelexpectedinput.md | 40 +++++++++++ docs-devsite/ai.languagemodelmessage.md | 40 +++++++++++ .../ai.languagemodelmessagecontent.md | 40 +++++++++++ .../ai.languagemodelmessageshorthand.md | 40 +++++++++++ docs-devsite/ai.md | 42 ++++++++++++ docs-devsite/ai.ondeviceparams.md | 2 +- packages/ai/src/methods/chrome-adapter.ts | 4 +- packages/ai/src/types/index.ts | 12 ++++ packages/ai/src/types/language-model.ts | 14 ++-- 13 files changed, 399 insertions(+), 12 deletions(-) create mode 100644 docs-devsite/ai.languagemodelcreatecoreoptions.md create mode 100644 docs-devsite/ai.languagemodelcreateoptions.md create mode 100644 docs-devsite/ai.languagemodelexpectedinput.md create mode 100644 docs-devsite/ai.languagemodelmessage.md create mode 100644 docs-devsite/ai.languagemodelmessagecontent.md create mode 100644 docs-devsite/ai.languagemodelmessageshorthand.md diff --git a/common/api-review/ai.api.md b/common/api-review/ai.api.md index a603a531358..a7da6210ada 100644 --- a/common/api-review/ai.api.md +++ b/common/api-review/ai.api.md @@ -664,6 +664,70 @@ export class IntegerSchema extends Schema { constructor(schemaParams?: SchemaParams); } +// @public (undocumented) +export interface LanguageModelCreateCoreOptions { + // (undocumented) + expectedInputs?: LanguageModelExpectedInput[]; + // (undocumented) + temperature?: number; + // (undocumented) + topK?: number; +} + +// @public (undocumented) +export interface LanguageModelCreateOptions extends LanguageModelCreateCoreOptions { + // (undocumented) + initialPrompts?: LanguageModelInitialPrompts; + // (undocumented) + signal?: AbortSignal; + // (undocumented) + systemPrompt?: string; +} + +// @public (undocumented) +export interface LanguageModelExpectedInput { + // (undocumented) + languages?: string[]; + // (undocumented) + type: LanguageModelMessageType; +} + +// @public (undocumented) +export type LanguageModelInitialPrompts = LanguageModelMessage[] | LanguageModelMessageShorthand[]; + +// @public (undocumented) +export interface LanguageModelMessage { + // (undocumented) + content: LanguageModelMessageContent[]; + // (undocumented) + role: LanguageModelMessageRole; +} + +// @public (undocumented) +export interface LanguageModelMessageContent { + // (undocumented) + content: LanguageModelMessageContentValue; + // (undocumented) + type: LanguageModelMessageType; +} + +// @public (undocumented) +export type LanguageModelMessageContentValue = ImageBitmapSource | AudioBuffer | BufferSource | string; + +// @public (undocumented) +export type LanguageModelMessageRole = 'system' | 'user' | 'assistant'; + +// @public (undocumented) +export interface LanguageModelMessageShorthand { + // (undocumented) + content: string; + // (undocumented) + role: LanguageModelMessageRole; +} + +// @public (undocumented) +export type LanguageModelMessageType = 'text' | 'image' | 'audio'; + // @public export enum Modality { AUDIO = "AUDIO", @@ -722,8 +786,6 @@ export interface ObjectSchemaInterface extends SchemaInterface { // @public export interface OnDeviceParams { - // Warning: (ae-forgotten-export) The symbol "LanguageModelCreateOptions" needs to be exported by the entry point index.d.ts - // // (undocumented) createOptions?: LanguageModelCreateOptions; // Warning: (ae-forgotten-export) The symbol "LanguageModelPromptOptions" needs to be exported by the entry point index.d.ts diff --git a/docs-devsite/_toc.yaml b/docs-devsite/_toc.yaml index 4076c443ddc..e8359727cda 100644 --- a/docs-devsite/_toc.yaml +++ b/docs-devsite/_toc.yaml @@ -102,6 +102,18 @@ toc: path: /docs/reference/js/ai.inlinedatapart.md - title: IntegerSchema path: /docs/reference/js/ai.integerschema.md + - title: LanguageModelCreateCoreOptions + path: /docs/reference/js/ai.languagemodelcreatecoreoptions.md + - title: LanguageModelCreateOptions + path: /docs/reference/js/ai.languagemodelcreateoptions.md + - title: LanguageModelExpectedInput + path: /docs/reference/js/ai.languagemodelexpectedinput.md + - title: LanguageModelMessage + path: /docs/reference/js/ai.languagemodelmessage.md + - title: LanguageModelMessageContent + path: /docs/reference/js/ai.languagemodelmessagecontent.md + - title: LanguageModelMessageShorthand + path: /docs/reference/js/ai.languagemodelmessageshorthand.md - title: ModalityTokenCount path: /docs/reference/js/ai.modalitytokencount.md - title: ModelParams diff --git a/docs-devsite/ai.languagemodelcreatecoreoptions.md b/docs-devsite/ai.languagemodelcreatecoreoptions.md new file mode 100644 index 00000000000..2c9f61b149f --- /dev/null +++ b/docs-devsite/ai.languagemodelcreatecoreoptions.md @@ -0,0 +1,49 @@ +Project: /docs/reference/js/_project.yaml +Book: /docs/reference/_book.yaml +page_type: reference + +{% comment %} +DO NOT EDIT THIS FILE! +This is generated by the JS SDK team, and any local changes will be +overwritten. Changes should be made in the source code at +https://github.com/firebase/firebase-js-sdk +{% endcomment %} + +# LanguageModelCreateCoreOptions interface +Signature: + +```typescript +export interface LanguageModelCreateCoreOptions +``` + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [expectedInputs](./ai.languagemodelcreatecoreoptions.md#languagemodelcreatecoreoptionsexpectedinputs) | [LanguageModelExpectedInput](./ai.languagemodelexpectedinput.md#languagemodelexpectedinput_interface)\[\] | | +| [temperature](./ai.languagemodelcreatecoreoptions.md#languagemodelcreatecoreoptionstemperature) | number | | +| [topK](./ai.languagemodelcreatecoreoptions.md#languagemodelcreatecoreoptionstopk) | number | | + +## LanguageModelCreateCoreOptions.expectedInputs + +Signature: + +```typescript +expectedInputs?: LanguageModelExpectedInput[]; +``` + +## LanguageModelCreateCoreOptions.temperature + +Signature: + +```typescript +temperature?: number; +``` + +## LanguageModelCreateCoreOptions.topK + +Signature: + +```typescript +topK?: number; +``` diff --git a/docs-devsite/ai.languagemodelcreateoptions.md b/docs-devsite/ai.languagemodelcreateoptions.md new file mode 100644 index 00000000000..44edcf7e221 --- /dev/null +++ b/docs-devsite/ai.languagemodelcreateoptions.md @@ -0,0 +1,50 @@ +Project: /docs/reference/js/_project.yaml +Book: /docs/reference/_book.yaml +page_type: reference + +{% comment %} +DO NOT EDIT THIS FILE! +This is generated by the JS SDK team, and any local changes will be +overwritten. Changes should be made in the source code at +https://github.com/firebase/firebase-js-sdk +{% endcomment %} + +# LanguageModelCreateOptions interface +Signature: + +```typescript +export interface LanguageModelCreateOptions extends LanguageModelCreateCoreOptions +``` +Extends: [LanguageModelCreateCoreOptions](./ai.languagemodelcreatecoreoptions.md#languagemodelcreatecoreoptions_interface) + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [initialPrompts](./ai.languagemodelcreateoptions.md#languagemodelcreateoptionsinitialprompts) | [LanguageModelInitialPrompts](./ai.md#languagemodelinitialprompts) | | +| [signal](./ai.languagemodelcreateoptions.md#languagemodelcreateoptionssignal) | AbortSignal | | +| [systemPrompt](./ai.languagemodelcreateoptions.md#languagemodelcreateoptionssystemprompt) | string | | + +## LanguageModelCreateOptions.initialPrompts + +Signature: + +```typescript +initialPrompts?: LanguageModelInitialPrompts; +``` + +## LanguageModelCreateOptions.signal + +Signature: + +```typescript +signal?: AbortSignal; +``` + +## LanguageModelCreateOptions.systemPrompt + +Signature: + +```typescript +systemPrompt?: string; +``` diff --git a/docs-devsite/ai.languagemodelexpectedinput.md b/docs-devsite/ai.languagemodelexpectedinput.md new file mode 100644 index 00000000000..d6cbe028fc1 --- /dev/null +++ b/docs-devsite/ai.languagemodelexpectedinput.md @@ -0,0 +1,40 @@ +Project: /docs/reference/js/_project.yaml +Book: /docs/reference/_book.yaml +page_type: reference + +{% comment %} +DO NOT EDIT THIS FILE! +This is generated by the JS SDK team, and any local changes will be +overwritten. Changes should be made in the source code at +https://github.com/firebase/firebase-js-sdk +{% endcomment %} + +# LanguageModelExpectedInput interface +Signature: + +```typescript +export interface LanguageModelExpectedInput +``` + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [languages](./ai.languagemodelexpectedinput.md#languagemodelexpectedinputlanguages) | string\[\] | | +| [type](./ai.languagemodelexpectedinput.md#languagemodelexpectedinputtype) | [LanguageModelMessageType](./ai.md#languagemodelmessagetype) | | + +## LanguageModelExpectedInput.languages + +Signature: + +```typescript +languages?: string[]; +``` + +## LanguageModelExpectedInput.type + +Signature: + +```typescript +type: LanguageModelMessageType; +``` diff --git a/docs-devsite/ai.languagemodelmessage.md b/docs-devsite/ai.languagemodelmessage.md new file mode 100644 index 00000000000..420059e4892 --- /dev/null +++ b/docs-devsite/ai.languagemodelmessage.md @@ -0,0 +1,40 @@ +Project: /docs/reference/js/_project.yaml +Book: /docs/reference/_book.yaml +page_type: reference + +{% comment %} +DO NOT EDIT THIS FILE! +This is generated by the JS SDK team, and any local changes will be +overwritten. Changes should be made in the source code at +https://github.com/firebase/firebase-js-sdk +{% endcomment %} + +# LanguageModelMessage interface +Signature: + +```typescript +export interface LanguageModelMessage +``` + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [content](./ai.languagemodelmessage.md#languagemodelmessagecontent) | [LanguageModelMessageContent](./ai.languagemodelmessagecontent.md#languagemodelmessagecontent_interface)\[\] | | +| [role](./ai.languagemodelmessage.md#languagemodelmessagerole) | [LanguageModelMessageRole](./ai.md#languagemodelmessagerole) | | + +## LanguageModelMessage.content + +Signature: + +```typescript +content: LanguageModelMessageContent[]; +``` + +## LanguageModelMessage.role + +Signature: + +```typescript +role: LanguageModelMessageRole; +``` diff --git a/docs-devsite/ai.languagemodelmessagecontent.md b/docs-devsite/ai.languagemodelmessagecontent.md new file mode 100644 index 00000000000..06830ace272 --- /dev/null +++ b/docs-devsite/ai.languagemodelmessagecontent.md @@ -0,0 +1,40 @@ +Project: /docs/reference/js/_project.yaml +Book: /docs/reference/_book.yaml +page_type: reference + +{% comment %} +DO NOT EDIT THIS FILE! +This is generated by the JS SDK team, and any local changes will be +overwritten. Changes should be made in the source code at +https://github.com/firebase/firebase-js-sdk +{% endcomment %} + +# LanguageModelMessageContent interface +Signature: + +```typescript +export interface LanguageModelMessageContent +``` + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [content](./ai.languagemodelmessagecontent.md#languagemodelmessagecontentcontent) | [LanguageModelMessageContentValue](./ai.md#languagemodelmessagecontentvalue) | | +| [type](./ai.languagemodelmessagecontent.md#languagemodelmessagecontenttype) | [LanguageModelMessageType](./ai.md#languagemodelmessagetype) | | + +## LanguageModelMessageContent.content + +Signature: + +```typescript +content: LanguageModelMessageContentValue; +``` + +## LanguageModelMessageContent.type + +Signature: + +```typescript +type: LanguageModelMessageType; +``` diff --git a/docs-devsite/ai.languagemodelmessageshorthand.md b/docs-devsite/ai.languagemodelmessageshorthand.md new file mode 100644 index 00000000000..bf821b31d52 --- /dev/null +++ b/docs-devsite/ai.languagemodelmessageshorthand.md @@ -0,0 +1,40 @@ +Project: /docs/reference/js/_project.yaml +Book: /docs/reference/_book.yaml +page_type: reference + +{% comment %} +DO NOT EDIT THIS FILE! +This is generated by the JS SDK team, and any local changes will be +overwritten. Changes should be made in the source code at +https://github.com/firebase/firebase-js-sdk +{% endcomment %} + +# LanguageModelMessageShorthand interface +Signature: + +```typescript +export interface LanguageModelMessageShorthand +``` + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [content](./ai.languagemodelmessageshorthand.md#languagemodelmessageshorthandcontent) | string | | +| [role](./ai.languagemodelmessageshorthand.md#languagemodelmessageshorthandrole) | [LanguageModelMessageRole](./ai.md#languagemodelmessagerole) | | + +## LanguageModelMessageShorthand.content + +Signature: + +```typescript +content: string; +``` + +## LanguageModelMessageShorthand.role + +Signature: + +```typescript +role: LanguageModelMessageRole; +``` diff --git a/docs-devsite/ai.md b/docs-devsite/ai.md index 01b3a455682..699d3a83cd6 100644 --- a/docs-devsite/ai.md +++ b/docs-devsite/ai.md @@ -105,6 +105,12 @@ The Firebase AI Web SDK. | [ImagenModelParams](./ai.imagenmodelparams.md#imagenmodelparams_interface) | (Public Preview) Parameters for configuring an [ImagenModel](./ai.imagenmodel.md#imagenmodel_class). | | [ImagenSafetySettings](./ai.imagensafetysettings.md#imagensafetysettings_interface) | (Public Preview) Settings for controlling the aggressiveness of filtering out sensitive content.See the [documentation](http://firebase.google.com/docs/vertex-ai/generate-images) for more details. | | [InlineDataPart](./ai.inlinedatapart.md#inlinedatapart_interface) | Content part interface if the part represents an image. | +| [LanguageModelCreateCoreOptions](./ai.languagemodelcreatecoreoptions.md#languagemodelcreatecoreoptions_interface) | | +| [LanguageModelCreateOptions](./ai.languagemodelcreateoptions.md#languagemodelcreateoptions_interface) | | +| [LanguageModelExpectedInput](./ai.languagemodelexpectedinput.md#languagemodelexpectedinput_interface) | | +| [LanguageModelMessage](./ai.languagemodelmessage.md#languagemodelmessage_interface) | | +| [LanguageModelMessageContent](./ai.languagemodelmessagecontent.md#languagemodelmessagecontent_interface) | | +| [LanguageModelMessageShorthand](./ai.languagemodelmessageshorthand.md#languagemodelmessageshorthand_interface) | | | [ModalityTokenCount](./ai.modalitytokencount.md#modalitytokencount_interface) | Represents token counting info for a single modality. | | [ModelParams](./ai.modelparams.md#modelparams_interface) | Params passed to [getGenerativeModel()](./ai.md#getgenerativemodel_c63f46a). | | [ObjectSchemaInterface](./ai.objectschemainterface.md#objectschemainterface_interface) | Interface for [ObjectSchema](./ai.objectschema.md#objectschema_class) class. | @@ -143,6 +149,10 @@ The Firebase AI Web SDK. | --- | --- | | [BackendType](./ai.md#backendtype) | Type alias representing valid backend types. It can be either 'VERTEX_AI' or 'GOOGLE_AI'. | | [InferenceMode](./ai.md#inferencemode) | Determines whether inference happens on-device or in-cloud. | +| [LanguageModelInitialPrompts](./ai.md#languagemodelinitialprompts) | | +| [LanguageModelMessageContentValue](./ai.md#languagemodelmessagecontentvalue) | | +| [LanguageModelMessageRole](./ai.md#languagemodelmessagerole) | | +| [LanguageModelMessageType](./ai.md#languagemodelmessagetype) | | | [Part](./ai.md#part) | Content part - includes text, image/video, or function call/response part types. | | [ResponseModality](./ai.md#responsemodality) | (Public Preview) Generation modalities to be returned in generation responses. | | [Role](./ai.md#role) | Role is the producer of the content. | @@ -373,6 +383,38 @@ Determines whether inference happens on-device or in-cloud. export type InferenceMode = 'prefer_on_device' | 'only_on_device' | 'only_in_cloud'; ``` +## LanguageModelInitialPrompts + +Signature: + +```typescript +export type LanguageModelInitialPrompts = LanguageModelMessage[] | LanguageModelMessageShorthand[]; +``` + +## LanguageModelMessageContentValue + +Signature: + +```typescript +export type LanguageModelMessageContentValue = ImageBitmapSource | AudioBuffer | BufferSource | string; +``` + +## LanguageModelMessageRole + +Signature: + +```typescript +export type LanguageModelMessageRole = 'system' | 'user' | 'assistant'; +``` + +## LanguageModelMessageType + +Signature: + +```typescript +export type LanguageModelMessageType = 'text' | 'image' | 'audio'; +``` + ## Part Content part - includes text, image/video, or function call/response part types. diff --git a/docs-devsite/ai.ondeviceparams.md b/docs-devsite/ai.ondeviceparams.md index f4bfcbb5cff..16fed65560d 100644 --- a/docs-devsite/ai.ondeviceparams.md +++ b/docs-devsite/ai.ondeviceparams.md @@ -22,7 +22,7 @@ export interface OnDeviceParams | Property | Type | Description | | --- | --- | --- | -| [createOptions](./ai.ondeviceparams.md#ondeviceparamscreateoptions) | LanguageModelCreateOptions | | +| [createOptions](./ai.ondeviceparams.md#ondeviceparamscreateoptions) | [LanguageModelCreateOptions](./ai.languagemodelcreateoptions.md#languagemodelcreateoptions_interface) | | | [promptOptions](./ai.ondeviceparams.md#ondeviceparamspromptoptions) | LanguageModelPromptOptions | | ## OnDeviceParams.createOptions diff --git a/packages/ai/src/methods/chrome-adapter.ts b/packages/ai/src/methods/chrome-adapter.ts index aa3709048a2..d6de108668d 100644 --- a/packages/ai/src/methods/chrome-adapter.ts +++ b/packages/ai/src/methods/chrome-adapter.ts @@ -104,7 +104,7 @@ export class ChromeAdapter { * *

This is comparable to {@link GenerativeModel.generateContent} for generating content in * Cloud.

- * @param request a standard Vertex {@link GenerateContentRequest} + * @param request - a standard Vertex {@link GenerateContentRequest} * @returns {@link Response}, so we can reuse common response formatting. */ async generateContent(request: GenerateContentRequest): Promise { @@ -126,7 +126,7 @@ export class ChromeAdapter { * *

This is comparable to {@link GenerativeModel.generateContentStream} for generating content in * Cloud.

- * @param request a standard Vertex {@link GenerateContentRequest} + * @param request - a standard Vertex {@link GenerateContentRequest} * @returns {@link Response}, so we can reuse common response formatting. */ async generateContentStream( diff --git a/packages/ai/src/types/index.ts b/packages/ai/src/types/index.ts index 01f3e7a701a..698f15b8aea 100644 --- a/packages/ai/src/types/index.ts +++ b/packages/ai/src/types/index.ts @@ -23,3 +23,15 @@ export * from './error'; export * from './schema'; export * from './imagen'; export * from './googleai'; +export { + LanguageModelCreateOptions, + LanguageModelCreateCoreOptions, + LanguageModelExpectedInput, + LanguageModelInitialPrompts, + LanguageModelMessage, + LanguageModelMessageContent, + LanguageModelMessageContentValue, + LanguageModelMessageRole, + LanguageModelMessageShorthand, + LanguageModelMessageType +} from './language-model'; diff --git a/packages/ai/src/types/language-model.ts b/packages/ai/src/types/language-model.ts index 22916e7ff96..de4020f66bf 100644 --- a/packages/ai/src/types/language-model.ts +++ b/packages/ai/src/types/language-model.ts @@ -53,20 +53,20 @@ export interface LanguageModelPromptOptions { responseConstraint?: object; // TODO: Restore AbortSignal once the API is defined. } -interface LanguageModelExpectedInput { +export interface LanguageModelExpectedInput { type: LanguageModelMessageType; languages?: string[]; } // TODO: revert to type from Prompt API explainer once it's supported. export type LanguageModelPrompt = LanguageModelMessageContent[]; -type LanguageModelInitialPrompts = +export type LanguageModelInitialPrompts = | LanguageModelMessage[] | LanguageModelMessageShorthand[]; -interface LanguageModelMessage { +export interface LanguageModelMessage { role: LanguageModelMessageRole; content: LanguageModelMessageContent[]; } -interface LanguageModelMessageShorthand { +export interface LanguageModelMessageShorthand { role: LanguageModelMessageRole; content: string; } @@ -74,9 +74,9 @@ export interface LanguageModelMessageContent { type: LanguageModelMessageType; content: LanguageModelMessageContentValue; } -type LanguageModelMessageRole = 'system' | 'user' | 'assistant'; -type LanguageModelMessageType = 'text' | 'image' | 'audio'; -type LanguageModelMessageContentValue = +export type LanguageModelMessageRole = 'system' | 'user' | 'assistant'; +export type LanguageModelMessageType = 'text' | 'image' | 'audio'; +export type LanguageModelMessageContentValue = | ImageBitmapSource | AudioBuffer | BufferSource From 72cd62665d0cbf10c3879cc27f5977acc4ad74ee Mon Sep 17 00:00:00 2001 From: Erik Eldridge Date: Thu, 15 May 2025 10:08:10 -0700 Subject: [PATCH 04/20] AI Hybrid Inference: guard against undefined mode (#9045) --- packages/ai/src/methods/chrome-adapter.test.ts | 18 ++++++++++++++++-- packages/ai/src/methods/chrome-adapter.ts | 6 ++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/packages/ai/src/methods/chrome-adapter.test.ts b/packages/ai/src/methods/chrome-adapter.test.ts index fbe7ec1a5c5..adb9ae47d87 100644 --- a/packages/ai/src/methods/chrome-adapter.test.ts +++ b/packages/ai/src/methods/chrome-adapter.test.ts @@ -109,6 +109,14 @@ describe('ChromeAdapter', () => { }); }); describe('isAvailable', () => { + it('returns false if mode is undefined', async () => { + const adapter = new ChromeAdapter(); + expect( + await adapter.isAvailable({ + contents: [] + }) + ).to.be.false; + }); it('returns false if mode is only cloud', async () => { const adapter = new ChromeAdapter(undefined, 'only_in_cloud'); expect( @@ -239,7 +247,10 @@ describe('ChromeAdapter', () => { const createStub = stub(languageModelProvider, 'create').returns( downloadPromise ); - const adapter = new ChromeAdapter(languageModelProvider); + const adapter = new ChromeAdapter( + languageModelProvider, + 'prefer_on_device' + ); await adapter.isAvailable({ contents: [{ role: 'user', parts: [{ text: 'hi' }] }] }); @@ -260,7 +271,10 @@ describe('ChromeAdapter', () => { const createStub = stub(languageModelProvider, 'create').returns( downloadPromise ); - const adapter = new ChromeAdapter(languageModelProvider); + const adapter = new ChromeAdapter( + languageModelProvider, + 'prefer_on_device' + ); await adapter.isAvailable({ contents: [{ role: 'user', parts: [{ text: 'hi' }] }] }); diff --git a/packages/ai/src/methods/chrome-adapter.ts b/packages/ai/src/methods/chrome-adapter.ts index d6de108668d..b61ad9b5f09 100644 --- a/packages/ai/src/methods/chrome-adapter.ts +++ b/packages/ai/src/methods/chrome-adapter.ts @@ -68,6 +68,12 @@ export class ChromeAdapter { * separation of concerns.

*/ async isAvailable(request: GenerateContentRequest): Promise { + if (!this.mode) { + logger.debug( + `On-device inference unavailable because mode is undefined.` + ); + return false; + } if (this.mode === 'only_in_cloud') { logger.debug( `On-device inference unavailable because mode is "only_in_cloud".` From 177f546912ee1ce5726b4ea3807886b607291197 Mon Sep 17 00:00:00 2001 From: Erik Eldridge Date: Wed, 28 May 2025 14:25:43 -0700 Subject: [PATCH 05/20] AI Hybrid Inference: migrate to LanguageModelMessage (#9027) --- common/api-review/ai.api.md | 4 +- .../ai/src/methods/chrome-adapter.test.ts | 144 ++++++++++++++---- packages/ai/src/methods/chrome-adapter.ts | 53 +++++-- packages/ai/src/types/language-model.ts | 12 +- 4 files changed, 165 insertions(+), 48 deletions(-) diff --git a/common/api-review/ai.api.md b/common/api-review/ai.api.md index a7da6210ada..97d25b9e03d 100644 --- a/common/api-review/ai.api.md +++ b/common/api-review/ai.api.md @@ -705,10 +705,10 @@ export interface LanguageModelMessage { // @public (undocumented) export interface LanguageModelMessageContent { - // (undocumented) - content: LanguageModelMessageContentValue; // (undocumented) type: LanguageModelMessageType; + // (undocumented) + value: LanguageModelMessageContentValue; } // @public (undocumented) diff --git a/packages/ai/src/methods/chrome-adapter.test.ts b/packages/ai/src/methods/chrome-adapter.test.ts index adb9ae47d87..f8ea80b0e09 100644 --- a/packages/ai/src/methods/chrome-adapter.test.ts +++ b/packages/ai/src/methods/chrome-adapter.test.ts @@ -24,7 +24,7 @@ import { Availability, LanguageModel, LanguageModelCreateOptions, - LanguageModelMessageContent + LanguageModelMessage } from '../types/language-model'; import { match, stub } from 'sinon'; import { GenerateContentRequest, AIErrorCode } from '../types'; @@ -146,7 +146,7 @@ describe('ChromeAdapter', () => { }) ).to.be.false; }); - it('returns false if request content has non-user role', async () => { + it('returns false if request content has "function" role', async () => { const adapter = new ChromeAdapter( { availability: async () => Availability.available @@ -157,7 +157,7 @@ describe('ChromeAdapter', () => { await adapter.isAvailable({ contents: [ { - role: 'model', + role: 'function', parts: [] } ] @@ -320,7 +320,7 @@ describe('ChromeAdapter', () => { } as LanguageModel; const languageModel = { // eslint-disable-next-line @typescript-eslint/no-unused-vars - prompt: (p: LanguageModelMessageContent[]) => Promise.resolve('') + prompt: (p: LanguageModelMessage[]) => Promise.resolve('') } as LanguageModel; const createStub = stub(languageModelProvider, 'create').resolves( languageModel @@ -345,8 +345,13 @@ describe('ChromeAdapter', () => { // Asserts Vertex input type is mapped to Chrome type. expect(promptStub).to.have.been.calledOnceWith([ { - type: 'text', - content: request.contents[0].parts[0].text + role: request.contents[0].role, + content: [ + { + type: 'text', + value: request.contents[0].parts[0].text + } + ] } ]); // Asserts expected output. @@ -366,7 +371,7 @@ describe('ChromeAdapter', () => { } as LanguageModel; const languageModel = { // eslint-disable-next-line @typescript-eslint/no-unused-vars - prompt: (p: LanguageModelMessageContent[]) => Promise.resolve('') + prompt: (p: LanguageModelMessage[]) => Promise.resolve('') } as LanguageModel; const createStub = stub(languageModelProvider, 'create').resolves( languageModel @@ -404,12 +409,17 @@ describe('ChromeAdapter', () => { // Asserts Vertex input type is mapped to Chrome type. expect(promptStub).to.have.been.calledOnceWith([ { - type: 'text', - content: request.contents[0].parts[0].text - }, - { - type: 'image', - content: match.instanceOf(ImageBitmap) + role: request.contents[0].role, + content: [ + { + type: 'text', + value: request.contents[0].parts[0].text + }, + { + type: 'image', + value: match.instanceOf(ImageBitmap) + } + ] } ]); // Asserts expected output. @@ -426,7 +436,7 @@ describe('ChromeAdapter', () => { it('honors prompt options', async () => { const languageModel = { // eslint-disable-next-line @typescript-eslint/no-unused-vars - prompt: (p: LanguageModelMessageContent[]) => Promise.resolve('') + prompt: (p: LanguageModelMessage[]) => Promise.resolve('') } as LanguageModel; const languageModelProvider = { create: () => Promise.resolve(languageModel) @@ -450,13 +460,48 @@ describe('ChromeAdapter', () => { expect(promptStub).to.have.been.calledOnceWith( [ { - type: 'text', - content: request.contents[0].parts[0].text + role: request.contents[0].role, + content: [ + { + type: 'text', + value: request.contents[0].parts[0].text + } + ] } ], promptOptions ); }); + it('normalizes roles', async () => { + const languageModel = { + // eslint-disable-next-line @typescript-eslint/no-unused-vars + prompt: (p: LanguageModelMessage[]) => Promise.resolve('unused') + } as LanguageModel; + const promptStub = stub(languageModel, 'prompt').resolves('unused'); + const languageModelProvider = { + create: () => Promise.resolve(languageModel) + } as LanguageModel; + const adapter = new ChromeAdapter( + languageModelProvider, + 'prefer_on_device' + ); + const request = { + contents: [{ role: 'model', parts: [{ text: 'unused' }] }] + } as GenerateContentRequest; + await adapter.generateContent(request); + expect(promptStub).to.have.been.calledOnceWith([ + { + // Asserts Vertex's "model" role normalized to Chrome's "assistant" role. + role: 'assistant', + content: [ + { + type: 'text', + value: request.contents[0].parts[0].text + } + ] + } + ]); + }); }); describe('countTokens', () => { it('counts tokens is not yet available', async () => { @@ -528,8 +573,13 @@ describe('ChromeAdapter', () => { expect(createStub).to.have.been.calledOnceWith(createOptions); expect(promptStub).to.have.been.calledOnceWith([ { - type: 'text', - content: request.contents[0].parts[0].text + role: request.contents[0].role, + content: [ + { + type: 'text', + value: request.contents[0].parts[0].text + } + ] } ]); const actual = await toStringArray(response.body!); @@ -584,12 +634,17 @@ describe('ChromeAdapter', () => { expect(createStub).to.have.been.calledOnceWith(createOptions); expect(promptStub).to.have.been.calledOnceWith([ { - type: 'text', - content: request.contents[0].parts[0].text - }, - { - type: 'image', - content: match.instanceOf(ImageBitmap) + role: request.contents[0].role, + content: [ + { + type: 'text', + value: request.contents[0].parts[0].text + }, + { + type: 'image', + value: match.instanceOf(ImageBitmap) + } + ] } ]); const actual = await toStringArray(response.body!); @@ -625,13 +680,50 @@ describe('ChromeAdapter', () => { expect(promptStub).to.have.been.calledOnceWith( [ { - type: 'text', - content: request.contents[0].parts[0].text + role: request.contents[0].role, + content: [ + { + type: 'text', + value: request.contents[0].parts[0].text + } + ] } ], promptOptions ); }); + it('normalizes roles', async () => { + const languageModel = { + // eslint-disable-next-line @typescript-eslint/no-unused-vars + promptStreaming: p => new ReadableStream() + } as LanguageModel; + const promptStub = stub(languageModel, 'promptStreaming').returns( + new ReadableStream() + ); + const languageModelProvider = { + create: () => Promise.resolve(languageModel) + } as LanguageModel; + const adapter = new ChromeAdapter( + languageModelProvider, + 'prefer_on_device' + ); + const request = { + contents: [{ role: 'model', parts: [{ text: 'unused' }] }] + } as GenerateContentRequest; + await adapter.generateContentStream(request); + expect(promptStub).to.have.been.calledOnceWith([ + { + // Asserts Vertex's "model" role normalized to Chrome's "assistant" role. + role: 'assistant', + content: [ + { + type: 'text', + value: request.contents[0].parts[0].text + } + ] + } + ]); + }); }); }); diff --git a/packages/ai/src/methods/chrome-adapter.ts b/packages/ai/src/methods/chrome-adapter.ts index b61ad9b5f09..e7bb39c34c8 100644 --- a/packages/ai/src/methods/chrome-adapter.ts +++ b/packages/ai/src/methods/chrome-adapter.ts @@ -23,12 +23,16 @@ import { InferenceMode, Part, AIErrorCode, - OnDeviceParams + OnDeviceParams, + Content, + Role } from '../types'; import { Availability, LanguageModel, - LanguageModelMessageContent + LanguageModelMessage, + LanguageModelMessageContent, + LanguageModelMessageRole } from '../types/language-model'; /** @@ -115,10 +119,8 @@ export class ChromeAdapter { */ async generateContent(request: GenerateContentRequest): Promise { const session = await this.createSession(); - // TODO: support multiple content objects when Chrome supports - // sequence const contents = await Promise.all( - request.contents[0].parts.map(ChromeAdapter.toLanguageModelMessageContent) + request.contents.map(ChromeAdapter.toLanguageModelMessage) ); const text = await session.prompt( contents, @@ -139,10 +141,8 @@ export class ChromeAdapter { request: GenerateContentRequest ): Promise { const session = await this.createSession(); - // TODO: support multiple content objects when Chrome supports - // sequence const contents = await Promise.all( - request.contents[0].parts.map(ChromeAdapter.toLanguageModelMessageContent) + request.contents.map(ChromeAdapter.toLanguageModelMessage) ); const stream = await session.promptStreaming( contents, @@ -169,12 +169,8 @@ export class ChromeAdapter { } for (const content of request.contents) { - // Returns false if the request contains multiple roles, eg a chat history. - // TODO: remove this guard once LanguageModelMessage is supported. - if (content.role !== 'user') { - logger.debug( - `Non-user role "${content.role}" rejected for on-device inference.` - ); + if (content.role === 'function') { + logger.debug(`"Function" role rejected for on-device inference.`); return false; } @@ -233,6 +229,21 @@ export class ChromeAdapter { }); } + /** + * Converts Vertex {@link Content} object to a Chrome {@link LanguageModelMessage} object. + */ + private static async toLanguageModelMessage( + content: Content + ): Promise { + const languageModelMessageContents = await Promise.all( + content.parts.map(ChromeAdapter.toLanguageModelMessageContent) + ); + return { + role: ChromeAdapter.toLanguageModelMessageRole(content.role), + content: languageModelMessageContents + }; + } + /** * Converts a Vertex Part object to a Chrome LanguageModelMessageContent object. */ @@ -242,7 +253,7 @@ export class ChromeAdapter { if (part.text) { return { type: 'text', - content: part.text + value: part.text }; } else if (part.inlineData) { const formattedImageContent = await fetch( @@ -252,7 +263,7 @@ export class ChromeAdapter { const imageBitmap = await createImageBitmap(imageBlob); return { type: 'image', - content: imageBitmap + value: imageBitmap }; } // Assumes contents have been verified to contain only a single TextPart. @@ -260,6 +271,16 @@ export class ChromeAdapter { throw new Error('Not yet implemented'); } + /** + * Converts a Vertex {@link Role} string to a {@link LanguageModelMessageRole} string. + */ + private static toLanguageModelMessageRole( + role: Role + ): LanguageModelMessageRole { + // Assumes 'function' rule has been filtered by isOnDeviceRequest + return role === 'model' ? 'assistant' : 'user'; + } + /** * Abstracts Chrome session creation. * diff --git a/packages/ai/src/types/language-model.ts b/packages/ai/src/types/language-model.ts index de4020f66bf..503f3d49d05 100644 --- a/packages/ai/src/types/language-model.ts +++ b/packages/ai/src/types/language-model.ts @@ -14,7 +14,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - +/** + * {@see https://github.com/webmachinelearning/prompt-api#full-api-surface-in-web-idl} + */ export interface LanguageModel extends EventTarget { create(options?: LanguageModelCreateOptions): Promise; availability(options?: LanguageModelCreateCoreOptions): Promise; @@ -57,8 +59,10 @@ export interface LanguageModelExpectedInput { type: LanguageModelMessageType; languages?: string[]; } -// TODO: revert to type from Prompt API explainer once it's supported. -export type LanguageModelPrompt = LanguageModelMessageContent[]; +export type LanguageModelPrompt = + | LanguageModelMessage[] + | LanguageModelMessageShorthand[] + | string; export type LanguageModelInitialPrompts = | LanguageModelMessage[] | LanguageModelMessageShorthand[]; @@ -72,7 +76,7 @@ export interface LanguageModelMessageShorthand { } export interface LanguageModelMessageContent { type: LanguageModelMessageType; - content: LanguageModelMessageContentValue; + value: LanguageModelMessageContentValue; } export type LanguageModelMessageRole = 'system' | 'user' | 'assistant'; export type LanguageModelMessageType = 'text' | 'image' | 'audio'; From 58d92df3314c77f132002652cbe00b676eb3a00f Mon Sep 17 00:00:00 2001 From: Erik Eldridge Date: Wed, 28 May 2025 15:02:18 -0700 Subject: [PATCH 06/20] AI Hybrid Inference: flatten initial prompts type (#9066) --- common/api-review/ai.api.md | 19 ++------- docs-devsite/_toc.yaml | 6 +-- .../ai.languagemodelcreatecoreoptions.md | 4 +- docs-devsite/ai.languagemodelcreateoptions.md | 13 +----- ...edinput.md => ai.languagemodelexpected.md} | 12 +++--- .../ai.languagemodelmessagecontent.md | 10 ++--- .../ai.languagemodelmessageshorthand.md | 40 ------------------- docs-devsite/ai.md | 12 +----- packages/ai/src/types/index.ts | 4 +- packages/ai/src/types/language-model.ts | 23 ++++------- 10 files changed, 29 insertions(+), 114 deletions(-) rename docs-devsite/{ai.languagemodelexpectedinput.md => ai.languagemodelexpected.md} (57%) delete mode 100644 docs-devsite/ai.languagemodelmessageshorthand.md diff --git a/common/api-review/ai.api.md b/common/api-review/ai.api.md index 97d25b9e03d..76e0b48dda8 100644 --- a/common/api-review/ai.api.md +++ b/common/api-review/ai.api.md @@ -667,7 +667,7 @@ export class IntegerSchema extends Schema { // @public (undocumented) export interface LanguageModelCreateCoreOptions { // (undocumented) - expectedInputs?: LanguageModelExpectedInput[]; + expectedInputs?: LanguageModelExpected[]; // (undocumented) temperature?: number; // (undocumented) @@ -677,24 +677,19 @@ export interface LanguageModelCreateCoreOptions { // @public (undocumented) export interface LanguageModelCreateOptions extends LanguageModelCreateCoreOptions { // (undocumented) - initialPrompts?: LanguageModelInitialPrompts; + initialPrompts?: LanguageModelMessage[]; // (undocumented) signal?: AbortSignal; - // (undocumented) - systemPrompt?: string; } // @public (undocumented) -export interface LanguageModelExpectedInput { +export interface LanguageModelExpected { // (undocumented) languages?: string[]; // (undocumented) type: LanguageModelMessageType; } -// @public (undocumented) -export type LanguageModelInitialPrompts = LanguageModelMessage[] | LanguageModelMessageShorthand[]; - // @public (undocumented) export interface LanguageModelMessage { // (undocumented) @@ -717,14 +712,6 @@ export type LanguageModelMessageContentValue = ImageBitmapSource | AudioBuffer | // @public (undocumented) export type LanguageModelMessageRole = 'system' | 'user' | 'assistant'; -// @public (undocumented) -export interface LanguageModelMessageShorthand { - // (undocumented) - content: string; - // (undocumented) - role: LanguageModelMessageRole; -} - // @public (undocumented) export type LanguageModelMessageType = 'text' | 'image' | 'audio'; diff --git a/docs-devsite/_toc.yaml b/docs-devsite/_toc.yaml index e8359727cda..621c6af9e91 100644 --- a/docs-devsite/_toc.yaml +++ b/docs-devsite/_toc.yaml @@ -106,14 +106,12 @@ toc: path: /docs/reference/js/ai.languagemodelcreatecoreoptions.md - title: LanguageModelCreateOptions path: /docs/reference/js/ai.languagemodelcreateoptions.md - - title: LanguageModelExpectedInput - path: /docs/reference/js/ai.languagemodelexpectedinput.md + - title: LanguageModelExpected + path: /docs/reference/js/ai.languagemodelexpected.md - title: LanguageModelMessage path: /docs/reference/js/ai.languagemodelmessage.md - title: LanguageModelMessageContent path: /docs/reference/js/ai.languagemodelmessagecontent.md - - title: LanguageModelMessageShorthand - path: /docs/reference/js/ai.languagemodelmessageshorthand.md - title: ModalityTokenCount path: /docs/reference/js/ai.modalitytokencount.md - title: ModelParams diff --git a/docs-devsite/ai.languagemodelcreatecoreoptions.md b/docs-devsite/ai.languagemodelcreatecoreoptions.md index 2c9f61b149f..45c2e7f5db4 100644 --- a/docs-devsite/ai.languagemodelcreatecoreoptions.md +++ b/docs-devsite/ai.languagemodelcreatecoreoptions.md @@ -20,7 +20,7 @@ export interface LanguageModelCreateCoreOptions | Property | Type | Description | | --- | --- | --- | -| [expectedInputs](./ai.languagemodelcreatecoreoptions.md#languagemodelcreatecoreoptionsexpectedinputs) | [LanguageModelExpectedInput](./ai.languagemodelexpectedinput.md#languagemodelexpectedinput_interface)\[\] | | +| [expectedInputs](./ai.languagemodelcreatecoreoptions.md#languagemodelcreatecoreoptionsexpectedinputs) | [LanguageModelExpected](./ai.languagemodelexpected.md#languagemodelexpected_interface)\[\] | | | [temperature](./ai.languagemodelcreatecoreoptions.md#languagemodelcreatecoreoptionstemperature) | number | | | [topK](./ai.languagemodelcreatecoreoptions.md#languagemodelcreatecoreoptionstopk) | number | | @@ -29,7 +29,7 @@ export interface LanguageModelCreateCoreOptions Signature: ```typescript -expectedInputs?: LanguageModelExpectedInput[]; +expectedInputs?: LanguageModelExpected[]; ``` ## LanguageModelCreateCoreOptions.temperature diff --git a/docs-devsite/ai.languagemodelcreateoptions.md b/docs-devsite/ai.languagemodelcreateoptions.md index 44edcf7e221..417519a54b6 100644 --- a/docs-devsite/ai.languagemodelcreateoptions.md +++ b/docs-devsite/ai.languagemodelcreateoptions.md @@ -21,16 +21,15 @@ export interface LanguageModelCreateOptions extends LanguageModelCreateCoreOptio | Property | Type | Description | | --- | --- | --- | -| [initialPrompts](./ai.languagemodelcreateoptions.md#languagemodelcreateoptionsinitialprompts) | [LanguageModelInitialPrompts](./ai.md#languagemodelinitialprompts) | | +| [initialPrompts](./ai.languagemodelcreateoptions.md#languagemodelcreateoptionsinitialprompts) | [LanguageModelMessage](./ai.languagemodelmessage.md#languagemodelmessage_interface)\[\] | | | [signal](./ai.languagemodelcreateoptions.md#languagemodelcreateoptionssignal) | AbortSignal | | -| [systemPrompt](./ai.languagemodelcreateoptions.md#languagemodelcreateoptionssystemprompt) | string | | ## LanguageModelCreateOptions.initialPrompts Signature: ```typescript -initialPrompts?: LanguageModelInitialPrompts; +initialPrompts?: LanguageModelMessage[]; ``` ## LanguageModelCreateOptions.signal @@ -40,11 +39,3 @@ initialPrompts?: LanguageModelInitialPrompts; ```typescript signal?: AbortSignal; ``` - -## LanguageModelCreateOptions.systemPrompt - -Signature: - -```typescript -systemPrompt?: string; -``` diff --git a/docs-devsite/ai.languagemodelexpectedinput.md b/docs-devsite/ai.languagemodelexpected.md similarity index 57% rename from docs-devsite/ai.languagemodelexpectedinput.md rename to docs-devsite/ai.languagemodelexpected.md index d6cbe028fc1..26ed28b741e 100644 --- a/docs-devsite/ai.languagemodelexpectedinput.md +++ b/docs-devsite/ai.languagemodelexpected.md @@ -9,21 +9,21 @@ overwritten. Changes should be made in the source code at https://github.com/firebase/firebase-js-sdk {% endcomment %} -# LanguageModelExpectedInput interface +# LanguageModelExpected interface Signature: ```typescript -export interface LanguageModelExpectedInput +export interface LanguageModelExpected ``` ## Properties | Property | Type | Description | | --- | --- | --- | -| [languages](./ai.languagemodelexpectedinput.md#languagemodelexpectedinputlanguages) | string\[\] | | -| [type](./ai.languagemodelexpectedinput.md#languagemodelexpectedinputtype) | [LanguageModelMessageType](./ai.md#languagemodelmessagetype) | | +| [languages](./ai.languagemodelexpected.md#languagemodelexpectedlanguages) | string\[\] | | +| [type](./ai.languagemodelexpected.md#languagemodelexpectedtype) | [LanguageModelMessageType](./ai.md#languagemodelmessagetype) | | -## LanguageModelExpectedInput.languages +## LanguageModelExpected.languages Signature: @@ -31,7 +31,7 @@ export interface LanguageModelExpectedInput languages?: string[]; ``` -## LanguageModelExpectedInput.type +## LanguageModelExpected.type Signature: diff --git a/docs-devsite/ai.languagemodelmessagecontent.md b/docs-devsite/ai.languagemodelmessagecontent.md index 06830ace272..40b4cc16bce 100644 --- a/docs-devsite/ai.languagemodelmessagecontent.md +++ b/docs-devsite/ai.languagemodelmessagecontent.md @@ -20,21 +20,21 @@ export interface LanguageModelMessageContent | Property | Type | Description | | --- | --- | --- | -| [content](./ai.languagemodelmessagecontent.md#languagemodelmessagecontentcontent) | [LanguageModelMessageContentValue](./ai.md#languagemodelmessagecontentvalue) | | | [type](./ai.languagemodelmessagecontent.md#languagemodelmessagecontenttype) | [LanguageModelMessageType](./ai.md#languagemodelmessagetype) | | +| [value](./ai.languagemodelmessagecontent.md#languagemodelmessagecontentvalue) | [LanguageModelMessageContentValue](./ai.md#languagemodelmessagecontentvalue) | | -## LanguageModelMessageContent.content +## LanguageModelMessageContent.type Signature: ```typescript -content: LanguageModelMessageContentValue; +type: LanguageModelMessageType; ``` -## LanguageModelMessageContent.type +## LanguageModelMessageContent.value Signature: ```typescript -type: LanguageModelMessageType; +value: LanguageModelMessageContentValue; ``` diff --git a/docs-devsite/ai.languagemodelmessageshorthand.md b/docs-devsite/ai.languagemodelmessageshorthand.md deleted file mode 100644 index bf821b31d52..00000000000 --- a/docs-devsite/ai.languagemodelmessageshorthand.md +++ /dev/null @@ -1,40 +0,0 @@ -Project: /docs/reference/js/_project.yaml -Book: /docs/reference/_book.yaml -page_type: reference - -{% comment %} -DO NOT EDIT THIS FILE! -This is generated by the JS SDK team, and any local changes will be -overwritten. Changes should be made in the source code at -https://github.com/firebase/firebase-js-sdk -{% endcomment %} - -# LanguageModelMessageShorthand interface -Signature: - -```typescript -export interface LanguageModelMessageShorthand -``` - -## Properties - -| Property | Type | Description | -| --- | --- | --- | -| [content](./ai.languagemodelmessageshorthand.md#languagemodelmessageshorthandcontent) | string | | -| [role](./ai.languagemodelmessageshorthand.md#languagemodelmessageshorthandrole) | [LanguageModelMessageRole](./ai.md#languagemodelmessagerole) | | - -## LanguageModelMessageShorthand.content - -Signature: - -```typescript -content: string; -``` - -## LanguageModelMessageShorthand.role - -Signature: - -```typescript -role: LanguageModelMessageRole; -``` diff --git a/docs-devsite/ai.md b/docs-devsite/ai.md index 699d3a83cd6..b087d7037e6 100644 --- a/docs-devsite/ai.md +++ b/docs-devsite/ai.md @@ -107,10 +107,9 @@ The Firebase AI Web SDK. | [InlineDataPart](./ai.inlinedatapart.md#inlinedatapart_interface) | Content part interface if the part represents an image. | | [LanguageModelCreateCoreOptions](./ai.languagemodelcreatecoreoptions.md#languagemodelcreatecoreoptions_interface) | | | [LanguageModelCreateOptions](./ai.languagemodelcreateoptions.md#languagemodelcreateoptions_interface) | | -| [LanguageModelExpectedInput](./ai.languagemodelexpectedinput.md#languagemodelexpectedinput_interface) | | +| [LanguageModelExpected](./ai.languagemodelexpected.md#languagemodelexpected_interface) | | | [LanguageModelMessage](./ai.languagemodelmessage.md#languagemodelmessage_interface) | | | [LanguageModelMessageContent](./ai.languagemodelmessagecontent.md#languagemodelmessagecontent_interface) | | -| [LanguageModelMessageShorthand](./ai.languagemodelmessageshorthand.md#languagemodelmessageshorthand_interface) | | | [ModalityTokenCount](./ai.modalitytokencount.md#modalitytokencount_interface) | Represents token counting info for a single modality. | | [ModelParams](./ai.modelparams.md#modelparams_interface) | Params passed to [getGenerativeModel()](./ai.md#getgenerativemodel_c63f46a). | | [ObjectSchemaInterface](./ai.objectschemainterface.md#objectschemainterface_interface) | Interface for [ObjectSchema](./ai.objectschema.md#objectschema_class) class. | @@ -149,7 +148,6 @@ The Firebase AI Web SDK. | --- | --- | | [BackendType](./ai.md#backendtype) | Type alias representing valid backend types. It can be either 'VERTEX_AI' or 'GOOGLE_AI'. | | [InferenceMode](./ai.md#inferencemode) | Determines whether inference happens on-device or in-cloud. | -| [LanguageModelInitialPrompts](./ai.md#languagemodelinitialprompts) | | | [LanguageModelMessageContentValue](./ai.md#languagemodelmessagecontentvalue) | | | [LanguageModelMessageRole](./ai.md#languagemodelmessagerole) | | | [LanguageModelMessageType](./ai.md#languagemodelmessagetype) | | @@ -383,14 +381,6 @@ Determines whether inference happens on-device or in-cloud. export type InferenceMode = 'prefer_on_device' | 'only_on_device' | 'only_in_cloud'; ``` -## LanguageModelInitialPrompts - -Signature: - -```typescript -export type LanguageModelInitialPrompts = LanguageModelMessage[] | LanguageModelMessageShorthand[]; -``` - ## LanguageModelMessageContentValue Signature: diff --git a/packages/ai/src/types/index.ts b/packages/ai/src/types/index.ts index 698f15b8aea..bd13140566f 100644 --- a/packages/ai/src/types/index.ts +++ b/packages/ai/src/types/index.ts @@ -26,12 +26,10 @@ export * from './googleai'; export { LanguageModelCreateOptions, LanguageModelCreateCoreOptions, - LanguageModelExpectedInput, - LanguageModelInitialPrompts, + LanguageModelExpected, LanguageModelMessage, LanguageModelMessageContent, LanguageModelMessageContentValue, LanguageModelMessageRole, - LanguageModelMessageShorthand, LanguageModelMessageType } from './language-model'; diff --git a/packages/ai/src/types/language-model.ts b/packages/ai/src/types/language-model.ts index 503f3d49d05..83a728dc3be 100644 --- a/packages/ai/src/types/language-model.ts +++ b/packages/ai/src/types/language-model.ts @@ -15,7 +15,9 @@ * limitations under the License. */ /** - * {@see https://github.com/webmachinelearning/prompt-api#full-api-surface-in-web-idl} + * The subset of the Prompt API + * ({@see https://github.com/webmachinelearning/prompt-api#full-api-surface-in-web-idl}) + * required for hybrid functionality. */ export interface LanguageModel extends EventTarget { create(options?: LanguageModelCreateOptions): Promise; @@ -43,37 +45,26 @@ export enum Availability { export interface LanguageModelCreateCoreOptions { topK?: number; temperature?: number; - expectedInputs?: LanguageModelExpectedInput[]; + expectedInputs?: LanguageModelExpected[]; } export interface LanguageModelCreateOptions extends LanguageModelCreateCoreOptions { signal?: AbortSignal; - systemPrompt?: string; - initialPrompts?: LanguageModelInitialPrompts; + initialPrompts?: LanguageModelMessage[]; } export interface LanguageModelPromptOptions { responseConstraint?: object; // TODO: Restore AbortSignal once the API is defined. } -export interface LanguageModelExpectedInput { +export interface LanguageModelExpected { type: LanguageModelMessageType; languages?: string[]; } -export type LanguageModelPrompt = - | LanguageModelMessage[] - | LanguageModelMessageShorthand[] - | string; -export type LanguageModelInitialPrompts = - | LanguageModelMessage[] - | LanguageModelMessageShorthand[]; +export type LanguageModelPrompt = LanguageModelMessage[]; export interface LanguageModelMessage { role: LanguageModelMessageRole; content: LanguageModelMessageContent[]; } -export interface LanguageModelMessageShorthand { - role: LanguageModelMessageRole; - content: string; -} export interface LanguageModelMessageContent { type: LanguageModelMessageType; value: LanguageModelMessageContentValue; From e5e8c360611d13f2ac67f02b63b8a6e8bc231507 Mon Sep 17 00:00:00 2001 From: Christina Holland Date: Tue, 29 Jul 2025 15:29:12 -0700 Subject: [PATCH 07/20] rerun docgen --- common/api-review/ai.api.md | 2 +- docs-devsite/_toc.yaml | 8 +++--- docs-devsite/ai.md | 44 ++++++++++++++++++++++++++++--- docs-devsite/ai.thinkingconfig.md | 4 --- 4 files changed, 46 insertions(+), 12 deletions(-) diff --git a/common/api-review/ai.api.md b/common/api-review/ai.api.md index f026d3dc780..7c4b6853def 100644 --- a/common/api-review/ai.api.md +++ b/common/api-review/ai.api.md @@ -1032,7 +1032,7 @@ export interface TextPart { text: string; } -// @public +// @public (undocumented) export interface ThinkingConfig { thinkingBudget?: number; } diff --git a/docs-devsite/_toc.yaml b/docs-devsite/_toc.yaml index 81ec5fc9179..fd9ae62b0d9 100644 --- a/docs-devsite/_toc.yaml +++ b/docs-devsite/_toc.yaml @@ -86,10 +86,10 @@ toc: path: /docs/reference/js/ai.groundingchunk.md - title: GroundingMetadata path: /docs/reference/js/ai.groundingmetadata.md - - title: HybridParams - path: /docs/reference/js/ai.hybridparams.md - title: GroundingSupport path: /docs/reference/js/ai.groundingsupport.md + - title: HybridParams + path: /docs/reference/js/ai.hybridparams.md - title: ImagenGCSImage path: /docs/reference/js/ai.imagengcsimage.md - title: ImagenGenerationConfig @@ -128,10 +128,10 @@ toc: path: /docs/reference/js/ai.numberschema.md - title: ObjectSchema path: /docs/reference/js/ai.objectschema.md - - title: OnDeviceParams - path: /docs/reference/js/ai.ondeviceparams.md - title: ObjectSchemaRequest path: /docs/reference/js/ai.objectschemarequest.md + - title: OnDeviceParams + path: /docs/reference/js/ai.ondeviceparams.md - title: PromptFeedback path: /docs/reference/js/ai.promptfeedback.md - title: RequestOptions diff --git a/docs-devsite/ai.md b/docs-devsite/ai.md index 1f8722e98ef..63f01b6c6b6 100644 --- a/docs-devsite/ai.md +++ b/docs-devsite/ai.md @@ -76,12 +76,12 @@ The Firebase AI Web SDK. | [GenerateContentStreamResult](./ai.generatecontentstreamresult.md#generatecontentstreamresult_interface) | Result object returned from [GenerativeModel.generateContentStream()](./ai.generativemodel.md#generativemodelgeneratecontentstream) call. Iterate over stream to get chunks as they come in and/or use the response promise to get the aggregated response when the stream is done. | | [GenerationConfig](./ai.generationconfig.md#generationconfig_interface) | Config options for content-related requests | | [GenerativeContentBlob](./ai.generativecontentblob.md#generativecontentblob_interface) | Interface for sending an image. | -| [HybridParams](./ai.hybridparams.md#hybridparams_interface) | Toggles hybrid inference. | | [GoogleSearch](./ai.googlesearch.md#googlesearch_interface) | Specifies the Google Search configuration. | | [GoogleSearchTool](./ai.googlesearchtool.md#googlesearchtool_interface) | A tool that allows a Gemini model to connect to Google Search to access and incorporate up-to-date information from the web into its responses.Important: If using Grounding with Google Search, you are required to comply with the "Grounding with Google Search" usage requirements for your chosen API provider: [Gemini Developer API](https://ai.google.dev/gemini-api/terms#grounding-with-google-search) or Vertex AI Gemini API (see [Service Terms](https://cloud.google.com/terms/service-terms) section within the Service Specific Terms). | | [GroundingChunk](./ai.groundingchunk.md#groundingchunk_interface) | Represents a chunk of retrieved data that supports a claim in the model's response. This is part of the grounding information provided when grounding is enabled. | | [GroundingMetadata](./ai.groundingmetadata.md#groundingmetadata_interface) | Metadata returned when grounding is enabled.Currently, only Grounding with Google Search is supported (see [GoogleSearchTool](./ai.googlesearchtool.md#googlesearchtool_interface)).Important: If using Grounding with Google Search, you are required to comply with the "Grounding with Google Search" usage requirements for your chosen API provider: [Gemini Developer API](https://ai.google.dev/gemini-api/terms#grounding-with-google-search) or Vertex AI Gemini API (see [Service Terms](https://cloud.google.com/terms/service-terms) section within the Service Specific Terms). | | [GroundingSupport](./ai.groundingsupport.md#groundingsupport_interface) | Provides information about how a specific segment of the model's response is supported by the retrieved grounding chunks. | +| [HybridParams](./ai.hybridparams.md#hybridparams_interface) | Toggles hybrid inference. | | [ImagenGCSImage](./ai.imagengcsimage.md#imagengcsimage_interface) | An image generated by Imagen, stored in a Cloud Storage for Firebase bucket.This feature is not available yet. | | [ImagenGenerationConfig](./ai.imagengenerationconfig.md#imagengenerationconfig_interface) | (Public Preview) Configuration options for generating images with Imagen.See the [documentation](http://firebase.google.com/docs/vertex-ai/generate-images-imagen) for more details. | | [ImagenGenerationResponse](./ai.imagengenerationresponse.md#imagengenerationresponse_interface) | (Public Preview) The response from a request to generate images with Imagen. | @@ -95,7 +95,7 @@ The Firebase AI Web SDK. | [LanguageModelMessage](./ai.languagemodelmessage.md#languagemodelmessage_interface) | | | [LanguageModelMessageContent](./ai.languagemodelmessagecontent.md#languagemodelmessagecontent_interface) | | | [ModalityTokenCount](./ai.modalitytokencount.md#modalitytokencount_interface) | Represents token counting info for a single modality. | -| [ModelParams](./ai.modelparams.md#modelparams_interface) | Params passed to [getGenerativeModel()](./ai.md#getgenerativemodel_80bd839). | +| [ModelParams](./ai.modelparams.md#modelparams_interface) | Params passed to [getGenerativeModel()](./ai.md#getgenerativemodel_c63f46a). | | [ObjectSchemaRequest](./ai.objectschemarequest.md#objectschemarequest_interface) | Interface for JSON parameters in a schema of "object" when not using the Schema.object() helper. | | [OnDeviceParams](./ai.ondeviceparams.md#ondeviceparams_interface) | Encapsulates configuration for on-device inference. | | [PromptFeedback](./ai.promptfeedback.md#promptfeedback_interface) | If the prompt was blocked, this will be populated with blockReason and the relevant safetyRatings. | @@ -111,7 +111,7 @@ The Firebase AI Web SDK. | [Segment](./ai.segment.md#segment_interface) | Represents a specific segment within a [Content](./ai.content.md#content_interface) object, often used to pinpoint the exact location of text or data that grounding information refers to. | | [StartChatParams](./ai.startchatparams.md#startchatparams_interface) | Params for [GenerativeModel.startChat()](./ai.generativemodel.md#generativemodelstartchat). | | [TextPart](./ai.textpart.md#textpart_interface) | Content part interface if the part represents a text string. | -| [ThinkingConfig](./ai.thinkingconfig.md#thinkingconfig_interface) | Configuration for "thinking" behavior of compatible Gemini models.Certain models utilize a thinking process before generating a response. This allows them to reason through complex problems and plan a more coherent and accurate answer. | +| [ThinkingConfig](./ai.thinkingconfig.md#thinkingconfig_interface) | | | [ToolConfig](./ai.toolconfig.md#toolconfig_interface) | Tool config. This config is shared for all tools provided in the request. | | [UsageMetadata](./ai.usagemetadata.md#usagemetadata_interface) | Usage metadata about a [GenerateContentResponse](./ai.generatecontentresponse.md#generatecontentresponse_interface). | | [VideoMetadata](./ai.videometadata.md#videometadata_interface) | Describes the input video content. | @@ -157,6 +157,10 @@ The Firebase AI Web SDK. | [ImagenAspectRatio](./ai.md#imagenaspectratio) | (Public Preview) Aspect ratios for Imagen images.To specify an aspect ratio for generated images, set the aspectRatio property in your [ImagenGenerationConfig](./ai.imagengenerationconfig.md#imagengenerationconfig_interface).See the [documentation](http://firebase.google.com/docs/vertex-ai/generate-images) for more details and examples of the supported aspect ratios. | | [ImagenPersonFilterLevel](./ai.md#imagenpersonfilterlevel) | (Public Preview) A filter level controlling whether generation of images containing people or faces is allowed.See the personGeneration documentation for more details. | | [ImagenSafetyFilterLevel](./ai.md#imagensafetyfilterlevel) | (Public Preview) A filter level controlling how aggressively to filter sensitive content.Text prompts provided as inputs and images (generated or uploaded) through Imagen on Vertex AI are assessed against a list of safety filters, which include 'harmful categories' (for example, violence, sexual, derogatory, and toxic). This filter level controls how aggressively to filter out potentially harmful content from responses. See the [documentation](http://firebase.google.com/docs/vertex-ai/generate-images) and the [Responsible AI and usage guidelines](https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen#safety-filters) for more details. | +| [InferenceMode](./ai.md#inferencemode) | Determines whether inference happens on-device or in-cloud. | +| [LanguageModelMessageContentValue](./ai.md#languagemodelmessagecontentvalue) | | +| [LanguageModelMessageRole](./ai.md#languagemodelmessagerole) | | +| [LanguageModelMessageType](./ai.md#languagemodelmessagetype) | | | [Modality](./ai.md#modality) | Content part modality. | | [Part](./ai.md#part) | Content part - includes text, image/video, or function call/response part types. | | [ResponseModality](./ai.md#responsemodality) | (Public Preview) Generation modalities to be returned in generation responses. | @@ -700,6 +704,40 @@ Text prompts provided as inputs and images (generated or uploaded) through Image export type ImagenSafetyFilterLevel = (typeof ImagenSafetyFilterLevel)[keyof typeof ImagenSafetyFilterLevel]; ``` +## InferenceMode + +Determines whether inference happens on-device or in-cloud. + +Signature: + +```typescript +export type InferenceMode = 'prefer_on_device' | 'only_on_device' | 'only_in_cloud'; +``` + +## LanguageModelMessageContentValue + +Signature: + +```typescript +export type LanguageModelMessageContentValue = ImageBitmapSource | AudioBuffer | BufferSource | string; +``` + +## LanguageModelMessageRole + +Signature: + +```typescript +export type LanguageModelMessageRole = 'system' | 'user' | 'assistant'; +``` + +## LanguageModelMessageType + +Signature: + +```typescript +export type LanguageModelMessageType = 'text' | 'image' | 'audio'; +``` + ## Modality Content part modality. diff --git a/docs-devsite/ai.thinkingconfig.md b/docs-devsite/ai.thinkingconfig.md index ec348a20487..92e58e56c4c 100644 --- a/docs-devsite/ai.thinkingconfig.md +++ b/docs-devsite/ai.thinkingconfig.md @@ -10,10 +10,6 @@ https://github.com/firebase/firebase-js-sdk {% endcomment %} # ThinkingConfig interface -Configuration for "thinking" behavior of compatible Gemini models. - -Certain models utilize a thinking process before generating a response. This allows them to reason through complex problems and plan a more coherent and accurate answer. - Signature: ```typescript From e9dc43e89efcfb470181713304995746755288c7 Mon Sep 17 00:00:00 2001 From: Christina Holland Date: Wed, 30 Jul 2025 14:57:27 -0700 Subject: [PATCH 08/20] Make ChromeAdapter optional, fix doc comments --- common/api-review/ai.api.md | 38 +++++-- docs-devsite/_toc.yaml | 4 + docs-devsite/ai.chatsession.md | 4 +- docs-devsite/ai.chromeadapter.md | 101 ++++++++++++++++++ docs-devsite/ai.generativemodel.md | 6 +- docs-devsite/ai.hybridparams.md | 2 +- docs-devsite/ai.imagengcsimage.md | 13 ++- .../ai.languagemodelcreatecoreoptions.md | 1 + docs-devsite/ai.languagemodelcreateoptions.md | 1 + docs-devsite/ai.languagemodelexpected.md | 1 + docs-devsite/ai.languagemodelmessage.md | 1 + .../ai.languagemodelmessagecontent.md | 1 + docs-devsite/ai.languagemodelpromptoptions.md | 32 ++++++ docs-devsite/ai.md | 35 ++++-- docs-devsite/ai.objectschemarequest.md | 2 +- docs-devsite/ai.ondeviceparams.md | 2 +- docs-devsite/ai.thinkingconfig.md | 4 + packages/ai/src/api.ts | 15 ++- packages/ai/src/methods/chat-session.test.ts | 8 +- packages/ai/src/methods/chat-session.ts | 4 +- .../ai/src/methods/chrome-adapter.test.ts | 50 ++++----- packages/ai/src/methods/chrome-adapter.ts | 50 ++++----- packages/ai/src/methods/count-tokens.test.ts | 14 +-- packages/ai/src/methods/count-tokens.ts | 6 +- .../ai/src/methods/generate-content.test.ts | 48 +++------ packages/ai/src/methods/generate-content.ts | 10 +- .../ai/src/models/generative-model.test.ts | 20 ++-- packages/ai/src/models/generative-model.ts | 6 +- packages/ai/src/types/chrome-adapter.ts | 16 +++ packages/ai/src/types/enums.ts | 23 +++- packages/ai/src/types/imagen/responses.ts | 1 + packages/ai/src/types/index.ts | 4 +- packages/ai/src/types/language-model.ts | 49 ++++++++- packages/ai/src/types/requests.ts | 14 +-- packages/ai/src/types/schema.ts | 2 +- 35 files changed, 424 insertions(+), 164 deletions(-) create mode 100644 docs-devsite/ai.chromeadapter.md create mode 100644 docs-devsite/ai.languagemodelpromptoptions.md create mode 100644 packages/ai/src/types/chrome-adapter.ts diff --git a/common/api-review/ai.api.md b/common/api-review/ai.api.md index 7c4b6853def..030e1a899ab 100644 --- a/common/api-review/ai.api.md +++ b/common/api-review/ai.api.md @@ -125,8 +125,7 @@ export class BooleanSchema extends Schema { // @public export class ChatSession { - // Warning: (ae-forgotten-export) The symbol "ChromeAdapter" needs to be exported by the entry point index.d.ts - constructor(apiSettings: ApiSettings, model: string, chromeAdapter: ChromeAdapter, params?: StartChatParams | undefined, requestOptions?: RequestOptions | undefined); + constructor(apiSettings: ApiSettings, model: string, chromeAdapter?: ChromeAdapter | undefined, params?: StartChatParams | undefined, requestOptions?: RequestOptions | undefined); getHistory(): Promise; // (undocumented) model: string; @@ -138,6 +137,18 @@ export class ChatSession { sendMessageStream(request: string | Array): Promise; } +// @public +export interface ChromeAdapter { + // (undocumented) + countTokens(_request: CountTokensRequest): Promise; + // (undocumented) + generateContent(request: GenerateContentRequest): Promise; + // (undocumented) + generateContentStream(request: GenerateContentRequest): Promise; + // (undocumented) + isAvailable(request: GenerateContentRequest): Promise; +} + // @public export interface Citation { // (undocumented) @@ -417,7 +428,7 @@ export interface GenerativeContentBlob { // @public export class GenerativeModel extends AIModel { - constructor(ai: AI, modelParams: ModelParams, chromeAdapter: ChromeAdapter, requestOptions?: RequestOptions); + constructor(ai: AI, modelParams: ModelParams, requestOptions?: RequestOptions, chromeAdapter?: ChromeAdapter | undefined); countTokens(request: CountTokensRequest | string | Array): Promise; static DEFAULT_HYBRID_IN_CLOUD_MODEL: string; generateContent(request: GenerateContentRequest | string | Array): Promise; @@ -609,7 +620,7 @@ export const ImagenAspectRatio: { // @beta export type ImagenAspectRatio = (typeof ImagenAspectRatio)[keyof typeof ImagenAspectRatio]; -// @public +// @beta export interface ImagenGCSImage { gcsURI: string; mimeType: string; @@ -691,7 +702,14 @@ export interface ImagenSafetySettings { } // @public -export type InferenceMode = 'prefer_on_device' | 'only_on_device' | 'only_in_cloud'; +export const InferenceMode: { + readonly PREFER_ON_DEVICE: "prefer_on_device"; + readonly ONLY_ON_DEVICE: "only_on_device"; + readonly ONLY_IN_CLOUD: "only_in_cloud"; +}; + +// @public +export type InferenceMode = (typeof InferenceMode)[keyof typeof InferenceMode]; // @public export interface InlineDataPart { @@ -762,6 +780,12 @@ export type LanguageModelMessageRole = 'system' | 'user' | 'assistant'; // @public (undocumented) export type LanguageModelMessageType = 'text' | 'image' | 'audio'; +// @public (undocumented) +export interface LanguageModelPromptOptions { + // (undocumented) + responseConstraint?: object; +} + // @public export const Modality: { readonly MODALITY_UNSPECIFIED: "MODALITY_UNSPECIFIED"; @@ -824,8 +848,6 @@ export interface ObjectSchemaRequest extends SchemaRequest { export interface OnDeviceParams { // (undocumented) createOptions?: LanguageModelCreateOptions; - // Warning: (ae-forgotten-export) The symbol "LanguageModelPromptOptions" needs to be exported by the entry point index.d.ts - // // (undocumented) promptOptions?: LanguageModelPromptOptions; } @@ -1032,7 +1054,7 @@ export interface TextPart { text: string; } -// @public (undocumented) +// @public export interface ThinkingConfig { thinkingBudget?: number; } diff --git a/docs-devsite/_toc.yaml b/docs-devsite/_toc.yaml index fd9ae62b0d9..da7c2500894 100644 --- a/docs-devsite/_toc.yaml +++ b/docs-devsite/_toc.yaml @@ -24,6 +24,8 @@ toc: path: /docs/reference/js/ai.booleanschema.md - title: ChatSession path: /docs/reference/js/ai.chatsession.md + - title: ChromeAdapter + path: /docs/reference/js/ai.chromeadapter.md - title: Citation path: /docs/reference/js/ai.citation.md - title: CitationMetadata @@ -120,6 +122,8 @@ toc: path: /docs/reference/js/ai.languagemodelmessage.md - title: LanguageModelMessageContent path: /docs/reference/js/ai.languagemodelmessagecontent.md + - title: LanguageModelPromptOptions + path: /docs/reference/js/ai.languagemodelpromptoptions.md - title: ModalityTokenCount path: /docs/reference/js/ai.modalitytokencount.md - title: ModelParams diff --git a/docs-devsite/ai.chatsession.md b/docs-devsite/ai.chatsession.md index 610fb2274dd..4e4358898a5 100644 --- a/docs-devsite/ai.chatsession.md +++ b/docs-devsite/ai.chatsession.md @@ -47,7 +47,7 @@ Constructs a new instance of the `ChatSession` class Signature: ```typescript -constructor(apiSettings: ApiSettings, model: string, chromeAdapter: ChromeAdapter, params?: StartChatParams | undefined, requestOptions?: RequestOptions | undefined); +constructor(apiSettings: ApiSettings, model: string, chromeAdapter?: ChromeAdapter | undefined, params?: StartChatParams | undefined, requestOptions?: RequestOptions | undefined); ``` #### Parameters @@ -56,7 +56,7 @@ constructor(apiSettings: ApiSettings, model: string, chromeAdapter: ChromeAdapte | --- | --- | --- | | apiSettings | ApiSettings | | | model | string | | -| chromeAdapter | ChromeAdapter | | +| chromeAdapter | [ChromeAdapter](./ai.chromeadapter.md#chromeadapter_interface) \| undefined | | | params | [StartChatParams](./ai.startchatparams.md#startchatparams_interface) \| undefined | | | requestOptions | [RequestOptions](./ai.requestoptions.md#requestoptions_interface) \| undefined | | diff --git a/docs-devsite/ai.chromeadapter.md b/docs-devsite/ai.chromeadapter.md new file mode 100644 index 00000000000..395a71c79d2 --- /dev/null +++ b/docs-devsite/ai.chromeadapter.md @@ -0,0 +1,101 @@ +Project: /docs/reference/js/_project.yaml +Book: /docs/reference/_book.yaml +page_type: reference + +{% comment %} +DO NOT EDIT THIS FILE! +This is generated by the JS SDK team, and any local changes will be +overwritten. Changes should be made in the source code at +https://github.com/firebase/firebase-js-sdk +{% endcomment %} + +# ChromeAdapter interface +Defines an inference "backend" that uses Chrome's on-device model, and encapsulates logic for detecting when on-device is possible. + +Signature: + +```typescript +export interface ChromeAdapter +``` + +## Methods + +| Method | Description | +| --- | --- | +| [countTokens(\_request)](./ai.chromeadapter.md#chromeadaptercounttokens) | | +| [generateContent(request)](./ai.chromeadapter.md#chromeadaptergeneratecontent) | | +| [generateContentStream(request)](./ai.chromeadapter.md#chromeadaptergeneratecontentstream) | | +| [isAvailable(request)](./ai.chromeadapter.md#chromeadapterisavailable) | | + +## ChromeAdapter.countTokens() + +Signature: + +```typescript +countTokens(_request: CountTokensRequest): Promise; +``` + +#### Parameters + +| Parameter | Type | Description | +| --- | --- | --- | +| \_request | [CountTokensRequest](./ai.counttokensrequest.md#counttokensrequest_interface) | | + +Returns: + +Promise<Response> + +## ChromeAdapter.generateContent() + +Signature: + +```typescript +generateContent(request: GenerateContentRequest): Promise; +``` + +#### Parameters + +| Parameter | Type | Description | +| --- | --- | --- | +| request | [GenerateContentRequest](./ai.generatecontentrequest.md#generatecontentrequest_interface) | | + +Returns: + +Promise<Response> + +## ChromeAdapter.generateContentStream() + +Signature: + +```typescript +generateContentStream(request: GenerateContentRequest): Promise; +``` + +#### Parameters + +| Parameter | Type | Description | +| --- | --- | --- | +| request | [GenerateContentRequest](./ai.generatecontentrequest.md#generatecontentrequest_interface) | | + +Returns: + +Promise<Response> + +## ChromeAdapter.isAvailable() + +Signature: + +```typescript +isAvailable(request: GenerateContentRequest): Promise; +``` + +#### Parameters + +| Parameter | Type | Description | +| --- | --- | --- | +| request | [GenerateContentRequest](./ai.generatecontentrequest.md#generatecontentrequest_interface) | | + +Returns: + +Promise<boolean> + diff --git a/docs-devsite/ai.generativemodel.md b/docs-devsite/ai.generativemodel.md index 17c9d3c0863..fa29218e60e 100644 --- a/docs-devsite/ai.generativemodel.md +++ b/docs-devsite/ai.generativemodel.md @@ -23,7 +23,7 @@ export declare class GenerativeModel extends AIModel | Constructor | Modifiers | Description | | --- | --- | --- | -| [(constructor)(ai, modelParams, chromeAdapter, requestOptions)](./ai.generativemodel.md#generativemodelconstructor) | | Constructs a new instance of the GenerativeModel class | +| [(constructor)(ai, modelParams, requestOptions, chromeAdapter)](./ai.generativemodel.md#generativemodelconstructor) | | Constructs a new instance of the GenerativeModel class | ## Properties @@ -53,7 +53,7 @@ Constructs a new instance of the `GenerativeModel` class Signature: ```typescript -constructor(ai: AI, modelParams: ModelParams, chromeAdapter: ChromeAdapter, requestOptions?: RequestOptions); +constructor(ai: AI, modelParams: ModelParams, requestOptions?: RequestOptions, chromeAdapter?: ChromeAdapter | undefined); ``` #### Parameters @@ -62,8 +62,8 @@ constructor(ai: AI, modelParams: ModelParams, chromeAdapter: ChromeAdapter, requ | --- | --- | --- | | ai | [AI](./ai.ai.md#ai_interface) | | | modelParams | [ModelParams](./ai.modelparams.md#modelparams_interface) | | -| chromeAdapter | ChromeAdapter | | | requestOptions | [RequestOptions](./ai.requestoptions.md#requestoptions_interface) | | +| chromeAdapter | [ChromeAdapter](./ai.chromeadapter.md#chromeadapter_interface) \| undefined | | ## GenerativeModel.DEFAULT\_HYBRID\_IN\_CLOUD\_MODEL diff --git a/docs-devsite/ai.hybridparams.md b/docs-devsite/ai.hybridparams.md index b2b3b1030fe..1934b68597f 100644 --- a/docs-devsite/ai.hybridparams.md +++ b/docs-devsite/ai.hybridparams.md @@ -10,7 +10,7 @@ https://github.com/firebase/firebase-js-sdk {% endcomment %} # HybridParams interface -Toggles hybrid inference. +Configures hybrid inference. Signature: diff --git a/docs-devsite/ai.imagengcsimage.md b/docs-devsite/ai.imagengcsimage.md index cd11d8ee354..ec51c714e0f 100644 --- a/docs-devsite/ai.imagengcsimage.md +++ b/docs-devsite/ai.imagengcsimage.md @@ -10,6 +10,9 @@ https://github.com/firebase/firebase-js-sdk {% endcomment %} # ImagenGCSImage interface +> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment. +> + An image generated by Imagen, stored in a Cloud Storage for Firebase bucket. This feature is not available yet. @@ -24,11 +27,14 @@ export interface ImagenGCSImage | Property | Type | Description | | --- | --- | --- | -| [gcsURI](./ai.imagengcsimage.md#imagengcsimagegcsuri) | string | The URI of the file stored in a Cloud Storage for Firebase bucket. | -| [mimeType](./ai.imagengcsimage.md#imagengcsimagemimetype) | string | The MIME type of the image; either "image/png" or "image/jpeg".To request a different format, set the imageFormat property in your [ImagenGenerationConfig](./ai.imagengenerationconfig.md#imagengenerationconfig_interface). | +| [gcsURI](./ai.imagengcsimage.md#imagengcsimagegcsuri) | string | (Public Preview) The URI of the file stored in a Cloud Storage for Firebase bucket. | +| [mimeType](./ai.imagengcsimage.md#imagengcsimagemimetype) | string | (Public Preview) The MIME type of the image; either "image/png" or "image/jpeg".To request a different format, set the imageFormat property in your [ImagenGenerationConfig](./ai.imagengenerationconfig.md#imagengenerationconfig_interface). | ## ImagenGCSImage.gcsURI +> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment. +> + The URI of the file stored in a Cloud Storage for Firebase bucket. Signature: @@ -43,6 +49,9 @@ gcsURI: string; ## ImagenGCSImage.mimeType +> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment. +> + The MIME type of the image; either `"image/png"` or `"image/jpeg"`. To request a different format, set the `imageFormat` property in your [ImagenGenerationConfig](./ai.imagengenerationconfig.md#imagengenerationconfig_interface). diff --git a/docs-devsite/ai.languagemodelcreatecoreoptions.md b/docs-devsite/ai.languagemodelcreatecoreoptions.md index 45c2e7f5db4..832ea3b8ca8 100644 --- a/docs-devsite/ai.languagemodelcreatecoreoptions.md +++ b/docs-devsite/ai.languagemodelcreatecoreoptions.md @@ -10,6 +10,7 @@ https://github.com/firebase/firebase-js-sdk {% endcomment %} # LanguageModelCreateCoreOptions interface + Signature: ```typescript diff --git a/docs-devsite/ai.languagemodelcreateoptions.md b/docs-devsite/ai.languagemodelcreateoptions.md index 417519a54b6..54d1ecaa803 100644 --- a/docs-devsite/ai.languagemodelcreateoptions.md +++ b/docs-devsite/ai.languagemodelcreateoptions.md @@ -10,6 +10,7 @@ https://github.com/firebase/firebase-js-sdk {% endcomment %} # LanguageModelCreateOptions interface + Signature: ```typescript diff --git a/docs-devsite/ai.languagemodelexpected.md b/docs-devsite/ai.languagemodelexpected.md index 26ed28b741e..e33d922007c 100644 --- a/docs-devsite/ai.languagemodelexpected.md +++ b/docs-devsite/ai.languagemodelexpected.md @@ -10,6 +10,7 @@ https://github.com/firebase/firebase-js-sdk {% endcomment %} # LanguageModelExpected interface + Signature: ```typescript diff --git a/docs-devsite/ai.languagemodelmessage.md b/docs-devsite/ai.languagemodelmessage.md index 420059e4892..efedf369945 100644 --- a/docs-devsite/ai.languagemodelmessage.md +++ b/docs-devsite/ai.languagemodelmessage.md @@ -10,6 +10,7 @@ https://github.com/firebase/firebase-js-sdk {% endcomment %} # LanguageModelMessage interface + Signature: ```typescript diff --git a/docs-devsite/ai.languagemodelmessagecontent.md b/docs-devsite/ai.languagemodelmessagecontent.md index 40b4cc16bce..b87f8a28b3a 100644 --- a/docs-devsite/ai.languagemodelmessagecontent.md +++ b/docs-devsite/ai.languagemodelmessagecontent.md @@ -10,6 +10,7 @@ https://github.com/firebase/firebase-js-sdk {% endcomment %} # LanguageModelMessageContent interface + Signature: ```typescript diff --git a/docs-devsite/ai.languagemodelpromptoptions.md b/docs-devsite/ai.languagemodelpromptoptions.md new file mode 100644 index 00000000000..cde9b9af3be --- /dev/null +++ b/docs-devsite/ai.languagemodelpromptoptions.md @@ -0,0 +1,32 @@ +Project: /docs/reference/js/_project.yaml +Book: /docs/reference/_book.yaml +page_type: reference + +{% comment %} +DO NOT EDIT THIS FILE! +This is generated by the JS SDK team, and any local changes will be +overwritten. Changes should be made in the source code at +https://github.com/firebase/firebase-js-sdk +{% endcomment %} + +# LanguageModelPromptOptions interface + +Signature: + +```typescript +export interface LanguageModelPromptOptions +``` + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [responseConstraint](./ai.languagemodelpromptoptions.md#languagemodelpromptoptionsresponseconstraint) | object | | + +## LanguageModelPromptOptions.responseConstraint + +Signature: + +```typescript +responseConstraint?: object; +``` diff --git a/docs-devsite/ai.md b/docs-devsite/ai.md index 63f01b6c6b6..464c14bf1be 100644 --- a/docs-devsite/ai.md +++ b/docs-devsite/ai.md @@ -51,6 +51,7 @@ The Firebase AI Web SDK. | [AI](./ai.ai.md#ai_interface) | An instance of the Firebase AI SDK.Do not create this instance directly. Instead, use [getAI()](./ai.md#getai_a94a413). | | [AIOptions](./ai.aioptions.md#aioptions_interface) | Options for initializing the AI service using [getAI()](./ai.md#getai_a94a413). This allows specifying which backend to use (Vertex AI Gemini API or Gemini Developer API) and configuring its specific options (like location for Vertex AI). | | [BaseParams](./ai.baseparams.md#baseparams_interface) | Base parameters for a number of methods. | +| [ChromeAdapter](./ai.chromeadapter.md#chromeadapter_interface) | Defines an inference "backend" that uses Chrome's on-device model, and encapsulates logic for detecting when on-device is possible. | | [Citation](./ai.citation.md#citation_interface) | A single citation. | | [CitationMetadata](./ai.citationmetadata.md#citationmetadata_interface) | Citation metadata that may be found on a [GenerateContentCandidate](./ai.generatecontentcandidate.md#generatecontentcandidate_interface). | | [Content](./ai.content.md#content_interface) | Content type for both prompts and response candidates. | @@ -81,8 +82,8 @@ The Firebase AI Web SDK. | [GroundingChunk](./ai.groundingchunk.md#groundingchunk_interface) | Represents a chunk of retrieved data that supports a claim in the model's response. This is part of the grounding information provided when grounding is enabled. | | [GroundingMetadata](./ai.groundingmetadata.md#groundingmetadata_interface) | Metadata returned when grounding is enabled.Currently, only Grounding with Google Search is supported (see [GoogleSearchTool](./ai.googlesearchtool.md#googlesearchtool_interface)).Important: If using Grounding with Google Search, you are required to comply with the "Grounding with Google Search" usage requirements for your chosen API provider: [Gemini Developer API](https://ai.google.dev/gemini-api/terms#grounding-with-google-search) or Vertex AI Gemini API (see [Service Terms](https://cloud.google.com/terms/service-terms) section within the Service Specific Terms). | | [GroundingSupport](./ai.groundingsupport.md#groundingsupport_interface) | Provides information about how a specific segment of the model's response is supported by the retrieved grounding chunks. | -| [HybridParams](./ai.hybridparams.md#hybridparams_interface) | Toggles hybrid inference. | -| [ImagenGCSImage](./ai.imagengcsimage.md#imagengcsimage_interface) | An image generated by Imagen, stored in a Cloud Storage for Firebase bucket.This feature is not available yet. | +| [HybridParams](./ai.hybridparams.md#hybridparams_interface) | Configures hybrid inference. | +| [ImagenGCSImage](./ai.imagengcsimage.md#imagengcsimage_interface) | (Public Preview) An image generated by Imagen, stored in a Cloud Storage for Firebase bucket.This feature is not available yet. | | [ImagenGenerationConfig](./ai.imagengenerationconfig.md#imagengenerationconfig_interface) | (Public Preview) Configuration options for generating images with Imagen.See the [documentation](http://firebase.google.com/docs/vertex-ai/generate-images-imagen) for more details. | | [ImagenGenerationResponse](./ai.imagengenerationresponse.md#imagengenerationresponse_interface) | (Public Preview) The response from a request to generate images with Imagen. | | [ImagenInlineImage](./ai.imageninlineimage.md#imageninlineimage_interface) | (Public Preview) An image generated by Imagen, represented as inline data. | @@ -94,9 +95,10 @@ The Firebase AI Web SDK. | [LanguageModelExpected](./ai.languagemodelexpected.md#languagemodelexpected_interface) | | | [LanguageModelMessage](./ai.languagemodelmessage.md#languagemodelmessage_interface) | | | [LanguageModelMessageContent](./ai.languagemodelmessagecontent.md#languagemodelmessagecontent_interface) | | +| [LanguageModelPromptOptions](./ai.languagemodelpromptoptions.md#languagemodelpromptoptions_interface) | | | [ModalityTokenCount](./ai.modalitytokencount.md#modalitytokencount_interface) | Represents token counting info for a single modality. | | [ModelParams](./ai.modelparams.md#modelparams_interface) | Params passed to [getGenerativeModel()](./ai.md#getgenerativemodel_c63f46a). | -| [ObjectSchemaRequest](./ai.objectschemarequest.md#objectschemarequest_interface) | Interface for JSON parameters in a schema of "object" when not using the Schema.object() helper. | +| [ObjectSchemaRequest](./ai.objectschemarequest.md#objectschemarequest_interface) | Interface for JSON parameters in a schema of [SchemaType](./ai.md#schematype) "object" when not using the Schema.object() helper. | | [OnDeviceParams](./ai.ondeviceparams.md#ondeviceparams_interface) | Encapsulates configuration for on-device inference. | | [PromptFeedback](./ai.promptfeedback.md#promptfeedback_interface) | If the prompt was blocked, this will be populated with blockReason and the relevant safetyRatings. | | [RequestOptions](./ai.requestoptions.md#requestoptions_interface) | Params passed to [getGenerativeModel()](./ai.md#getgenerativemodel_c63f46a). | @@ -111,7 +113,7 @@ The Firebase AI Web SDK. | [Segment](./ai.segment.md#segment_interface) | Represents a specific segment within a [Content](./ai.content.md#content_interface) object, often used to pinpoint the exact location of text or data that grounding information refers to. | | [StartChatParams](./ai.startchatparams.md#startchatparams_interface) | Params for [GenerativeModel.startChat()](./ai.generativemodel.md#generativemodelstartchat). | | [TextPart](./ai.textpart.md#textpart_interface) | Content part interface if the part represents a text string. | -| [ThinkingConfig](./ai.thinkingconfig.md#thinkingconfig_interface) | | +| [ThinkingConfig](./ai.thinkingconfig.md#thinkingconfig_interface) | Configuration for "thinking" behavior of compatible Gemini models.Certain models utilize a thinking process before generating a response. This allows them to reason through complex problems and plan a more coherent and accurate answer. | | [ToolConfig](./ai.toolconfig.md#toolconfig_interface) | Tool config. This config is shared for all tools provided in the request. | | [UsageMetadata](./ai.usagemetadata.md#usagemetadata_interface) | Usage metadata about a [GenerateContentResponse](./ai.generatecontentresponse.md#generatecontentresponse_interface). | | [VideoMetadata](./ai.videometadata.md#videometadata_interface) | Describes the input video content. | @@ -135,6 +137,7 @@ The Firebase AI Web SDK. | [ImagenAspectRatio](./ai.md#imagenaspectratio) | (Public Preview) Aspect ratios for Imagen images.To specify an aspect ratio for generated images, set the aspectRatio property in your [ImagenGenerationConfig](./ai.imagengenerationconfig.md#imagengenerationconfig_interface).See the [documentation](http://firebase.google.com/docs/vertex-ai/generate-images) for more details and examples of the supported aspect ratios. | | [ImagenPersonFilterLevel](./ai.md#imagenpersonfilterlevel) | (Public Preview) A filter level controlling whether generation of images containing people or faces is allowed.See the personGeneration documentation for more details. | | [ImagenSafetyFilterLevel](./ai.md#imagensafetyfilterlevel) | (Public Preview) A filter level controlling how aggressively to filter sensitive content.Text prompts provided as inputs and images (generated or uploaded) through Imagen on Vertex AI are assessed against a list of safety filters, which include 'harmful categories' (for example, violence, sexual, derogatory, and toxic). This filter level controls how aggressively to filter out potentially harmful content from responses. See the [documentation](http://firebase.google.com/docs/vertex-ai/generate-images) and the [Responsible AI and usage guidelines](https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen#safety-filters) for more details. | +| [InferenceMode](./ai.md#inferencemode) | EXPERIMENTAL FEATURE Determines whether inference happens on-device or in-cloud. | | [Modality](./ai.md#modality) | Content part modality. | | [POSSIBLE\_ROLES](./ai.md#possible_roles) | Possible roles. | | [ResponseModality](./ai.md#responsemodality) | (Public Preview) Generation modalities to be returned in generation responses. | @@ -157,7 +160,7 @@ The Firebase AI Web SDK. | [ImagenAspectRatio](./ai.md#imagenaspectratio) | (Public Preview) Aspect ratios for Imagen images.To specify an aspect ratio for generated images, set the aspectRatio property in your [ImagenGenerationConfig](./ai.imagengenerationconfig.md#imagengenerationconfig_interface).See the [documentation](http://firebase.google.com/docs/vertex-ai/generate-images) for more details and examples of the supported aspect ratios. | | [ImagenPersonFilterLevel](./ai.md#imagenpersonfilterlevel) | (Public Preview) A filter level controlling whether generation of images containing people or faces is allowed.See the personGeneration documentation for more details. | | [ImagenSafetyFilterLevel](./ai.md#imagensafetyfilterlevel) | (Public Preview) A filter level controlling how aggressively to filter sensitive content.Text prompts provided as inputs and images (generated or uploaded) through Imagen on Vertex AI are assessed against a list of safety filters, which include 'harmful categories' (for example, violence, sexual, derogatory, and toxic). This filter level controls how aggressively to filter out potentially harmful content from responses. See the [documentation](http://firebase.google.com/docs/vertex-ai/generate-images) and the [Responsible AI and usage guidelines](https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen#safety-filters) for more details. | -| [InferenceMode](./ai.md#inferencemode) | Determines whether inference happens on-device or in-cloud. | +| [InferenceMode](./ai.md#inferencemode) | EXPERIMENTAL FEATURE Determines whether inference happens on-device or in-cloud. | | [LanguageModelMessageContentValue](./ai.md#languagemodelmessagecontentvalue) | | | [LanguageModelMessageRole](./ai.md#languagemodelmessagerole) | | | [LanguageModelMessageType](./ai.md#languagemodelmessagetype) | | @@ -499,6 +502,20 @@ ImagenSafetyFilterLevel: { } ``` +## InferenceMode + +EXPERIMENTAL FEATURE Determines whether inference happens on-device or in-cloud. + +Signature: + +```typescript +InferenceMode: { + readonly PREFER_ON_DEVICE: "prefer_on_device"; + readonly ONLY_ON_DEVICE: "only_on_device"; + readonly ONLY_IN_CLOUD: "only_in_cloud"; +} +``` + ## Modality Content part modality. @@ -601,6 +618,7 @@ export type FinishReason = (typeof FinishReason)[keyof typeof FinishReason]; ## FunctionCallingMode + Signature: ```typescript @@ -706,16 +724,17 @@ export type ImagenSafetyFilterLevel = (typeof ImagenSafetyFilterLevel)[keyof typ ## InferenceMode -Determines whether inference happens on-device or in-cloud. +EXPERIMENTAL FEATURE Determines whether inference happens on-device or in-cloud. Signature: ```typescript -export type InferenceMode = 'prefer_on_device' | 'only_on_device' | 'only_in_cloud'; +export type InferenceMode = (typeof InferenceMode)[keyof typeof InferenceMode]; ``` ## LanguageModelMessageContentValue + Signature: ```typescript @@ -724,6 +743,7 @@ export type LanguageModelMessageContentValue = ImageBitmapSource | AudioBuffer | ## LanguageModelMessageRole + Signature: ```typescript @@ -732,6 +752,7 @@ export type LanguageModelMessageRole = 'system' | 'user' | 'assistant'; ## LanguageModelMessageType + Signature: ```typescript diff --git a/docs-devsite/ai.objectschemarequest.md b/docs-devsite/ai.objectschemarequest.md index bde646e0ac0..267e2d43345 100644 --- a/docs-devsite/ai.objectschemarequest.md +++ b/docs-devsite/ai.objectschemarequest.md @@ -10,7 +10,7 @@ https://github.com/firebase/firebase-js-sdk {% endcomment %} # ObjectSchemaRequest interface -Interface for JSON parameters in a schema of "object" when not using the `Schema.object()` helper. +Interface for JSON parameters in a schema of [SchemaType](./ai.md#schematype) "object" when not using the `Schema.object()` helper. Signature: diff --git a/docs-devsite/ai.ondeviceparams.md b/docs-devsite/ai.ondeviceparams.md index 16fed65560d..0e23d1fda98 100644 --- a/docs-devsite/ai.ondeviceparams.md +++ b/docs-devsite/ai.ondeviceparams.md @@ -23,7 +23,7 @@ export interface OnDeviceParams | Property | Type | Description | | --- | --- | --- | | [createOptions](./ai.ondeviceparams.md#ondeviceparamscreateoptions) | [LanguageModelCreateOptions](./ai.languagemodelcreateoptions.md#languagemodelcreateoptions_interface) | | -| [promptOptions](./ai.ondeviceparams.md#ondeviceparamspromptoptions) | LanguageModelPromptOptions | | +| [promptOptions](./ai.ondeviceparams.md#ondeviceparamspromptoptions) | [LanguageModelPromptOptions](./ai.languagemodelpromptoptions.md#languagemodelpromptoptions_interface) | | ## OnDeviceParams.createOptions diff --git a/docs-devsite/ai.thinkingconfig.md b/docs-devsite/ai.thinkingconfig.md index 92e58e56c4c..ec348a20487 100644 --- a/docs-devsite/ai.thinkingconfig.md +++ b/docs-devsite/ai.thinkingconfig.md @@ -10,6 +10,10 @@ https://github.com/firebase/firebase-js-sdk {% endcomment %} # ThinkingConfig interface +Configuration for "thinking" behavior of compatible Gemini models. + +Certain models utilize a thinking process before generating a response. This allows them to reason through complex problems and plan a more coherent and accurate answer. + Signature: ```typescript diff --git a/packages/ai/src/api.ts b/packages/ai/src/api.ts index 6f97933efa1..6ae0acadf20 100644 --- a/packages/ai/src/api.ts +++ b/packages/ai/src/api.ts @@ -32,7 +32,7 @@ import { AIError } from './errors'; import { AIModel, GenerativeModel, ImagenModel } from './models'; import { encodeInstanceIdentifier } from './helpers'; import { GoogleAIBackend } from './backend'; -import { ChromeAdapter } from './methods/chrome-adapter'; +import { ChromeAdapterImpl } from './methods/chrome-adapter'; import { LanguageModel } from './types/language-model'; export { ChatSession } from './methods/chat-session'; @@ -117,16 +117,15 @@ export function getGenerativeModel( `Must provide a model name. Example: getGenerativeModel({ model: 'my-model-name' })` ); } - return new GenerativeModel( - ai, - inCloudParams, - new ChromeAdapter( + let chromeAdapter: ChromeAdapterImpl | undefined; + if (typeof window !== 'undefined' && hybridParams.mode) { + chromeAdapter = new ChromeAdapterImpl( window.LanguageModel as LanguageModel, hybridParams.mode, hybridParams.onDeviceParams - ), - requestOptions - ); + ); + } + return new GenerativeModel(ai, inCloudParams, requestOptions, chromeAdapter); } /** diff --git a/packages/ai/src/methods/chat-session.test.ts b/packages/ai/src/methods/chat-session.test.ts index ed0b4d4877f..0883920847f 100644 --- a/packages/ai/src/methods/chat-session.test.ts +++ b/packages/ai/src/methods/chat-session.test.ts @@ -24,7 +24,7 @@ import { GenerateContentStreamResult } from '../types'; import { ChatSession } from './chat-session'; import { ApiSettings } from '../types/internal'; import { VertexAIBackend } from '../backend'; -import { ChromeAdapter } from './chrome-adapter'; +import { ChromeAdapterImpl } from './chrome-adapter'; use(sinonChai); use(chaiAsPromised); @@ -50,7 +50,7 @@ describe('ChatSession', () => { const chatSession = new ChatSession( fakeApiSettings, 'a-model', - new ChromeAdapter() + new ChromeAdapterImpl() ); await expect(chatSession.sendMessage('hello')).to.be.rejected; expect(generateContentStub).to.be.calledWith( @@ -71,7 +71,7 @@ describe('ChatSession', () => { const chatSession = new ChatSession( fakeApiSettings, 'a-model', - new ChromeAdapter() + new ChromeAdapterImpl() ); await expect(chatSession.sendMessageStream('hello')).to.be.rejected; expect(generateContentStreamStub).to.be.calledWith( @@ -94,7 +94,7 @@ describe('ChatSession', () => { const chatSession = new ChatSession( fakeApiSettings, 'a-model', - new ChromeAdapter() + new ChromeAdapterImpl() ); await chatSession.sendMessageStream('hello'); expect(generateContentStreamStub).to.be.calledWith( diff --git a/packages/ai/src/methods/chat-session.ts b/packages/ai/src/methods/chat-session.ts index 112ddf5857e..dac16430b7a 100644 --- a/packages/ai/src/methods/chat-session.ts +++ b/packages/ai/src/methods/chat-session.ts @@ -30,7 +30,7 @@ import { validateChatHistory } from './chat-session-helpers'; import { generateContent, generateContentStream } from './generate-content'; import { ApiSettings } from '../types/internal'; import { logger } from '../logger'; -import { ChromeAdapter } from './chrome-adapter'; +import { ChromeAdapter } from '../types/chrome-adapter'; /** * Do not log a message for this error. @@ -51,7 +51,7 @@ export class ChatSession { constructor( apiSettings: ApiSettings, public model: string, - private chromeAdapter: ChromeAdapter, + private chromeAdapter?: ChromeAdapter, public params?: StartChatParams, public requestOptions?: RequestOptions ) { diff --git a/packages/ai/src/methods/chrome-adapter.test.ts b/packages/ai/src/methods/chrome-adapter.test.ts index f8ea80b0e09..fdc84be71be 100644 --- a/packages/ai/src/methods/chrome-adapter.test.ts +++ b/packages/ai/src/methods/chrome-adapter.test.ts @@ -19,7 +19,7 @@ import { AIError } from '../errors'; import { expect, use } from 'chai'; import sinonChai from 'sinon-chai'; import chaiAsPromised from 'chai-as-promised'; -import { ChromeAdapter } from './chrome-adapter'; +import { ChromeAdapterImpl } from './chrome-adapter'; import { Availability, LanguageModel, @@ -62,7 +62,7 @@ describe('ChromeAdapter', () => { languageModelProvider, 'availability' ).resolves(Availability.available); - const adapter = new ChromeAdapter( + const adapter = new ChromeAdapterImpl( languageModelProvider, 'prefer_on_device' ); @@ -90,7 +90,7 @@ describe('ChromeAdapter', () => { // Explicitly sets expected inputs. expectedInputs: [{ type: 'text' }] } as LanguageModelCreateOptions; - const adapter = new ChromeAdapter( + const adapter = new ChromeAdapterImpl( languageModelProvider, 'prefer_on_device', { @@ -110,7 +110,7 @@ describe('ChromeAdapter', () => { }); describe('isAvailable', () => { it('returns false if mode is undefined', async () => { - const adapter = new ChromeAdapter(); + const adapter = new ChromeAdapterImpl(); expect( await adapter.isAvailable({ contents: [] @@ -118,7 +118,7 @@ describe('ChromeAdapter', () => { ).to.be.false; }); it('returns false if mode is only cloud', async () => { - const adapter = new ChromeAdapter(undefined, 'only_in_cloud'); + const adapter = new ChromeAdapterImpl(undefined, 'only_in_cloud'); expect( await adapter.isAvailable({ contents: [] @@ -126,7 +126,7 @@ describe('ChromeAdapter', () => { ).to.be.false; }); it('returns false if LanguageModel API is undefined', async () => { - const adapter = new ChromeAdapter(undefined, 'prefer_on_device'); + const adapter = new ChromeAdapterImpl(undefined, 'prefer_on_device'); expect( await adapter.isAvailable({ contents: [] @@ -134,7 +134,7 @@ describe('ChromeAdapter', () => { ).to.be.false; }); it('returns false if request contents empty', async () => { - const adapter = new ChromeAdapter( + const adapter = new ChromeAdapterImpl( { availability: async () => Availability.available } as LanguageModel, @@ -147,7 +147,7 @@ describe('ChromeAdapter', () => { ).to.be.false; }); it('returns false if request content has "function" role', async () => { - const adapter = new ChromeAdapter( + const adapter = new ChromeAdapterImpl( { availability: async () => Availability.available } as LanguageModel, @@ -165,13 +165,13 @@ describe('ChromeAdapter', () => { ).to.be.false; }); it('returns true if request has image with supported mime type', async () => { - const adapter = new ChromeAdapter( + const adapter = new ChromeAdapterImpl( { availability: async () => Availability.available } as LanguageModel, 'prefer_on_device' ); - for (const mimeType of ChromeAdapter.SUPPORTED_MIME_TYPES) { + for (const mimeType of ChromeAdapterImpl.SUPPORTED_MIME_TYPES) { expect( await adapter.isAvailable({ contents: [ @@ -195,7 +195,7 @@ describe('ChromeAdapter', () => { const languageModelProvider = { availability: () => Promise.resolve(Availability.available) } as LanguageModel; - const adapter = new ChromeAdapter( + const adapter = new ChromeAdapterImpl( languageModelProvider, 'prefer_on_device' ); @@ -224,7 +224,7 @@ describe('ChromeAdapter', () => { const createOptions = { expectedInputs: [{ type: 'image' }] } as LanguageModelCreateOptions; - const adapter = new ChromeAdapter( + const adapter = new ChromeAdapterImpl( languageModelProvider, 'prefer_on_device', { createOptions } @@ -247,7 +247,7 @@ describe('ChromeAdapter', () => { const createStub = stub(languageModelProvider, 'create').returns( downloadPromise ); - const adapter = new ChromeAdapter( + const adapter = new ChromeAdapterImpl( languageModelProvider, 'prefer_on_device' ); @@ -271,7 +271,7 @@ describe('ChromeAdapter', () => { const createStub = stub(languageModelProvider, 'create').returns( downloadPromise ); - const adapter = new ChromeAdapter( + const adapter = new ChromeAdapterImpl( languageModelProvider, 'prefer_on_device' ); @@ -289,7 +289,7 @@ describe('ChromeAdapter', () => { availability: () => Promise.resolve(Availability.unavailable), create: () => Promise.resolve({}) } as LanguageModel; - const adapter = new ChromeAdapter( + const adapter = new ChromeAdapterImpl( languageModelProvider, 'prefer_on_device' ); @@ -302,7 +302,7 @@ describe('ChromeAdapter', () => { }); describe('generateContent', () => { it('throws if Chrome API is undefined', async () => { - const adapter = new ChromeAdapter(undefined, 'only_on_device'); + const adapter = new ChromeAdapterImpl(undefined, 'only_on_device'); await expect( adapter.generateContent({ contents: [] @@ -331,7 +331,7 @@ describe('ChromeAdapter', () => { systemPrompt: 'be yourself', expectedInputs: [{ type: 'image' }] } as LanguageModelCreateOptions; - const adapter = new ChromeAdapter( + const adapter = new ChromeAdapterImpl( languageModelProvider, 'prefer_on_device', { createOptions } @@ -382,7 +382,7 @@ describe('ChromeAdapter', () => { systemPrompt: 'be yourself', expectedInputs: [{ type: 'image' }] } as LanguageModelCreateOptions; - const adapter = new ChromeAdapter( + const adapter = new ChromeAdapterImpl( languageModelProvider, 'prefer_on_device', { createOptions } @@ -448,7 +448,7 @@ describe('ChromeAdapter', () => { properties: {} }) }; - const adapter = new ChromeAdapter( + const adapter = new ChromeAdapterImpl( languageModelProvider, 'prefer_on_device', { promptOptions } @@ -481,7 +481,7 @@ describe('ChromeAdapter', () => { const languageModelProvider = { create: () => Promise.resolve(languageModel) } as LanguageModel; - const adapter = new ChromeAdapter( + const adapter = new ChromeAdapterImpl( languageModelProvider, 'prefer_on_device' ); @@ -517,7 +517,7 @@ describe('ChromeAdapter', () => { languageModel ); - const adapter = new ChromeAdapter( + const adapter = new ChromeAdapterImpl( languageModelProvider, 'prefer_on_device' ); @@ -561,7 +561,7 @@ describe('ChromeAdapter', () => { const createOptions = { expectedInputs: [{ type: 'image' }] } as LanguageModelCreateOptions; - const adapter = new ChromeAdapter( + const adapter = new ChromeAdapterImpl( languageModelProvider, 'prefer_on_device', { createOptions } @@ -609,7 +609,7 @@ describe('ChromeAdapter', () => { const createOptions = { expectedInputs: [{ type: 'image' }] } as LanguageModelCreateOptions; - const adapter = new ChromeAdapter( + const adapter = new ChromeAdapterImpl( languageModelProvider, 'prefer_on_device', { createOptions } @@ -668,7 +668,7 @@ describe('ChromeAdapter', () => { properties: {} }) }; - const adapter = new ChromeAdapter( + const adapter = new ChromeAdapterImpl( languageModelProvider, 'prefer_on_device', { promptOptions } @@ -703,7 +703,7 @@ describe('ChromeAdapter', () => { const languageModelProvider = { create: () => Promise.resolve(languageModel) } as LanguageModel; - const adapter = new ChromeAdapter( + const adapter = new ChromeAdapterImpl( languageModelProvider, 'prefer_on_device' ); diff --git a/packages/ai/src/methods/chrome-adapter.ts b/packages/ai/src/methods/chrome-adapter.ts index e7bb39c34c8..71a87b55a46 100644 --- a/packages/ai/src/methods/chrome-adapter.ts +++ b/packages/ai/src/methods/chrome-adapter.ts @@ -27,6 +27,7 @@ import { Content, Role } from '../types'; +import { ChromeAdapter } from '../types/chrome-adapter'; import { Availability, LanguageModel, @@ -39,7 +40,7 @@ import { * Defines an inference "backend" that uses Chrome's on-device model, * and encapsulates logic for detecting when on-device is possible. */ -export class ChromeAdapter { +export class ChromeAdapterImpl implements ChromeAdapter { // Visible for testing static SUPPORTED_MIME_TYPES = ['image/jpeg', 'image/png']; private isDownloading = false; @@ -99,7 +100,7 @@ export class ChromeAdapter { ); return false; } - if (!ChromeAdapter.isOnDeviceRequest(request)) { + if (!ChromeAdapterImpl.isOnDeviceRequest(request)) { logger.debug( `On-device inference unavailable because request is incompatible.` ); @@ -114,19 +115,19 @@ export class ChromeAdapter { * *

This is comparable to {@link GenerativeModel.generateContent} for generating content in * Cloud.

- * @param request - a standard Vertex {@link GenerateContentRequest} + * @param request - a standard Firebase AI {@link GenerateContentRequest} * @returns {@link Response}, so we can reuse common response formatting. */ async generateContent(request: GenerateContentRequest): Promise { const session = await this.createSession(); const contents = await Promise.all( - request.contents.map(ChromeAdapter.toLanguageModelMessage) + request.contents.map(ChromeAdapterImpl.toLanguageModelMessage) ); const text = await session.prompt( contents, this.onDeviceParams.promptOptions ); - return ChromeAdapter.toResponse(text); + return ChromeAdapterImpl.toResponse(text); } /** @@ -134,7 +135,7 @@ export class ChromeAdapter { * *

This is comparable to {@link GenerativeModel.generateContentStream} for generating content in * Cloud.

- * @param request - a standard Vertex {@link GenerateContentRequest} + * @param request - a standard Firebase AI {@link GenerateContentRequest} * @returns {@link Response}, so we can reuse common response formatting. */ async generateContentStream( @@ -142,13 +143,13 @@ export class ChromeAdapter { ): Promise { const session = await this.createSession(); const contents = await Promise.all( - request.contents.map(ChromeAdapter.toLanguageModelMessage) + request.contents.map(ChromeAdapterImpl.toLanguageModelMessage) ); - const stream = await session.promptStreaming( + const stream = session.promptStreaming( contents, this.onDeviceParams.promptOptions ); - return ChromeAdapter.toStreamResponse(stream); + return ChromeAdapterImpl.toStreamResponse(stream); } async countTokens(_request: CountTokensRequest): Promise { @@ -178,7 +179,7 @@ export class ChromeAdapter { for (const part of content.parts) { if ( part.inlineData && - ChromeAdapter.SUPPORTED_MIME_TYPES.indexOf( + ChromeAdapterImpl.SUPPORTED_MIME_TYPES.indexOf( part.inlineData.mimeType ) === -1 ) { @@ -224,28 +225,28 @@ export class ChromeAdapter { this.isDownloading = true; this.downloadPromise = this.languageModelProvider ?.create(this.onDeviceParams.createOptions) - .then(() => { + .finally(() => { this.isDownloading = false; }); } /** - * Converts Vertex {@link Content} object to a Chrome {@link LanguageModelMessage} object. + * Converts Firebase AI {@link Content} object to a Chrome {@link LanguageModelMessage} object. */ private static async toLanguageModelMessage( content: Content ): Promise { const languageModelMessageContents = await Promise.all( - content.parts.map(ChromeAdapter.toLanguageModelMessageContent) + content.parts.map(ChromeAdapterImpl.toLanguageModelMessageContent) ); return { - role: ChromeAdapter.toLanguageModelMessageRole(content.role), + role: ChromeAdapterImpl.toLanguageModelMessageRole(content.role), content: languageModelMessageContents }; } /** - * Converts a Vertex Part object to a Chrome LanguageModelMessageContent object. + * Converts a Firebase AI Part object to a Chrome LanguageModelMessageContent object. */ private static async toLanguageModelMessageContent( part: Part @@ -266,13 +267,14 @@ export class ChromeAdapter { value: imageBitmap }; } - // Assumes contents have been verified to contain only a single TextPart. - // TODO: support other input types - throw new Error('Not yet implemented'); + throw new AIError( + AIErrorCode.REQUEST_ERROR, + `Processing of this Part type is not currently supported.` + ); } /** - * Converts a Vertex {@link Role} string to a {@link LanguageModelMessageRole} string. + * Converts a Firebase AI {@link Role} string to a {@link LanguageModelMessageRole} string. */ private static toLanguageModelMessageRole( role: Role @@ -284,8 +286,8 @@ export class ChromeAdapter { /** * Abstracts Chrome session creation. * - *

Chrome uses a multi-turn session for all inference. Vertex uses single-turn for all - * inference. To map the Vertex API to Chrome's API, the SDK creates a new session for all + *

Chrome uses a multi-turn session for all inference. Firebase AI uses single-turn for all + * inference. To map the Firebase AI API to Chrome's API, the SDK creates a new session for all * inference.

* *

Chrome will remove a model from memory if it's no longer in use, so this method ensures a @@ -294,7 +296,7 @@ export class ChromeAdapter { private async createSession(): Promise { if (!this.languageModelProvider) { throw new AIError( - AIErrorCode.REQUEST_ERROR, + AIErrorCode.UNSUPPORTED, 'Chrome AI requested for unsupported browser version.' ); } @@ -310,7 +312,7 @@ export class ChromeAdapter { } /** - * Formats string returned by Chrome as a {@link Response} returned by Vertex. + * Formats string returned by Chrome as a {@link Response} returned by Firebase AI. */ private static toResponse(text: string): Response { return { @@ -327,7 +329,7 @@ export class ChromeAdapter { } /** - * Formats string stream returned by Chrome as SSE returned by Vertex. + * Formats string stream returned by Chrome as SSE returned by Firebase AI. */ private static toStreamResponse(stream: ReadableStream): Response { const encoder = new TextEncoder(); diff --git a/packages/ai/src/methods/count-tokens.test.ts b/packages/ai/src/methods/count-tokens.test.ts index 78c51d3f5b7..2da99b9e938 100644 --- a/packages/ai/src/methods/count-tokens.test.ts +++ b/packages/ai/src/methods/count-tokens.test.ts @@ -27,7 +27,7 @@ import { ApiSettings } from '../types/internal'; import { Task } from '../requests/request'; import { mapCountTokensRequest } from '../googleai-mappers'; import { GoogleAIBackend, VertexAIBackend } from '../backend'; -import { ChromeAdapter } from './chrome-adapter'; +import { ChromeAdapterImpl } from './chrome-adapter'; use(sinonChai); use(chaiAsPromised); @@ -68,7 +68,7 @@ describe('countTokens()', () => { fakeApiSettings, 'model', fakeRequestParams, - new ChromeAdapter() + new ChromeAdapterImpl() ); expect(result.totalTokens).to.equal(6); expect(result.totalBillableCharacters).to.equal(16); @@ -95,7 +95,7 @@ describe('countTokens()', () => { fakeApiSettings, 'model', fakeRequestParams, - new ChromeAdapter() + new ChromeAdapterImpl() ); expect(result.totalTokens).to.equal(1837); expect(result.totalBillableCharacters).to.equal(117); @@ -124,7 +124,7 @@ describe('countTokens()', () => { fakeApiSettings, 'model', fakeRequestParams, - new ChromeAdapter() + new ChromeAdapterImpl() ); expect(result.totalTokens).to.equal(258); expect(result).to.not.have.property('totalBillableCharacters'); @@ -154,7 +154,7 @@ describe('countTokens()', () => { fakeApiSettings, 'model', fakeRequestParams, - new ChromeAdapter() + new ChromeAdapterImpl() ) ).to.be.rejectedWith(/404.*not found/); expect(mockFetch).to.be.called; @@ -177,7 +177,7 @@ describe('countTokens()', () => { fakeGoogleAIApiSettings, 'model', fakeRequestParams, - new ChromeAdapter() + new ChromeAdapterImpl() ); expect(makeRequestStub).to.be.calledWith( @@ -191,7 +191,7 @@ describe('countTokens()', () => { }); }); it('on-device', async () => { - const chromeAdapter = new ChromeAdapter(); + const chromeAdapter = new ChromeAdapterImpl(); const isAvailableStub = stub(chromeAdapter, 'isAvailable').resolves(true); const mockResponse = getMockResponse( 'vertexAI', diff --git a/packages/ai/src/methods/count-tokens.ts b/packages/ai/src/methods/count-tokens.ts index 81fb3ad061d..00dde84ab48 100644 --- a/packages/ai/src/methods/count-tokens.ts +++ b/packages/ai/src/methods/count-tokens.ts @@ -24,7 +24,7 @@ import { Task, makeRequest } from '../requests/request'; import { ApiSettings } from '../types/internal'; import * as GoogleAIMapper from '../googleai-mappers'; import { BackendType } from '../public-types'; -import { ChromeAdapter } from './chrome-adapter'; +import { ChromeAdapter } from '../types/chrome-adapter'; export async function countTokensOnCloud( apiSettings: ApiSettings, @@ -54,10 +54,10 @@ export async function countTokens( apiSettings: ApiSettings, model: string, params: CountTokensRequest, - chromeAdapter: ChromeAdapter, + chromeAdapter?: ChromeAdapter, requestOptions?: RequestOptions ): Promise { - if (await chromeAdapter.isAvailable(params)) { + if (chromeAdapter && (await chromeAdapter.isAvailable(params))) { return (await chromeAdapter.countTokens(params)).json(); } diff --git a/packages/ai/src/methods/generate-content.test.ts b/packages/ai/src/methods/generate-content.test.ts index 823755d03cb..bbc9d47fc70 100644 --- a/packages/ai/src/methods/generate-content.test.ts +++ b/packages/ai/src/methods/generate-content.test.ts @@ -34,7 +34,7 @@ import { Task } from '../requests/request'; import { AIError } from '../api'; import { mapGenerateContentRequest } from '../googleai-mappers'; import { GoogleAIBackend, VertexAIBackend } from '../backend'; -import { ChromeAdapter } from './chrome-adapter'; +import { ChromeAdapterImpl } from './chrome-adapter'; use(sinonChai); use(chaiAsPromised); @@ -97,8 +97,7 @@ describe('generateContent()', () => { const result = await generateContent( fakeApiSettings, 'model', - fakeRequestParams, - new ChromeAdapter() + fakeRequestParams ); expect(result.response.text()).to.include('Mountain View, California'); expect(makeRequestStub).to.be.calledWith( @@ -121,8 +120,7 @@ describe('generateContent()', () => { const result = await generateContent( fakeApiSettings, 'model', - fakeRequestParams, - new ChromeAdapter() + fakeRequestParams ); expect(result.response.text()).to.include('Use Freshly Ground Coffee'); expect(result.response.text()).to.include('30 minutes of brewing'); @@ -145,8 +143,7 @@ describe('generateContent()', () => { const result = await generateContent( fakeApiSettings, 'model', - fakeRequestParams, - new ChromeAdapter() + fakeRequestParams ); expect(result.response.usageMetadata?.totalTokenCount).to.equal(1913); expect(result.response.usageMetadata?.candidatesTokenCount).to.equal(76); @@ -181,8 +178,7 @@ describe('generateContent()', () => { const result = await generateContent( fakeApiSettings, 'model', - fakeRequestParams, - new ChromeAdapter() + fakeRequestParams ); expect(result.response.text()).to.include( 'Some information cited from an external source' @@ -256,8 +252,7 @@ describe('generateContent()', () => { const result = await generateContent( fakeApiSettings, 'model', - fakeRequestParams, - new ChromeAdapter() + fakeRequestParams ); expect(result.response.text).to.throw('SAFETY'); expect(makeRequestStub).to.be.calledWith( @@ -279,8 +274,7 @@ describe('generateContent()', () => { const result = await generateContent( fakeApiSettings, 'model', - fakeRequestParams, - new ChromeAdapter() + fakeRequestParams ); expect(result.response.text).to.throw('SAFETY'); expect(makeRequestStub).to.be.calledWith( @@ -302,8 +296,7 @@ describe('generateContent()', () => { const result = await generateContent( fakeApiSettings, 'model', - fakeRequestParams, - new ChromeAdapter() + fakeRequestParams ); expect(result.response.text()).to.equal(''); expect(makeRequestStub).to.be.calledWith( @@ -325,8 +318,7 @@ describe('generateContent()', () => { const result = await generateContent( fakeApiSettings, 'model', - fakeRequestParams, - new ChromeAdapter() + fakeRequestParams ); expect(result.response.text()).to.include('Some text'); expect(makeRequestStub).to.be.calledWith( @@ -348,12 +340,7 @@ describe('generateContent()', () => { json: mockResponse.json } as Response); await expect( - generateContent( - fakeApiSettings, - 'model', - fakeRequestParams, - new ChromeAdapter() - ) + generateContent(fakeApiSettings, 'model', fakeRequestParams) ).to.be.rejectedWith(/400.*invalid argument/); expect(mockFetch).to.be.called; }); @@ -368,12 +355,7 @@ describe('generateContent()', () => { json: mockResponse.json } as Response); await expect( - generateContent( - fakeApiSettings, - 'model', - fakeRequestParams, - new ChromeAdapter() - ) + generateContent(fakeApiSettings, 'model', fakeRequestParams) ).to.be.rejectedWith( /firebasevertexai\.googleapis[\s\S]*my-project[\s\S]*api-not-enabled/ ); @@ -413,8 +395,7 @@ describe('generateContent()', () => { generateContent( fakeGoogleAIApiSettings, 'model', - requestParamsWithMethod, - new ChromeAdapter() + requestParamsWithMethod ) ).to.be.rejectedWith(AIError, AIErrorCode.UNSUPPORTED); expect(makeRequestStub).to.not.be.called; @@ -429,8 +410,7 @@ describe('generateContent()', () => { await generateContent( fakeGoogleAIApiSettings, 'model', - fakeGoogleAIRequestParams, - new ChromeAdapter() + fakeGoogleAIRequestParams ); expect(makeRequestStub).to.be.calledWith( @@ -445,7 +425,7 @@ describe('generateContent()', () => { }); // TODO: define a similar test for generateContentStream it('on-device', async () => { - const chromeAdapter = new ChromeAdapter(); + const chromeAdapter = new ChromeAdapterImpl(); const isAvailableStub = stub(chromeAdapter, 'isAvailable').resolves(true); const mockResponse = getMockResponse( 'vertexAI', diff --git a/packages/ai/src/methods/generate-content.ts b/packages/ai/src/methods/generate-content.ts index ff99b306855..2c1c383641f 100644 --- a/packages/ai/src/methods/generate-content.ts +++ b/packages/ai/src/methods/generate-content.ts @@ -28,7 +28,7 @@ import { processStream } from '../requests/stream-reader'; import { ApiSettings } from '../types/internal'; import * as GoogleAIMapper from '../googleai-mappers'; import { BackendType } from '../public-types'; -import { ChromeAdapter } from './chrome-adapter'; +import { ChromeAdapter } from '../types/chrome-adapter'; async function generateContentStreamOnCloud( apiSettings: ApiSettings, @@ -53,11 +53,11 @@ export async function generateContentStream( apiSettings: ApiSettings, model: string, params: GenerateContentRequest, - chromeAdapter: ChromeAdapter, + chromeAdapter?: ChromeAdapter, requestOptions?: RequestOptions ): Promise { let response; - if (await chromeAdapter.isAvailable(params)) { + if (chromeAdapter && (await chromeAdapter.isAvailable(params))) { response = await chromeAdapter.generateContentStream(params); } else { response = await generateContentStreamOnCloud( @@ -93,11 +93,11 @@ export async function generateContent( apiSettings: ApiSettings, model: string, params: GenerateContentRequest, - chromeAdapter: ChromeAdapter, + chromeAdapter?: ChromeAdapter, requestOptions?: RequestOptions ): Promise { let response; - if (await chromeAdapter.isAvailable(params)) { + if (chromeAdapter && (await chromeAdapter.isAvailable(params))) { response = await chromeAdapter.generateContent(params); } else { response = await generateContentOnCloud( diff --git a/packages/ai/src/models/generative-model.test.ts b/packages/ai/src/models/generative-model.test.ts index e3d8f7fe011..5c7e80436e1 100644 --- a/packages/ai/src/models/generative-model.test.ts +++ b/packages/ai/src/models/generative-model.test.ts @@ -22,7 +22,7 @@ import { match, restore, stub } from 'sinon'; import { getMockResponse } from '../../test-utils/mock-response'; import sinonChai from 'sinon-chai'; import { VertexAIBackend } from '../backend'; -import { ChromeAdapter } from '../methods/chrome-adapter'; +import { ChromeAdapterImpl } from '../methods/chrome-adapter'; use(sinonChai); @@ -61,7 +61,7 @@ describe('GenerativeModel', () => { }, systemInstruction: { role: 'system', parts: [{ text: 'be friendly' }] } }, - new ChromeAdapter() + new ChromeAdapterImpl() ); expect(genModel.tools?.length).to.equal(1); expect(genModel.toolConfig?.functionCallingConfig?.mode).to.equal( @@ -99,7 +99,7 @@ describe('GenerativeModel', () => { model: 'my-model', systemInstruction: 'be friendly' }, - new ChromeAdapter() + new ChromeAdapterImpl() ); expect(genModel.systemInstruction?.parts[0].text).to.equal('be friendly'); const mockResponse = getMockResponse( @@ -142,7 +142,7 @@ describe('GenerativeModel', () => { }, systemInstruction: { role: 'system', parts: [{ text: 'be friendly' }] } }, - new ChromeAdapter() + new ChromeAdapterImpl() ); expect(genModel.tools?.length).to.equal(1); expect(genModel.toolConfig?.functionCallingConfig?.mode).to.equal( @@ -193,7 +193,7 @@ describe('GenerativeModel', () => { topK: 1 } }, - new ChromeAdapter() + new ChromeAdapterImpl() ); const chatSession = genModel.startChat(); expect(chatSession.params?.generationConfig).to.deep.equal({ @@ -210,7 +210,7 @@ describe('GenerativeModel', () => { topK: 1 } }, - new ChromeAdapter() + new ChromeAdapterImpl() ); const chatSession = genModel.startChat({ generationConfig: { @@ -237,7 +237,7 @@ describe('GenerativeModel', () => { topK: 1 } }, - new ChromeAdapter() + new ChromeAdapterImpl() ); expect(genModel.tools?.length).to.equal(1); expect(genModel.toolConfig?.functionCallingConfig?.mode).to.equal( @@ -276,7 +276,7 @@ describe('GenerativeModel', () => { model: 'my-model', systemInstruction: 'be friendly' }, - new ChromeAdapter() + new ChromeAdapterImpl() ); expect(genModel.systemInstruction?.parts[0].text).to.equal('be friendly'); const mockResponse = getMockResponse( @@ -315,7 +315,7 @@ describe('GenerativeModel', () => { responseMimeType: 'image/jpeg' } }, - new ChromeAdapter() + new ChromeAdapterImpl() ); expect(genModel.tools?.length).to.equal(1); expect(genModel.toolConfig?.functionCallingConfig?.mode).to.equal( @@ -369,7 +369,7 @@ describe('GenerativeModel', () => { const genModel = new GenerativeModel( fakeAI, { model: 'my-model' }, - new ChromeAdapter() + new ChromeAdapterImpl() ); const mockResponse = getMockResponse( 'vertexAI', diff --git a/packages/ai/src/models/generative-model.ts b/packages/ai/src/models/generative-model.ts index 98b662ebdb9..d0376d9bace 100644 --- a/packages/ai/src/models/generative-model.ts +++ b/packages/ai/src/models/generative-model.ts @@ -43,7 +43,7 @@ import { } from '../requests/request-helpers'; import { AI } from '../public-types'; import { AIModel } from './ai-model'; -import { ChromeAdapter } from '../methods/chrome-adapter'; +import { ChromeAdapter } from '../types/chrome-adapter'; /** * Class for generative model APIs. @@ -64,8 +64,8 @@ export class GenerativeModel extends AIModel { constructor( ai: AI, modelParams: ModelParams, - private chromeAdapter: ChromeAdapter, - requestOptions?: RequestOptions + requestOptions?: RequestOptions, + private chromeAdapter?: ChromeAdapter ) { super(ai, modelParams.model); this.generationConfig = modelParams.generationConfig || {}; diff --git a/packages/ai/src/types/chrome-adapter.ts b/packages/ai/src/types/chrome-adapter.ts new file mode 100644 index 00000000000..6e7a1313535 --- /dev/null +++ b/packages/ai/src/types/chrome-adapter.ts @@ -0,0 +1,16 @@ +import { CountTokensRequest, GenerateContentRequest } from "./requests"; + +/** + * Defines an inference "backend" that uses Chrome's on-device model, + * and encapsulates logic for detecting when on-device is possible. + * + * @public + */ +export interface ChromeAdapter { + isAvailable(request: GenerateContentRequest): Promise; + countTokens(_request: CountTokensRequest): Promise + generateContent(request: GenerateContentRequest): Promise + generateContentStream( + request: GenerateContentRequest + ): Promise +} \ No newline at end of file diff --git a/packages/ai/src/types/enums.ts b/packages/ai/src/types/enums.ts index b5e4e60ab4f..14e556408ee 100644 --- a/packages/ai/src/types/enums.ts +++ b/packages/ai/src/types/enums.ts @@ -67,7 +67,7 @@ export const HarmBlockThreshold = { BLOCK_NONE: 'BLOCK_NONE', /** * All content will be allowed. This is the same as `BLOCK_NONE`, but the metadata corresponding - * to the {@link HarmCategory} will not be present in the response. + * to the {@link (HarmCategory:type)} will not be present in the response. */ OFF: 'OFF' } as const; @@ -270,6 +270,9 @@ export const FunctionCallingMode = { NONE: 'NONE' } as const; +/** + * @public + */ export type FunctionCallingMode = (typeof FunctionCallingMode)[keyof typeof FunctionCallingMode]; @@ -335,3 +338,21 @@ export const ResponseModality = { */ export type ResponseModality = (typeof ResponseModality)[keyof typeof ResponseModality]; + +/** + * EXPERIMENTAL FEATURE + * Determines whether inference happens on-device or in-cloud. + * @public + */ +export const InferenceMode = { + 'PREFER_ON_DEVICE': 'prefer_on_device', + 'ONLY_ON_DEVICE': 'only_on_device', + 'ONLY_IN_CLOUD': 'only_in_cloud' +} as const; + +/** + * EXPERIMENTAL FEATURE + * Determines whether inference happens on-device or in-cloud. + * @public + */ +export type InferenceMode = (typeof InferenceMode)[keyof typeof InferenceMode]; diff --git a/packages/ai/src/types/imagen/responses.ts b/packages/ai/src/types/imagen/responses.ts index 4e4496e6b36..be99ab104bf 100644 --- a/packages/ai/src/types/imagen/responses.ts +++ b/packages/ai/src/types/imagen/responses.ts @@ -37,6 +37,7 @@ export interface ImagenInlineImage { * An image generated by Imagen, stored in a Cloud Storage for Firebase bucket. * * This feature is not available yet. + * @beta */ export interface ImagenGCSImage { /** diff --git a/packages/ai/src/types/index.ts b/packages/ai/src/types/index.ts index bd13140566f..2dfe73040ae 100644 --- a/packages/ai/src/types/index.ts +++ b/packages/ai/src/types/index.ts @@ -31,5 +31,7 @@ export { LanguageModelMessageContent, LanguageModelMessageContentValue, LanguageModelMessageRole, - LanguageModelMessageType + LanguageModelMessageType, + LanguageModelPromptOptions } from './language-model'; +export * from './chrome-adapter'; diff --git a/packages/ai/src/types/language-model.ts b/packages/ai/src/types/language-model.ts index 83a728dc3be..19a867f85a0 100644 --- a/packages/ai/src/types/language-model.ts +++ b/packages/ai/src/types/language-model.ts @@ -16,8 +16,10 @@ */ /** * The subset of the Prompt API - * ({@see https://github.com/webmachinelearning/prompt-api#full-api-surface-in-web-idl}) + * (see {@link https://github.com/webmachinelearning/prompt-api#full-api-surface-in-web-idl }) * required for hybrid functionality. + * + * @public */ export interface LanguageModel extends EventTarget { create(options?: LanguageModelCreateOptions): Promise; @@ -36,41 +38,84 @@ export interface LanguageModel extends EventTarget { ): Promise; destroy(): undefined; } + +/** + * @public + */ export enum Availability { 'unavailable' = 'unavailable', 'downloadable' = 'downloadable', 'downloading' = 'downloading', 'available' = 'available' } + +/** + * @public + */ export interface LanguageModelCreateCoreOptions { topK?: number; temperature?: number; expectedInputs?: LanguageModelExpected[]; } + +/** + * @public + */ export interface LanguageModelCreateOptions extends LanguageModelCreateCoreOptions { signal?: AbortSignal; initialPrompts?: LanguageModelMessage[]; } + +/** + * @public + */ export interface LanguageModelPromptOptions { responseConstraint?: object; // TODO: Restore AbortSignal once the API is defined. } -export interface LanguageModelExpected { + +/** + * @public + */export interface LanguageModelExpected { type: LanguageModelMessageType; languages?: string[]; } + +/** + * @public + */ export type LanguageModelPrompt = LanguageModelMessage[]; + +/** + * @public + */ export interface LanguageModelMessage { role: LanguageModelMessageRole; content: LanguageModelMessageContent[]; } + +/** + * @public + */ export interface LanguageModelMessageContent { type: LanguageModelMessageType; value: LanguageModelMessageContentValue; } + +/** + * @public + */ export type LanguageModelMessageRole = 'system' | 'user' | 'assistant'; + +/** + * @public + */ export type LanguageModelMessageType = 'text' | 'image' | 'audio'; + +/** + * @public + */ export type LanguageModelMessageContentValue = | ImageBitmapSource | AudioBuffer diff --git a/packages/ai/src/types/requests.ts b/packages/ai/src/types/requests.ts index 5a2ad131300..a039d686edf 100644 --- a/packages/ai/src/types/requests.ts +++ b/packages/ai/src/types/requests.ts @@ -26,6 +26,7 @@ import { HarmBlockMethod, HarmBlockThreshold, HarmCategory, + InferenceMode, ResponseModality } from './enums'; import { ObjectSchemaRequest, SchemaRequest } from './schema'; @@ -277,6 +278,8 @@ export interface FunctionCallingConfig { /** * Encapsulates configuration for on-device inference. + * + * @public */ export interface OnDeviceParams { createOptions?: LanguageModelCreateOptions; @@ -284,7 +287,8 @@ export interface OnDeviceParams { } /** - * Toggles hybrid inference. + * Configures hybrid inference. + * @public */ export interface HybridParams { /** @@ -302,14 +306,6 @@ export interface HybridParams { } /** - * Determines whether inference happens on-device or in-cloud. - */ -export type InferenceMode = - | 'prefer_on_device' - | 'only_on_device' - | 'only_in_cloud'; - -/* * Configuration for "thinking" behavior of compatible Gemini models. * * Certain models utilize a thinking process before generating a response. This allows them to diff --git a/packages/ai/src/types/schema.ts b/packages/ai/src/types/schema.ts index f8c91168bf2..8068ce62a91 100644 --- a/packages/ai/src/types/schema.ts +++ b/packages/ai/src/types/schema.ts @@ -128,7 +128,7 @@ export interface SchemaInterface extends SchemaShared { } /** - * Interface for JSON parameters in a schema of {@link SchemaType} + * Interface for JSON parameters in a schema of {@link (SchemaType:type)} * "object" when not using the `Schema.object()` helper. * @public */ From 9e4e910d9f151706c1da95b8edf77d9d5204e797 Mon Sep 17 00:00:00 2001 From: Christina Holland Date: Wed, 30 Jul 2025 15:32:09 -0700 Subject: [PATCH 09/20] Ensure EXPERIMENTAL added to doc comments, fill out doc comments --- common/api-review/ai.api.md | 19 ++++---- docs-devsite/ai.generativemodel.md | 11 ----- docs-devsite/ai.hybridparams.md | 2 +- .../ai.languagemodelcreatecoreoptions.md | 1 + docs-devsite/ai.languagemodelcreateoptions.md | 1 + docs-devsite/ai.languagemodelexpected.md | 1 + docs-devsite/ai.languagemodelmessage.md | 1 + .../ai.languagemodelmessagecontent.md | 1 + docs-devsite/ai.languagemodelpromptoptions.md | 1 + docs-devsite/ai.md | 33 +++++++------ docs-devsite/ai.ondeviceparams.md | 2 +- packages/ai/src/api.test.ts | 4 +- packages/ai/src/api.ts | 5 +- packages/ai/src/constants.ts | 5 ++ .../ai/src/methods/chrome-adapter.test.ts | 47 ++++++++++--------- .../ai/src/models/generative-model.test.ts | 9 ++++ packages/ai/src/models/generative-model.ts | 4 -- packages/ai/src/types/chrome-adapter.ts | 33 +++++++++---- packages/ai/src/types/enums.ts | 4 +- packages/ai/src/types/language-model.ts | 30 ++++++++++-- packages/ai/src/types/requests.ts | 4 +- 21 files changed, 133 insertions(+), 85 deletions(-) diff --git a/common/api-review/ai.api.md b/common/api-review/ai.api.md index 030e1a899ab..7cc599f92f9 100644 --- a/common/api-review/ai.api.md +++ b/common/api-review/ai.api.md @@ -430,7 +430,6 @@ export interface GenerativeContentBlob { export class GenerativeModel extends AIModel { constructor(ai: AI, modelParams: ModelParams, requestOptions?: RequestOptions, chromeAdapter?: ChromeAdapter | undefined); countTokens(request: CountTokensRequest | string | Array): Promise; - static DEFAULT_HYBRID_IN_CLOUD_MODEL: string; generateContent(request: GenerateContentRequest | string | Array): Promise; generateContentStream(request: GenerateContentRequest | string | Array): Promise; // (undocumented) @@ -729,7 +728,7 @@ export class IntegerSchema extends Schema { constructor(schemaParams?: SchemaParams); } -// @public (undocumented) +// @public export interface LanguageModelCreateCoreOptions { // (undocumented) expectedInputs?: LanguageModelExpected[]; @@ -739,7 +738,7 @@ export interface LanguageModelCreateCoreOptions { topK?: number; } -// @public (undocumented) +// @public export interface LanguageModelCreateOptions extends LanguageModelCreateCoreOptions { // (undocumented) initialPrompts?: LanguageModelMessage[]; @@ -747,7 +746,7 @@ export interface LanguageModelCreateOptions extends LanguageModelCreateCoreOptio signal?: AbortSignal; } -// @public (undocumented) +// @public export interface LanguageModelExpected { // (undocumented) languages?: string[]; @@ -755,7 +754,7 @@ export interface LanguageModelExpected { type: LanguageModelMessageType; } -// @public (undocumented) +// @public export interface LanguageModelMessage { // (undocumented) content: LanguageModelMessageContent[]; @@ -763,7 +762,7 @@ export interface LanguageModelMessage { role: LanguageModelMessageRole; } -// @public (undocumented) +// @public export interface LanguageModelMessageContent { // (undocumented) type: LanguageModelMessageType; @@ -771,16 +770,16 @@ export interface LanguageModelMessageContent { value: LanguageModelMessageContentValue; } -// @public (undocumented) +// @public export type LanguageModelMessageContentValue = ImageBitmapSource | AudioBuffer | BufferSource | string; -// @public (undocumented) +// @public export type LanguageModelMessageRole = 'system' | 'user' | 'assistant'; -// @public (undocumented) +// @public export type LanguageModelMessageType = 'text' | 'image' | 'audio'; -// @public (undocumented) +// @public export interface LanguageModelPromptOptions { // (undocumented) responseConstraint?: object; diff --git a/docs-devsite/ai.generativemodel.md b/docs-devsite/ai.generativemodel.md index fa29218e60e..323fcfe9d76 100644 --- a/docs-devsite/ai.generativemodel.md +++ b/docs-devsite/ai.generativemodel.md @@ -29,7 +29,6 @@ export declare class GenerativeModel extends AIModel | Property | Modifiers | Type | Description | | --- | --- | --- | --- | -| [DEFAULT\_HYBRID\_IN\_CLOUD\_MODEL](./ai.generativemodel.md#generativemodeldefault_hybrid_in_cloud_model) | static | string | Defines the name of the default in-cloud model to use for hybrid inference. | | [generationConfig](./ai.generativemodel.md#generativemodelgenerationconfig) | | [GenerationConfig](./ai.generationconfig.md#generationconfig_interface) | | | [requestOptions](./ai.generativemodel.md#generativemodelrequestoptions) | | [RequestOptions](./ai.requestoptions.md#requestoptions_interface) | | | [safetySettings](./ai.generativemodel.md#generativemodelsafetysettings) | | [SafetySetting](./ai.safetysetting.md#safetysetting_interface)\[\] | | @@ -65,16 +64,6 @@ constructor(ai: AI, modelParams: ModelParams, requestOptions?: RequestOptions, c | requestOptions | [RequestOptions](./ai.requestoptions.md#requestoptions_interface) | | | chromeAdapter | [ChromeAdapter](./ai.chromeadapter.md#chromeadapter_interface) \| undefined | | -## GenerativeModel.DEFAULT\_HYBRID\_IN\_CLOUD\_MODEL - -Defines the name of the default in-cloud model to use for hybrid inference. - -Signature: - -```typescript -static DEFAULT_HYBRID_IN_CLOUD_MODEL: string; -``` - ## GenerativeModel.generationConfig Signature: diff --git a/docs-devsite/ai.hybridparams.md b/docs-devsite/ai.hybridparams.md index 1934b68597f..383e9baafa5 100644 --- a/docs-devsite/ai.hybridparams.md +++ b/docs-devsite/ai.hybridparams.md @@ -10,7 +10,7 @@ https://github.com/firebase/firebase-js-sdk {% endcomment %} # HybridParams interface -Configures hybrid inference. +(EXPERIMENTAL) Configures hybrid inference. Signature: diff --git a/docs-devsite/ai.languagemodelcreatecoreoptions.md b/docs-devsite/ai.languagemodelcreatecoreoptions.md index 832ea3b8ca8..e6432543958 100644 --- a/docs-devsite/ai.languagemodelcreatecoreoptions.md +++ b/docs-devsite/ai.languagemodelcreatecoreoptions.md @@ -10,6 +10,7 @@ https://github.com/firebase/firebase-js-sdk {% endcomment %} # LanguageModelCreateCoreOptions interface +(EXPERIMENTAL) Used to configure the creation of an on-device language model session. Signature: diff --git a/docs-devsite/ai.languagemodelcreateoptions.md b/docs-devsite/ai.languagemodelcreateoptions.md index 54d1ecaa803..22e88cbfdcd 100644 --- a/docs-devsite/ai.languagemodelcreateoptions.md +++ b/docs-devsite/ai.languagemodelcreateoptions.md @@ -10,6 +10,7 @@ https://github.com/firebase/firebase-js-sdk {% endcomment %} # LanguageModelCreateOptions interface +(EXPERIMENTAL) Used to configure the creation of an on-device language model session. Signature: diff --git a/docs-devsite/ai.languagemodelexpected.md b/docs-devsite/ai.languagemodelexpected.md index e33d922007c..ffec338c773 100644 --- a/docs-devsite/ai.languagemodelexpected.md +++ b/docs-devsite/ai.languagemodelexpected.md @@ -10,6 +10,7 @@ https://github.com/firebase/firebase-js-sdk {% endcomment %} # LanguageModelExpected interface +(EXPERIMENTAL) Options for an on-device language model expected inputs. Signature: diff --git a/docs-devsite/ai.languagemodelmessage.md b/docs-devsite/ai.languagemodelmessage.md index efedf369945..5f9e62a8ab0 100644 --- a/docs-devsite/ai.languagemodelmessage.md +++ b/docs-devsite/ai.languagemodelmessage.md @@ -10,6 +10,7 @@ https://github.com/firebase/firebase-js-sdk {% endcomment %} # LanguageModelMessage interface +(EXPERIMENTAL) An on-device language model message. Signature: diff --git a/docs-devsite/ai.languagemodelmessagecontent.md b/docs-devsite/ai.languagemodelmessagecontent.md index b87f8a28b3a..1600d312c89 100644 --- a/docs-devsite/ai.languagemodelmessagecontent.md +++ b/docs-devsite/ai.languagemodelmessagecontent.md @@ -10,6 +10,7 @@ https://github.com/firebase/firebase-js-sdk {% endcomment %} # LanguageModelMessageContent interface +(EXPERIMENTAL) An on-device language model content object. Signature: diff --git a/docs-devsite/ai.languagemodelpromptoptions.md b/docs-devsite/ai.languagemodelpromptoptions.md index cde9b9af3be..9ff8c5d5941 100644 --- a/docs-devsite/ai.languagemodelpromptoptions.md +++ b/docs-devsite/ai.languagemodelpromptoptions.md @@ -10,6 +10,7 @@ https://github.com/firebase/firebase-js-sdk {% endcomment %} # LanguageModelPromptOptions interface +(EXPERIMENTAL) Options for an on-device language model prompt. Signature: diff --git a/docs-devsite/ai.md b/docs-devsite/ai.md index 464c14bf1be..520163a369f 100644 --- a/docs-devsite/ai.md +++ b/docs-devsite/ai.md @@ -82,7 +82,7 @@ The Firebase AI Web SDK. | [GroundingChunk](./ai.groundingchunk.md#groundingchunk_interface) | Represents a chunk of retrieved data that supports a claim in the model's response. This is part of the grounding information provided when grounding is enabled. | | [GroundingMetadata](./ai.groundingmetadata.md#groundingmetadata_interface) | Metadata returned when grounding is enabled.Currently, only Grounding with Google Search is supported (see [GoogleSearchTool](./ai.googlesearchtool.md#googlesearchtool_interface)).Important: If using Grounding with Google Search, you are required to comply with the "Grounding with Google Search" usage requirements for your chosen API provider: [Gemini Developer API](https://ai.google.dev/gemini-api/terms#grounding-with-google-search) or Vertex AI Gemini API (see [Service Terms](https://cloud.google.com/terms/service-terms) section within the Service Specific Terms). | | [GroundingSupport](./ai.groundingsupport.md#groundingsupport_interface) | Provides information about how a specific segment of the model's response is supported by the retrieved grounding chunks. | -| [HybridParams](./ai.hybridparams.md#hybridparams_interface) | Configures hybrid inference. | +| [HybridParams](./ai.hybridparams.md#hybridparams_interface) | (EXPERIMENTAL) Configures hybrid inference. | | [ImagenGCSImage](./ai.imagengcsimage.md#imagengcsimage_interface) | (Public Preview) An image generated by Imagen, stored in a Cloud Storage for Firebase bucket.This feature is not available yet. | | [ImagenGenerationConfig](./ai.imagengenerationconfig.md#imagengenerationconfig_interface) | (Public Preview) Configuration options for generating images with Imagen.See the [documentation](http://firebase.google.com/docs/vertex-ai/generate-images-imagen) for more details. | | [ImagenGenerationResponse](./ai.imagengenerationresponse.md#imagengenerationresponse_interface) | (Public Preview) The response from a request to generate images with Imagen. | @@ -90,16 +90,16 @@ The Firebase AI Web SDK. | [ImagenModelParams](./ai.imagenmodelparams.md#imagenmodelparams_interface) | (Public Preview) Parameters for configuring an [ImagenModel](./ai.imagenmodel.md#imagenmodel_class). | | [ImagenSafetySettings](./ai.imagensafetysettings.md#imagensafetysettings_interface) | (Public Preview) Settings for controlling the aggressiveness of filtering out sensitive content.See the [documentation](http://firebase.google.com/docs/vertex-ai/generate-images) for more details. | | [InlineDataPart](./ai.inlinedatapart.md#inlinedatapart_interface) | Content part interface if the part represents an image. | -| [LanguageModelCreateCoreOptions](./ai.languagemodelcreatecoreoptions.md#languagemodelcreatecoreoptions_interface) | | -| [LanguageModelCreateOptions](./ai.languagemodelcreateoptions.md#languagemodelcreateoptions_interface) | | -| [LanguageModelExpected](./ai.languagemodelexpected.md#languagemodelexpected_interface) | | -| [LanguageModelMessage](./ai.languagemodelmessage.md#languagemodelmessage_interface) | | -| [LanguageModelMessageContent](./ai.languagemodelmessagecontent.md#languagemodelmessagecontent_interface) | | -| [LanguageModelPromptOptions](./ai.languagemodelpromptoptions.md#languagemodelpromptoptions_interface) | | +| [LanguageModelCreateCoreOptions](./ai.languagemodelcreatecoreoptions.md#languagemodelcreatecoreoptions_interface) | (EXPERIMENTAL) Used to configure the creation of an on-device language model session. | +| [LanguageModelCreateOptions](./ai.languagemodelcreateoptions.md#languagemodelcreateoptions_interface) | (EXPERIMENTAL) Used to configure the creation of an on-device language model session. | +| [LanguageModelExpected](./ai.languagemodelexpected.md#languagemodelexpected_interface) | (EXPERIMENTAL) Options for an on-device language model expected inputs. | +| [LanguageModelMessage](./ai.languagemodelmessage.md#languagemodelmessage_interface) | (EXPERIMENTAL) An on-device language model message. | +| [LanguageModelMessageContent](./ai.languagemodelmessagecontent.md#languagemodelmessagecontent_interface) | (EXPERIMENTAL) An on-device language model content object. | +| [LanguageModelPromptOptions](./ai.languagemodelpromptoptions.md#languagemodelpromptoptions_interface) | (EXPERIMENTAL) Options for an on-device language model prompt. | | [ModalityTokenCount](./ai.modalitytokencount.md#modalitytokencount_interface) | Represents token counting info for a single modality. | | [ModelParams](./ai.modelparams.md#modelparams_interface) | Params passed to [getGenerativeModel()](./ai.md#getgenerativemodel_c63f46a). | | [ObjectSchemaRequest](./ai.objectschemarequest.md#objectschemarequest_interface) | Interface for JSON parameters in a schema of [SchemaType](./ai.md#schematype) "object" when not using the Schema.object() helper. | -| [OnDeviceParams](./ai.ondeviceparams.md#ondeviceparams_interface) | Encapsulates configuration for on-device inference. | +| [OnDeviceParams](./ai.ondeviceparams.md#ondeviceparams_interface) | (EXPERIMENTAL) Encapsulates configuration for on-device inference. | | [PromptFeedback](./ai.promptfeedback.md#promptfeedback_interface) | If the prompt was blocked, this will be populated with blockReason and the relevant safetyRatings. | | [RequestOptions](./ai.requestoptions.md#requestoptions_interface) | Params passed to [getGenerativeModel()](./ai.md#getgenerativemodel_c63f46a). | | [RetrievedContextAttribution](./ai.retrievedcontextattribution.md#retrievedcontextattribution_interface) | | @@ -137,7 +137,7 @@ The Firebase AI Web SDK. | [ImagenAspectRatio](./ai.md#imagenaspectratio) | (Public Preview) Aspect ratios for Imagen images.To specify an aspect ratio for generated images, set the aspectRatio property in your [ImagenGenerationConfig](./ai.imagengenerationconfig.md#imagengenerationconfig_interface).See the [documentation](http://firebase.google.com/docs/vertex-ai/generate-images) for more details and examples of the supported aspect ratios. | | [ImagenPersonFilterLevel](./ai.md#imagenpersonfilterlevel) | (Public Preview) A filter level controlling whether generation of images containing people or faces is allowed.See the personGeneration documentation for more details. | | [ImagenSafetyFilterLevel](./ai.md#imagensafetyfilterlevel) | (Public Preview) A filter level controlling how aggressively to filter sensitive content.Text prompts provided as inputs and images (generated or uploaded) through Imagen on Vertex AI are assessed against a list of safety filters, which include 'harmful categories' (for example, violence, sexual, derogatory, and toxic). This filter level controls how aggressively to filter out potentially harmful content from responses. See the [documentation](http://firebase.google.com/docs/vertex-ai/generate-images) and the [Responsible AI and usage guidelines](https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen#safety-filters) for more details. | -| [InferenceMode](./ai.md#inferencemode) | EXPERIMENTAL FEATURE Determines whether inference happens on-device or in-cloud. | +| [InferenceMode](./ai.md#inferencemode) | (EXPERIMENTAL) Determines whether inference happens on-device or in-cloud. | | [Modality](./ai.md#modality) | Content part modality. | | [POSSIBLE\_ROLES](./ai.md#possible_roles) | Possible roles. | | [ResponseModality](./ai.md#responsemodality) | (Public Preview) Generation modalities to be returned in generation responses. | @@ -160,10 +160,10 @@ The Firebase AI Web SDK. | [ImagenAspectRatio](./ai.md#imagenaspectratio) | (Public Preview) Aspect ratios for Imagen images.To specify an aspect ratio for generated images, set the aspectRatio property in your [ImagenGenerationConfig](./ai.imagengenerationconfig.md#imagengenerationconfig_interface).See the [documentation](http://firebase.google.com/docs/vertex-ai/generate-images) for more details and examples of the supported aspect ratios. | | [ImagenPersonFilterLevel](./ai.md#imagenpersonfilterlevel) | (Public Preview) A filter level controlling whether generation of images containing people or faces is allowed.See the personGeneration documentation for more details. | | [ImagenSafetyFilterLevel](./ai.md#imagensafetyfilterlevel) | (Public Preview) A filter level controlling how aggressively to filter sensitive content.Text prompts provided as inputs and images (generated or uploaded) through Imagen on Vertex AI are assessed against a list of safety filters, which include 'harmful categories' (for example, violence, sexual, derogatory, and toxic). This filter level controls how aggressively to filter out potentially harmful content from responses. See the [documentation](http://firebase.google.com/docs/vertex-ai/generate-images) and the [Responsible AI and usage guidelines](https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen#safety-filters) for more details. | -| [InferenceMode](./ai.md#inferencemode) | EXPERIMENTAL FEATURE Determines whether inference happens on-device or in-cloud. | -| [LanguageModelMessageContentValue](./ai.md#languagemodelmessagecontentvalue) | | -| [LanguageModelMessageRole](./ai.md#languagemodelmessagerole) | | -| [LanguageModelMessageType](./ai.md#languagemodelmessagetype) | | +| [InferenceMode](./ai.md#inferencemode) | (EXPERIMENTAL) Determines whether inference happens on-device or in-cloud. | +| [LanguageModelMessageContentValue](./ai.md#languagemodelmessagecontentvalue) | (EXPERIMENTAL) Content formats that can be provided as on-device message content. | +| [LanguageModelMessageRole](./ai.md#languagemodelmessagerole) | (EXPERIMENTAL) Allowable roles for on-device language model usage. | +| [LanguageModelMessageType](./ai.md#languagemodelmessagetype) | (EXPERIMENTAL) Allowable types for on-device language model messages. | | [Modality](./ai.md#modality) | Content part modality. | | [Part](./ai.md#part) | Content part - includes text, image/video, or function call/response part types. | | [ResponseModality](./ai.md#responsemodality) | (Public Preview) Generation modalities to be returned in generation responses. | @@ -504,7 +504,7 @@ ImagenSafetyFilterLevel: { ## InferenceMode -EXPERIMENTAL FEATURE Determines whether inference happens on-device or in-cloud. +(EXPERIMENTAL) Determines whether inference happens on-device or in-cloud. Signature: @@ -724,7 +724,7 @@ export type ImagenSafetyFilterLevel = (typeof ImagenSafetyFilterLevel)[keyof typ ## InferenceMode -EXPERIMENTAL FEATURE Determines whether inference happens on-device or in-cloud. +(EXPERIMENTAL) Determines whether inference happens on-device or in-cloud. Signature: @@ -734,6 +734,7 @@ export type InferenceMode = (typeof InferenceMode)[keyof typeof InferenceMode]; ## LanguageModelMessageContentValue +(EXPERIMENTAL) Content formats that can be provided as on-device message content. Signature: @@ -743,6 +744,7 @@ export type LanguageModelMessageContentValue = ImageBitmapSource | AudioBuffer | ## LanguageModelMessageRole +(EXPERIMENTAL) Allowable roles for on-device language model usage. Signature: @@ -752,6 +754,7 @@ export type LanguageModelMessageRole = 'system' | 'user' | 'assistant'; ## LanguageModelMessageType +(EXPERIMENTAL) Allowable types for on-device language model messages. Signature: diff --git a/docs-devsite/ai.ondeviceparams.md b/docs-devsite/ai.ondeviceparams.md index 0e23d1fda98..77a4b8aab85 100644 --- a/docs-devsite/ai.ondeviceparams.md +++ b/docs-devsite/ai.ondeviceparams.md @@ -10,7 +10,7 @@ https://github.com/firebase/firebase-js-sdk {% endcomment %} # OnDeviceParams interface -Encapsulates configuration for on-device inference. +(EXPERIMENTAL) Encapsulates configuration for on-device inference. Signature: diff --git a/packages/ai/src/api.test.ts b/packages/ai/src/api.test.ts index 6ce353107ac..76a9b4523c2 100644 --- a/packages/ai/src/api.test.ts +++ b/packages/ai/src/api.test.ts @@ -21,7 +21,7 @@ import { expect } from 'chai'; import { AI } from './public-types'; import { GenerativeModel } from './models/generative-model'; import { VertexAIBackend } from './backend'; -import { AI_TYPE } from './constants'; +import { AI_TYPE, DEFAULT_HYBRID_IN_CLOUD_MODEL } from './constants'; const fakeAI: AI = { app: { @@ -107,7 +107,7 @@ describe('Top level API', () => { mode: 'only_on_device' }); expect(genModel.model).to.equal( - `publishers/google/models/${GenerativeModel.DEFAULT_HYBRID_IN_CLOUD_MODEL}` + `publishers/google/models/${DEFAULT_HYBRID_IN_CLOUD_MODEL}` ); }); it('getGenerativeModel with HybridParams honors a model override', () => { diff --git a/packages/ai/src/api.ts b/packages/ai/src/api.ts index 6ae0acadf20..62c7c27f07a 100644 --- a/packages/ai/src/api.ts +++ b/packages/ai/src/api.ts @@ -18,7 +18,7 @@ import { FirebaseApp, getApp, _getProvider } from '@firebase/app'; import { Provider } from '@firebase/component'; import { getModularInstance } from '@firebase/util'; -import { AI_TYPE } from './constants'; +import { AI_TYPE, DEFAULT_HYBRID_IN_CLOUD_MODEL } from './constants'; import { AIService } from './service'; import { AI, AIOptions } from './public-types'; import { @@ -105,7 +105,7 @@ export function getGenerativeModel( let inCloudParams: ModelParams; if (hybridParams.mode) { inCloudParams = hybridParams.inCloudParams || { - model: GenerativeModel.DEFAULT_HYBRID_IN_CLOUD_MODEL + model: DEFAULT_HYBRID_IN_CLOUD_MODEL }; } else { inCloudParams = modelParams as ModelParams; @@ -118,6 +118,7 @@ export function getGenerativeModel( ); } let chromeAdapter: ChromeAdapterImpl | undefined; + // Do not initialize a ChromeAdapter if we are not in hybrid mode. if (typeof window !== 'undefined' && hybridParams.mode) { chromeAdapter = new ChromeAdapterImpl( window.LanguageModel as LanguageModel, diff --git a/packages/ai/src/constants.ts b/packages/ai/src/constants.ts index cb54567735a..b6bd8e220ad 100644 --- a/packages/ai/src/constants.ts +++ b/packages/ai/src/constants.ts @@ -30,3 +30,8 @@ export const PACKAGE_VERSION = version; export const LANGUAGE_TAG = 'gl-js'; export const DEFAULT_FETCH_TIMEOUT_MS = 180 * 1000; + +/** + * Defines the name of the default in-cloud model to use for hybrid inference. + */ +export const DEFAULT_HYBRID_IN_CLOUD_MODEL = 'gemini-2.0-flash-lite'; diff --git a/packages/ai/src/methods/chrome-adapter.test.ts b/packages/ai/src/methods/chrome-adapter.test.ts index fdc84be71be..fe6b7144724 100644 --- a/packages/ai/src/methods/chrome-adapter.test.ts +++ b/packages/ai/src/methods/chrome-adapter.test.ts @@ -27,7 +27,7 @@ import { LanguageModelMessage } from '../types/language-model'; import { match, stub } from 'sinon'; -import { GenerateContentRequest, AIErrorCode } from '../types'; +import { GenerateContentRequest, AIErrorCode, InferenceMode } from '../types'; import { Schema } from '../api'; use(sinonChai); @@ -64,7 +64,7 @@ describe('ChromeAdapter', () => { ).resolves(Availability.available); const adapter = new ChromeAdapterImpl( languageModelProvider, - 'prefer_on_device' + InferenceMode.PREFER_ON_DEVICE ); await adapter.isAvailable({ contents: [ @@ -92,7 +92,7 @@ describe('ChromeAdapter', () => { } as LanguageModelCreateOptions; const adapter = new ChromeAdapterImpl( languageModelProvider, - 'prefer_on_device', + InferenceMode.PREFER_ON_DEVICE, { createOptions } @@ -126,7 +126,10 @@ describe('ChromeAdapter', () => { ).to.be.false; }); it('returns false if LanguageModel API is undefined', async () => { - const adapter = new ChromeAdapterImpl(undefined, 'prefer_on_device'); + const adapter = new ChromeAdapterImpl( + undefined, + InferenceMode.PREFER_ON_DEVICE + ); expect( await adapter.isAvailable({ contents: [] @@ -138,7 +141,7 @@ describe('ChromeAdapter', () => { { availability: async () => Availability.available } as LanguageModel, - 'prefer_on_device' + InferenceMode.PREFER_ON_DEVICE ); expect( await adapter.isAvailable({ @@ -151,7 +154,7 @@ describe('ChromeAdapter', () => { { availability: async () => Availability.available } as LanguageModel, - 'prefer_on_device' + InferenceMode.PREFER_ON_DEVICE ); expect( await adapter.isAvailable({ @@ -169,7 +172,7 @@ describe('ChromeAdapter', () => { { availability: async () => Availability.available } as LanguageModel, - 'prefer_on_device' + InferenceMode.PREFER_ON_DEVICE ); for (const mimeType of ChromeAdapterImpl.SUPPORTED_MIME_TYPES) { expect( @@ -197,7 +200,7 @@ describe('ChromeAdapter', () => { } as LanguageModel; const adapter = new ChromeAdapterImpl( languageModelProvider, - 'prefer_on_device' + InferenceMode.PREFER_ON_DEVICE ); expect( await adapter.isAvailable({ @@ -226,7 +229,7 @@ describe('ChromeAdapter', () => { } as LanguageModelCreateOptions; const adapter = new ChromeAdapterImpl( languageModelProvider, - 'prefer_on_device', + InferenceMode.PREFER_ON_DEVICE, { createOptions } ); expect( @@ -249,7 +252,7 @@ describe('ChromeAdapter', () => { ); const adapter = new ChromeAdapterImpl( languageModelProvider, - 'prefer_on_device' + InferenceMode.PREFER_ON_DEVICE ); await adapter.isAvailable({ contents: [{ role: 'user', parts: [{ text: 'hi' }] }] @@ -273,7 +276,7 @@ describe('ChromeAdapter', () => { ); const adapter = new ChromeAdapterImpl( languageModelProvider, - 'prefer_on_device' + InferenceMode.PREFER_ON_DEVICE ); await adapter.isAvailable({ contents: [{ role: 'user', parts: [{ text: 'hi' }] }] @@ -291,7 +294,7 @@ describe('ChromeAdapter', () => { } as LanguageModel; const adapter = new ChromeAdapterImpl( languageModelProvider, - 'prefer_on_device' + InferenceMode.PREFER_ON_DEVICE ); expect( await adapter.isAvailable({ @@ -312,7 +315,7 @@ describe('ChromeAdapter', () => { AIError, 'Chrome AI requested for unsupported browser version.' ) - .and.have.property('code', AIErrorCode.REQUEST_ERROR); + .and.have.property('code', AIErrorCode.UNSUPPORTED); }); it('generates content', async () => { const languageModelProvider = { @@ -333,7 +336,7 @@ describe('ChromeAdapter', () => { } as LanguageModelCreateOptions; const adapter = new ChromeAdapterImpl( languageModelProvider, - 'prefer_on_device', + InferenceMode.PREFER_ON_DEVICE, { createOptions } ); const request = { @@ -384,7 +387,7 @@ describe('ChromeAdapter', () => { } as LanguageModelCreateOptions; const adapter = new ChromeAdapterImpl( languageModelProvider, - 'prefer_on_device', + InferenceMode.PREFER_ON_DEVICE, { createOptions } ); const request = { @@ -450,7 +453,7 @@ describe('ChromeAdapter', () => { }; const adapter = new ChromeAdapterImpl( languageModelProvider, - 'prefer_on_device', + InferenceMode.PREFER_ON_DEVICE, { promptOptions } ); const request = { @@ -483,7 +486,7 @@ describe('ChromeAdapter', () => { } as LanguageModel; const adapter = new ChromeAdapterImpl( languageModelProvider, - 'prefer_on_device' + InferenceMode.PREFER_ON_DEVICE ); const request = { contents: [{ role: 'model', parts: [{ text: 'unused' }] }] @@ -519,7 +522,7 @@ describe('ChromeAdapter', () => { const adapter = new ChromeAdapterImpl( languageModelProvider, - 'prefer_on_device' + InferenceMode.PREFER_ON_DEVICE ); const countTokenRequest = { @@ -563,7 +566,7 @@ describe('ChromeAdapter', () => { } as LanguageModelCreateOptions; const adapter = new ChromeAdapterImpl( languageModelProvider, - 'prefer_on_device', + InferenceMode.PREFER_ON_DEVICE, { createOptions } ); const request = { @@ -611,7 +614,7 @@ describe('ChromeAdapter', () => { } as LanguageModelCreateOptions; const adapter = new ChromeAdapterImpl( languageModelProvider, - 'prefer_on_device', + InferenceMode.PREFER_ON_DEVICE, { createOptions } ); const request = { @@ -670,7 +673,7 @@ describe('ChromeAdapter', () => { }; const adapter = new ChromeAdapterImpl( languageModelProvider, - 'prefer_on_device', + InferenceMode.PREFER_ON_DEVICE, { promptOptions } ); const request = { @@ -705,7 +708,7 @@ describe('ChromeAdapter', () => { } as LanguageModel; const adapter = new ChromeAdapterImpl( languageModelProvider, - 'prefer_on_device' + InferenceMode.PREFER_ON_DEVICE ); const request = { contents: [{ role: 'model', parts: [{ text: 'unused' }] }] diff --git a/packages/ai/src/models/generative-model.test.ts b/packages/ai/src/models/generative-model.test.ts index 5c7e80436e1..6ea7470ef5f 100644 --- a/packages/ai/src/models/generative-model.test.ts +++ b/packages/ai/src/models/generative-model.test.ts @@ -61,6 +61,7 @@ describe('GenerativeModel', () => { }, systemInstruction: { role: 'system', parts: [{ text: 'be friendly' }] } }, + {}, new ChromeAdapterImpl() ); expect(genModel.tools?.length).to.equal(1); @@ -99,6 +100,7 @@ describe('GenerativeModel', () => { model: 'my-model', systemInstruction: 'be friendly' }, + {}, new ChromeAdapterImpl() ); expect(genModel.systemInstruction?.parts[0].text).to.equal('be friendly'); @@ -142,6 +144,7 @@ describe('GenerativeModel', () => { }, systemInstruction: { role: 'system', parts: [{ text: 'be friendly' }] } }, + {}, new ChromeAdapterImpl() ); expect(genModel.tools?.length).to.equal(1); @@ -193,6 +196,7 @@ describe('GenerativeModel', () => { topK: 1 } }, + {}, new ChromeAdapterImpl() ); const chatSession = genModel.startChat(); @@ -210,6 +214,7 @@ describe('GenerativeModel', () => { topK: 1 } }, + {}, new ChromeAdapterImpl() ); const chatSession = genModel.startChat({ @@ -237,6 +242,7 @@ describe('GenerativeModel', () => { topK: 1 } }, + {}, new ChromeAdapterImpl() ); expect(genModel.tools?.length).to.equal(1); @@ -276,6 +282,7 @@ describe('GenerativeModel', () => { model: 'my-model', systemInstruction: 'be friendly' }, + {}, new ChromeAdapterImpl() ); expect(genModel.systemInstruction?.parts[0].text).to.equal('be friendly'); @@ -315,6 +322,7 @@ describe('GenerativeModel', () => { responseMimeType: 'image/jpeg' } }, + {}, new ChromeAdapterImpl() ); expect(genModel.tools?.length).to.equal(1); @@ -369,6 +377,7 @@ describe('GenerativeModel', () => { const genModel = new GenerativeModel( fakeAI, { model: 'my-model' }, + {}, new ChromeAdapterImpl() ); const mockResponse = getMockResponse( diff --git a/packages/ai/src/models/generative-model.ts b/packages/ai/src/models/generative-model.ts index d0376d9bace..ffce645eeb1 100644 --- a/packages/ai/src/models/generative-model.ts +++ b/packages/ai/src/models/generative-model.ts @@ -50,10 +50,6 @@ import { ChromeAdapter } from '../types/chrome-adapter'; * @public */ export class GenerativeModel extends AIModel { - /** - * Defines the name of the default in-cloud model to use for hybrid inference. - */ - static DEFAULT_HYBRID_IN_CLOUD_MODEL = 'gemini-2.0-flash-lite'; generationConfig: GenerationConfig; safetySettings: SafetySetting[]; requestOptions?: RequestOptions; diff --git a/packages/ai/src/types/chrome-adapter.ts b/packages/ai/src/types/chrome-adapter.ts index 6e7a1313535..ec351b3d06a 100644 --- a/packages/ai/src/types/chrome-adapter.ts +++ b/packages/ai/src/types/chrome-adapter.ts @@ -1,16 +1,31 @@ -import { CountTokensRequest, GenerateContentRequest } from "./requests"; +/** + * @license + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { CountTokensRequest, GenerateContentRequest } from './requests'; /** * Defines an inference "backend" that uses Chrome's on-device model, * and encapsulates logic for detecting when on-device is possible. - * + * * @public */ export interface ChromeAdapter { - isAvailable(request: GenerateContentRequest): Promise; - countTokens(_request: CountTokensRequest): Promise - generateContent(request: GenerateContentRequest): Promise - generateContentStream( - request: GenerateContentRequest - ): Promise -} \ No newline at end of file + isAvailable(request: GenerateContentRequest): Promise; + countTokens(_request: CountTokensRequest): Promise; + generateContent(request: GenerateContentRequest): Promise; + generateContentStream(request: GenerateContentRequest): Promise; +} diff --git a/packages/ai/src/types/enums.ts b/packages/ai/src/types/enums.ts index 14e556408ee..d02b3be4ebe 100644 --- a/packages/ai/src/types/enums.ts +++ b/packages/ai/src/types/enums.ts @@ -340,7 +340,7 @@ export type ResponseModality = (typeof ResponseModality)[keyof typeof ResponseModality]; /** - * EXPERIMENTAL FEATURE + * (EXPERIMENTAL) * Determines whether inference happens on-device or in-cloud. * @public */ @@ -351,7 +351,7 @@ export const InferenceMode = { } as const; /** - * EXPERIMENTAL FEATURE + * (EXPERIMENTAL) * Determines whether inference happens on-device or in-cloud. * @public */ diff --git a/packages/ai/src/types/language-model.ts b/packages/ai/src/types/language-model.ts index 19a867f85a0..0d1db8d4971 100644 --- a/packages/ai/src/types/language-model.ts +++ b/packages/ai/src/types/language-model.ts @@ -16,10 +16,10 @@ */ /** * The subset of the Prompt API - * (see {@link https://github.com/webmachinelearning/prompt-api#full-api-surface-in-web-idl }) + * (see {@link https://github.com/webmachinelearning/prompt-api#full-api-surface-in-web-idl } * required for hybrid functionality. - * - * @public + * + * @internal */ export interface LanguageModel extends EventTarget { create(options?: LanguageModelCreateOptions): Promise; @@ -40,7 +40,7 @@ export interface LanguageModel extends EventTarget { } /** - * @public + * @internal */ export enum Availability { 'unavailable' = 'unavailable', @@ -50,6 +50,8 @@ export enum Availability { } /** + * (EXPERIMENTAL) + * Used to configure the creation of an on-device language model session. * @public */ export interface LanguageModelCreateCoreOptions { @@ -59,6 +61,8 @@ export interface LanguageModelCreateCoreOptions { } /** + * (EXPERIMENTAL) + * Used to configure the creation of an on-device language model session. * @public */ export interface LanguageModelCreateOptions @@ -68,6 +72,8 @@ export interface LanguageModelCreateOptions } /** + * (EXPERIMENTAL) + * Options for an on-device language model prompt. * @public */ export interface LanguageModelPromptOptions { @@ -76,18 +82,24 @@ export interface LanguageModelPromptOptions { } /** + * (EXPERIMENTAL) + * Options for an on-device language model expected inputs. * @public - */export interface LanguageModelExpected { + */ export interface LanguageModelExpected { type: LanguageModelMessageType; languages?: string[]; } /** + * (EXPERIMENTAL) + * An on-device language model prompt. * @public */ export type LanguageModelPrompt = LanguageModelMessage[]; /** + * (EXPERIMENTAL) + * An on-device language model message. * @public */ export interface LanguageModelMessage { @@ -96,6 +108,8 @@ export interface LanguageModelMessage { } /** + * (EXPERIMENTAL) + * An on-device language model content object. * @public */ export interface LanguageModelMessageContent { @@ -104,16 +118,22 @@ export interface LanguageModelMessageContent { } /** + * (EXPERIMENTAL) + * Allowable roles for on-device language model usage. * @public */ export type LanguageModelMessageRole = 'system' | 'user' | 'assistant'; /** + * (EXPERIMENTAL) + * Allowable types for on-device language model messages. * @public */ export type LanguageModelMessageType = 'text' | 'image' | 'audio'; /** + * (EXPERIMENTAL) + * Content formats that can be provided as on-device message content. * @public */ export type LanguageModelMessageContentValue = diff --git a/packages/ai/src/types/requests.ts b/packages/ai/src/types/requests.ts index a039d686edf..0673db17497 100644 --- a/packages/ai/src/types/requests.ts +++ b/packages/ai/src/types/requests.ts @@ -277,8 +277,9 @@ export interface FunctionCallingConfig { } /** + * (EXPERIMENTAL) * Encapsulates configuration for on-device inference. - * + * * @public */ export interface OnDeviceParams { @@ -287,6 +288,7 @@ export interface OnDeviceParams { } /** + * (EXPERIMENTAL) * Configures hybrid inference. * @public */ From f9f641def2092bde8dde3a512fa863b8764290d8 Mon Sep 17 00:00:00 2001 From: Christina Holland Date: Thu, 31 Jul 2025 09:58:26 -0700 Subject: [PATCH 10/20] Fill out chrome-adapter doc comments --- common/api-review/ai.api.md | 4 --- packages/ai/src/types/chrome-adapter.ts | 39 +++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 4 deletions(-) diff --git a/common/api-review/ai.api.md b/common/api-review/ai.api.md index 7cc599f92f9..fff19dfecbb 100644 --- a/common/api-review/ai.api.md +++ b/common/api-review/ai.api.md @@ -139,13 +139,9 @@ export class ChatSession { // @public export interface ChromeAdapter { - // (undocumented) countTokens(_request: CountTokensRequest): Promise; - // (undocumented) generateContent(request: GenerateContentRequest): Promise; - // (undocumented) generateContentStream(request: GenerateContentRequest): Promise; - // (undocumented) isAvailable(request: GenerateContentRequest): Promise; } diff --git a/packages/ai/src/types/chrome-adapter.ts b/packages/ai/src/types/chrome-adapter.ts index ec351b3d06a..584de7f7daf 100644 --- a/packages/ai/src/types/chrome-adapter.ts +++ b/packages/ai/src/types/chrome-adapter.ts @@ -21,11 +21,50 @@ import { CountTokensRequest, GenerateContentRequest } from './requests'; * Defines an inference "backend" that uses Chrome's on-device model, * and encapsulates logic for detecting when on-device is possible. * + * These methods should not be called directly by the user. + * * @public */ export interface ChromeAdapter { + /** + * Checks if a given request can be made on-device. + * + *

    Encapsulates a few concerns: + *
  1. the mode
    2. + *
  2. API existence
    3. + *
  3. prompt formatting
    4. + *
  4. model availability, including triggering download if necessary
    5. + *
+ * + *

Pros: callers needn't be concerned with details of on-device availability.

+ *

Cons: this method spans a few concerns and splits request validation from usage. + * If instance variables weren't already part of the API, we could consider a better + * separation of concerns.

+ */ isAvailable(request: GenerateContentRequest): Promise; + + /** + * Stub - not yet available for on-device. + */ countTokens(_request: CountTokensRequest): Promise; + + /** + * Generates content on device. + * + *

This is comparable to {@link GenerativeModel.generateContent} for generating content in + * Cloud.

+ * @param request - a standard Firebase AI {@link GenerateContentRequest} + * @returns {@link Response}, so we can reuse common response formatting. + */ generateContent(request: GenerateContentRequest): Promise; + + /** + * Generates content stream on device. + * + *

This is comparable to {@link GenerativeModel.generateContentStream} for generating content in + * Cloud.

+ * @param request - a standard Firebase AI {@link GenerateContentRequest} + * @returns {@link Response}, so we can reuse common response formatting. + */ generateContentStream(request: GenerateContentRequest): Promise; } From 5df0f21d551724c567b201078dd3998103e6c08b Mon Sep 17 00:00:00 2001 From: Christina Holland Date: Thu, 31 Jul 2025 10:18:12 -0700 Subject: [PATCH 11/20] add experimental tag to ChromeAdapter --- docs-devsite/ai.chromeadapter.md | 36 ++++++++++++++++++++----- docs-devsite/ai.md | 2 +- packages/ai/src/types/chrome-adapter.ts | 2 ++ 3 files changed, 33 insertions(+), 7 deletions(-) diff --git a/docs-devsite/ai.chromeadapter.md b/docs-devsite/ai.chromeadapter.md index 395a71c79d2..c4531144ea9 100644 --- a/docs-devsite/ai.chromeadapter.md +++ b/docs-devsite/ai.chromeadapter.md @@ -10,8 +10,12 @@ https://github.com/firebase/firebase-js-sdk {% endcomment %} # ChromeAdapter interface +(EXPERIMENTAL) + Defines an inference "backend" that uses Chrome's on-device model, and encapsulates logic for detecting when on-device is possible. +These methods should not be called directly by the user. + Signature: ```typescript @@ -22,13 +26,15 @@ export interface ChromeAdapter | Method | Description | | --- | --- | -| [countTokens(\_request)](./ai.chromeadapter.md#chromeadaptercounttokens) | | -| [generateContent(request)](./ai.chromeadapter.md#chromeadaptergeneratecontent) | | -| [generateContentStream(request)](./ai.chromeadapter.md#chromeadaptergeneratecontentstream) | | -| [isAvailable(request)](./ai.chromeadapter.md#chromeadapterisavailable) | | +| [countTokens(\_request)](./ai.chromeadapter.md#chromeadaptercounttokens) | Stub - not yet available for on-device. | +| [generateContent(request)](./ai.chromeadapter.md#chromeadaptergeneratecontent) | Generates content on device.

This is comparable to [GenerativeModel.generateContent()](./ai.generativemodel.md#generativemodelgeneratecontent) for generating content in Cloud.

| +| [generateContentStream(request)](./ai.chromeadapter.md#chromeadaptergeneratecontentstream) | Generates content stream on device.

This is comparable to [GenerativeModel.generateContentStream()](./ai.generativemodel.md#generativemodelgeneratecontentstream) for generating content in Cloud.

| +| [isAvailable(request)](./ai.chromeadapter.md#chromeadapterisavailable) | Checks if a given request can be made on-device.
    Encapsulates a few concerns:
  1. the mode
  2. API existence
  3. prompt formatting
  4. model availability, including triggering download if necessary

Pros: callers needn't be concerned with details of on-device availability.

Cons: this method spans a few concerns and splits request validation from usage. If instance variables weren't already part of the API, we could consider a better separation of concerns.

| ## ChromeAdapter.countTokens() +Stub - not yet available for on-device. + Signature: ```typescript @@ -47,6 +53,10 @@ Promise<Response> ## ChromeAdapter.generateContent() +Generates content on device. + +

This is comparable to [GenerativeModel.generateContent()](./ai.generativemodel.md#generativemodelgeneratecontent) for generating content in Cloud.

+ Signature: ```typescript @@ -57,14 +67,20 @@ generateContent(request: GenerateContentRequest): Promise; | Parameter | Type | Description | | --- | --- | --- | -| request | [GenerateContentRequest](./ai.generatecontentrequest.md#generatecontentrequest_interface) | | +| request | [GenerateContentRequest](./ai.generatecontentrequest.md#generatecontentrequest_interface) | a standard Firebase AI [GenerateContentRequest](./ai.generatecontentrequest.md#generatecontentrequest_interface) | Returns: Promise<Response> +, so we can reuse common response formatting. + ## ChromeAdapter.generateContentStream() +Generates content stream on device. + +

This is comparable to [GenerativeModel.generateContentStream()](./ai.generativemodel.md#generativemodelgeneratecontentstream) for generating content in Cloud.

+ Signature: ```typescript @@ -75,14 +91,22 @@ generateContentStream(request: GenerateContentRequest): Promise; | Parameter | Type | Description | | --- | --- | --- | -| request | [GenerateContentRequest](./ai.generatecontentrequest.md#generatecontentrequest_interface) | | +| request | [GenerateContentRequest](./ai.generatecontentrequest.md#generatecontentrequest_interface) | a standard Firebase AI [GenerateContentRequest](./ai.generatecontentrequest.md#generatecontentrequest_interface) | Returns: Promise<Response> +, so we can reuse common response formatting. + ## ChromeAdapter.isAvailable() +Checks if a given request can be made on-device. + +
    Encapsulates a few concerns:
  1. the mode
  2. API existence
  3. prompt formatting
  4. model availability, including triggering download if necessary
+ +

Pros: callers needn't be concerned with details of on-device availability.

Cons: this method spans a few concerns and splits request validation from usage. If instance variables weren't already part of the API, we could consider a better separation of concerns.

+ Signature: ```typescript diff --git a/docs-devsite/ai.md b/docs-devsite/ai.md index 520163a369f..0f5dd4628bd 100644 --- a/docs-devsite/ai.md +++ b/docs-devsite/ai.md @@ -51,7 +51,7 @@ The Firebase AI Web SDK. | [AI](./ai.ai.md#ai_interface) | An instance of the Firebase AI SDK.Do not create this instance directly. Instead, use [getAI()](./ai.md#getai_a94a413). | | [AIOptions](./ai.aioptions.md#aioptions_interface) | Options for initializing the AI service using [getAI()](./ai.md#getai_a94a413). This allows specifying which backend to use (Vertex AI Gemini API or Gemini Developer API) and configuring its specific options (like location for Vertex AI). | | [BaseParams](./ai.baseparams.md#baseparams_interface) | Base parameters for a number of methods. | -| [ChromeAdapter](./ai.chromeadapter.md#chromeadapter_interface) | Defines an inference "backend" that uses Chrome's on-device model, and encapsulates logic for detecting when on-device is possible. | +| [ChromeAdapter](./ai.chromeadapter.md#chromeadapter_interface) | (EXPERIMENTAL)Defines an inference "backend" that uses Chrome's on-device model, and encapsulates logic for detecting when on-device is possible.These methods should not be called directly by the user. | | [Citation](./ai.citation.md#citation_interface) | A single citation. | | [CitationMetadata](./ai.citationmetadata.md#citationmetadata_interface) | Citation metadata that may be found on a [GenerateContentCandidate](./ai.generatecontentcandidate.md#generatecontentcandidate_interface). | | [Content](./ai.content.md#content_interface) | Content type for both prompts and response candidates. | diff --git a/packages/ai/src/types/chrome-adapter.ts b/packages/ai/src/types/chrome-adapter.ts index 584de7f7daf..5657302a60f 100644 --- a/packages/ai/src/types/chrome-adapter.ts +++ b/packages/ai/src/types/chrome-adapter.ts @@ -18,6 +18,8 @@ import { CountTokensRequest, GenerateContentRequest } from './requests'; /** + * (EXPERIMENTAL) + * * Defines an inference "backend" that uses Chrome's on-device model, * and encapsulates logic for detecting when on-device is possible. * From ca3fc07efffdf142dc569d61c663634963fce724 Mon Sep 17 00:00:00 2001 From: Christina Holland Date: Thu, 31 Jul 2025 18:03:29 -0700 Subject: [PATCH 12/20] change back to private --- common/api-review/ai.api.md | 2 + docs-devsite/ai.chromeadapter.md | 18 ++++++++- packages/ai/src/methods/chat-session.test.ts | 14 +++++-- .../ai/src/methods/chrome-adapter.test.ts | 39 +++++++++---------- packages/ai/src/methods/chrome-adapter.ts | 27 ++++++++++--- packages/ai/src/methods/count-tokens.test.ts | 20 ++++++---- .../ai/src/methods/generate-content.test.ts | 11 +++++- .../ai/src/models/generative-model.test.ts | 26 ++++++++----- packages/ai/src/types/chrome-adapter.ts | 7 ++-- packages/ai/src/types/language-model.ts | 8 ++-- 10 files changed, 113 insertions(+), 59 deletions(-) diff --git a/common/api-review/ai.api.md b/common/api-review/ai.api.md index fff19dfecbb..695f0cd7cca 100644 --- a/common/api-review/ai.api.md +++ b/common/api-review/ai.api.md @@ -143,6 +143,8 @@ export interface ChromeAdapter { generateContent(request: GenerateContentRequest): Promise; generateContentStream(request: GenerateContentRequest): Promise; isAvailable(request: GenerateContentRequest): Promise; + // (undocumented) + mode: InferenceMode; } // @public diff --git a/docs-devsite/ai.chromeadapter.md b/docs-devsite/ai.chromeadapter.md index c4531144ea9..afa6ce46ff6 100644 --- a/docs-devsite/ai.chromeadapter.md +++ b/docs-devsite/ai.chromeadapter.md @@ -22,6 +22,12 @@ These methods should not be called directly by the user. export interface ChromeAdapter ``` +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [mode](./ai.chromeadapter.md#chromeadaptermode) | [InferenceMode](./ai.md#inferencemode) | | + ## Methods | Method | Description | @@ -31,6 +37,14 @@ export interface ChromeAdapter | [generateContentStream(request)](./ai.chromeadapter.md#chromeadaptergeneratecontentstream) | Generates content stream on device.

This is comparable to [GenerativeModel.generateContentStream()](./ai.generativemodel.md#generativemodelgeneratecontentstream) for generating content in Cloud.

| | [isAvailable(request)](./ai.chromeadapter.md#chromeadapterisavailable) | Checks if a given request can be made on-device.
    Encapsulates a few concerns:
  1. the mode
  2. API existence
  3. prompt formatting
  4. model availability, including triggering download if necessary

Pros: callers needn't be concerned with details of on-device availability.

Cons: this method spans a few concerns and splits request validation from usage. If instance variables weren't already part of the API, we could consider a better separation of concerns.

| +## ChromeAdapter.mode + +Signature: + +```typescript +mode: InferenceMode; +``` + ## ChromeAdapter.countTokens() Stub - not yet available for on-device. @@ -73,7 +87,7 @@ generateContent(request: GenerateContentRequest): Promise; Promise<Response> -, so we can reuse common response formatting. +Response, so we can reuse common response formatting. ## ChromeAdapter.generateContentStream() @@ -97,7 +111,7 @@ generateContentStream(request: GenerateContentRequest): Promise; Promise<Response> -, so we can reuse common response formatting. +Response, so we can reuse common response formatting. ## ChromeAdapter.isAvailable() diff --git a/packages/ai/src/methods/chat-session.test.ts b/packages/ai/src/methods/chat-session.test.ts index 0883920847f..f523672f5e2 100644 --- a/packages/ai/src/methods/chat-session.test.ts +++ b/packages/ai/src/methods/chat-session.test.ts @@ -20,7 +20,7 @@ import { match, restore, stub, useFakeTimers } from 'sinon'; import sinonChai from 'sinon-chai'; import chaiAsPromised from 'chai-as-promised'; import * as generateContentMethods from './generate-content'; -import { GenerateContentStreamResult } from '../types'; +import { GenerateContentStreamResult, InferenceMode } from '../types'; import { ChatSession } from './chat-session'; import { ApiSettings } from '../types/internal'; import { VertexAIBackend } from '../backend'; @@ -37,6 +37,12 @@ const fakeApiSettings: ApiSettings = { backend: new VertexAIBackend() }; +const fakeChromeAdapter = new ChromeAdapterImpl( + // @ts-expect-error + undefined, + InferenceMode.PREFER_ON_DEVICE +); + describe('ChatSession', () => { afterEach(() => { restore(); @@ -50,7 +56,7 @@ describe('ChatSession', () => { const chatSession = new ChatSession( fakeApiSettings, 'a-model', - new ChromeAdapterImpl() + fakeChromeAdapter ); await expect(chatSession.sendMessage('hello')).to.be.rejected; expect(generateContentStub).to.be.calledWith( @@ -71,7 +77,7 @@ describe('ChatSession', () => { const chatSession = new ChatSession( fakeApiSettings, 'a-model', - new ChromeAdapterImpl() + fakeChromeAdapter ); await expect(chatSession.sendMessageStream('hello')).to.be.rejected; expect(generateContentStreamStub).to.be.calledWith( @@ -94,7 +100,7 @@ describe('ChatSession', () => { const chatSession = new ChatSession( fakeApiSettings, 'a-model', - new ChromeAdapterImpl() + fakeChromeAdapter ); await chatSession.sendMessageStream('hello'); expect(generateContentStreamStub).to.be.calledWith( diff --git a/packages/ai/src/methods/chrome-adapter.test.ts b/packages/ai/src/methods/chrome-adapter.test.ts index fe6b7144724..0392e5bba3f 100644 --- a/packages/ai/src/methods/chrome-adapter.test.ts +++ b/packages/ai/src/methods/chrome-adapter.test.ts @@ -56,12 +56,12 @@ describe('ChromeAdapter', () => { describe('constructor', () => { it('sets image as expected input type by default', async () => { const languageModelProvider = { - availability: () => Promise.resolve(Availability.available) + availability: () => Promise.resolve(Availability.AVAILABLE) } as LanguageModel; const availabilityStub = stub( languageModelProvider, 'availability' - ).resolves(Availability.available); + ).resolves(Availability.AVAILABLE); const adapter = new ChromeAdapterImpl( languageModelProvider, InferenceMode.PREFER_ON_DEVICE @@ -80,12 +80,12 @@ describe('ChromeAdapter', () => { }); it('honors explicitly set expected inputs', async () => { const languageModelProvider = { - availability: () => Promise.resolve(Availability.available) + availability: () => Promise.resolve(Availability.AVAILABLE) } as LanguageModel; const availabilityStub = stub( languageModelProvider, 'availability' - ).resolves(Availability.available); + ).resolves(Availability.AVAILABLE); const createOptions = { // Explicitly sets expected inputs. expectedInputs: [{ type: 'text' }] @@ -109,16 +109,11 @@ describe('ChromeAdapter', () => { }); }); describe('isAvailable', () => { - it('returns false if mode is undefined', async () => { - const adapter = new ChromeAdapterImpl(); - expect( - await adapter.isAvailable({ - contents: [] - }) - ).to.be.false; - }); it('returns false if mode is only cloud', async () => { - const adapter = new ChromeAdapterImpl(undefined, 'only_in_cloud'); + const adapter = new ChromeAdapterImpl( + {} as LanguageModel, + 'only_in_cloud' + ); expect( await adapter.isAvailable({ contents: [] @@ -127,6 +122,7 @@ describe('ChromeAdapter', () => { }); it('returns false if LanguageModel API is undefined', async () => { const adapter = new ChromeAdapterImpl( + // @ts-expect-error undefined, InferenceMode.PREFER_ON_DEVICE ); @@ -139,7 +135,7 @@ describe('ChromeAdapter', () => { it('returns false if request contents empty', async () => { const adapter = new ChromeAdapterImpl( { - availability: async () => Availability.available + availability: async () => Availability.AVAILABLE } as LanguageModel, InferenceMode.PREFER_ON_DEVICE ); @@ -152,7 +148,7 @@ describe('ChromeAdapter', () => { it('returns false if request content has "function" role', async () => { const adapter = new ChromeAdapterImpl( { - availability: async () => Availability.available + availability: async () => Availability.AVAILABLE } as LanguageModel, InferenceMode.PREFER_ON_DEVICE ); @@ -170,7 +166,7 @@ describe('ChromeAdapter', () => { it('returns true if request has image with supported mime type', async () => { const adapter = new ChromeAdapterImpl( { - availability: async () => Availability.available + availability: async () => Availability.AVAILABLE } as LanguageModel, InferenceMode.PREFER_ON_DEVICE ); @@ -196,7 +192,7 @@ describe('ChromeAdapter', () => { }); it('returns true if model is readily available', async () => { const languageModelProvider = { - availability: () => Promise.resolve(Availability.available) + availability: () => Promise.resolve(Availability.AVAILABLE) } as LanguageModel; const adapter = new ChromeAdapterImpl( languageModelProvider, @@ -218,7 +214,7 @@ describe('ChromeAdapter', () => { }); it('returns false and triggers download when model is available after download', async () => { const languageModelProvider = { - availability: () => Promise.resolve(Availability.downloadable), + availability: () => Promise.resolve(Availability.DOWNLOADABLE), create: () => Promise.resolve({}) } as LanguageModel; const createStub = stub(languageModelProvider, 'create').resolves( @@ -241,7 +237,7 @@ describe('ChromeAdapter', () => { }); it('avoids redundant downloads', async () => { const languageModelProvider = { - availability: () => Promise.resolve(Availability.downloadable), + availability: () => Promise.resolve(Availability.DOWNLOADABLE), create: () => Promise.resolve({}) } as LanguageModel; const downloadPromise = new Promise(() => { @@ -264,7 +260,7 @@ describe('ChromeAdapter', () => { }); it('clears state when download completes', async () => { const languageModelProvider = { - availability: () => Promise.resolve(Availability.downloadable), + availability: () => Promise.resolve(Availability.DOWNLOADABLE), create: () => Promise.resolve({}) } as LanguageModel; let resolveDownload; @@ -289,7 +285,7 @@ describe('ChromeAdapter', () => { }); it('returns false when model is never available', async () => { const languageModelProvider = { - availability: () => Promise.resolve(Availability.unavailable), + availability: () => Promise.resolve(Availability.UNAVAILABLE), create: () => Promise.resolve({}) } as LanguageModel; const adapter = new ChromeAdapterImpl( @@ -305,6 +301,7 @@ describe('ChromeAdapter', () => { }); describe('generateContent', () => { it('throws if Chrome API is undefined', async () => { + // @ts-expect-error const adapter = new ChromeAdapterImpl(undefined, 'only_on_device'); await expect( adapter.generateContent({ diff --git a/packages/ai/src/methods/chrome-adapter.ts b/packages/ai/src/methods/chrome-adapter.ts index 71a87b55a46..a3c13e514d7 100644 --- a/packages/ai/src/methods/chrome-adapter.ts +++ b/packages/ai/src/methods/chrome-adapter.ts @@ -47,8 +47,8 @@ export class ChromeAdapterImpl implements ChromeAdapter { private downloadPromise: Promise | undefined; private oldSession: LanguageModel | undefined; constructor( - private languageModelProvider?: LanguageModel, - private mode?: InferenceMode, + private languageModelProvider: LanguageModel, + private mode: InferenceMode, private onDeviceParams: OnDeviceParams = { createOptions: { // Defaults to support image inputs for convenience. @@ -79,7 +79,7 @@ export class ChromeAdapterImpl implements ChromeAdapter { ); return false; } - if (this.mode === 'only_in_cloud') { + if (this.mode === InferenceMode.ONLY_IN_CLOUD) { logger.debug( `On-device inference unavailable because mode is "only_in_cloud".` ); @@ -89,12 +89,27 @@ export class ChromeAdapterImpl implements ChromeAdapter { // Triggers out-of-band download so model will eventually become available. const availability = await this.downloadIfAvailable(); - if (this.mode === 'only_on_device') { + if (this.mode === InferenceMode.ONLY_ON_DEVICE) { + // If it will never be available due to API inavailability, throw. + if (availability === Availability.UNAVAILABLE) { + throw new AIError( + AIErrorCode.API_NOT_ENABLED, + 'Local LanguageModel API not available in this environment.' + ); + } else if ( + availability === Availability.DOWNLOADABLE || + availability === Availability.DOWNLOADING + ) { + // TODO(chholland): Better user experience during download - progress? + logger.debug(`Waiting for download of LanguageModel to complete.`); + await this.downloadPromise; + return true; + } return true; } // Applies prefer_on_device logic. - if (availability !== Availability.available) { + if (availability !== Availability.AVAILABLE) { logger.debug( `On-device inference unavailable because availability is "${availability}".` ); @@ -202,7 +217,7 @@ export class ChromeAdapterImpl implements ChromeAdapter { this.onDeviceParams.createOptions ); - if (availability === Availability.downloadable) { + if (availability === Availability.DOWNLOADABLE) { this.download(); } diff --git a/packages/ai/src/methods/count-tokens.test.ts b/packages/ai/src/methods/count-tokens.test.ts index 2da99b9e938..56985b4d54e 100644 --- a/packages/ai/src/methods/count-tokens.test.ts +++ b/packages/ai/src/methods/count-tokens.test.ts @@ -22,7 +22,7 @@ import chaiAsPromised from 'chai-as-promised'; import { getMockResponse } from '../../test-utils/mock-response'; import * as request from '../requests/request'; import { countTokens } from './count-tokens'; -import { CountTokensRequest } from '../types'; +import { CountTokensRequest, InferenceMode } from '../types'; import { ApiSettings } from '../types/internal'; import { Task } from '../requests/request'; import { mapCountTokensRequest } from '../googleai-mappers'; @@ -52,6 +52,12 @@ const fakeRequestParams: CountTokensRequest = { contents: [{ parts: [{ text: 'hello' }], role: 'user' }] }; +const fakeChromeAdapter = new ChromeAdapterImpl( + // @ts-expect-error + undefined, + InferenceMode.PREFER_ON_DEVICE +); + describe('countTokens()', () => { afterEach(() => { restore(); @@ -68,7 +74,7 @@ describe('countTokens()', () => { fakeApiSettings, 'model', fakeRequestParams, - new ChromeAdapterImpl() + fakeChromeAdapter ); expect(result.totalTokens).to.equal(6); expect(result.totalBillableCharacters).to.equal(16); @@ -95,7 +101,7 @@ describe('countTokens()', () => { fakeApiSettings, 'model', fakeRequestParams, - new ChromeAdapterImpl() + fakeChromeAdapter ); expect(result.totalTokens).to.equal(1837); expect(result.totalBillableCharacters).to.equal(117); @@ -124,7 +130,7 @@ describe('countTokens()', () => { fakeApiSettings, 'model', fakeRequestParams, - new ChromeAdapterImpl() + fakeChromeAdapter ); expect(result.totalTokens).to.equal(258); expect(result).to.not.have.property('totalBillableCharacters'); @@ -154,7 +160,7 @@ describe('countTokens()', () => { fakeApiSettings, 'model', fakeRequestParams, - new ChromeAdapterImpl() + fakeChromeAdapter ) ).to.be.rejectedWith(/404.*not found/); expect(mockFetch).to.be.called; @@ -177,7 +183,7 @@ describe('countTokens()', () => { fakeGoogleAIApiSettings, 'model', fakeRequestParams, - new ChromeAdapterImpl() + fakeChromeAdapter ); expect(makeRequestStub).to.be.calledWith( @@ -191,7 +197,7 @@ describe('countTokens()', () => { }); }); it('on-device', async () => { - const chromeAdapter = new ChromeAdapterImpl(); + const chromeAdapter = fakeChromeAdapter; const isAvailableStub = stub(chromeAdapter, 'isAvailable').resolves(true); const mockResponse = getMockResponse( 'vertexAI', diff --git a/packages/ai/src/methods/generate-content.test.ts b/packages/ai/src/methods/generate-content.test.ts index bbc9d47fc70..19c0761949a 100644 --- a/packages/ai/src/methods/generate-content.test.ts +++ b/packages/ai/src/methods/generate-content.test.ts @@ -27,7 +27,8 @@ import { GenerateContentRequest, HarmBlockMethod, HarmBlockThreshold, - HarmCategory + HarmCategory, + InferenceMode } from '../types'; import { ApiSettings } from '../types/internal'; import { Task } from '../requests/request'; @@ -39,6 +40,12 @@ import { ChromeAdapterImpl } from './chrome-adapter'; use(sinonChai); use(chaiAsPromised); +const fakeChromeAdapter = new ChromeAdapterImpl( + // @ts-expect-error + undefined, + InferenceMode.PREFER_ON_DEVICE +); + const fakeApiSettings: ApiSettings = { apiKey: 'key', project: 'my-project', @@ -425,7 +432,7 @@ describe('generateContent()', () => { }); // TODO: define a similar test for generateContentStream it('on-device', async () => { - const chromeAdapter = new ChromeAdapterImpl(); + const chromeAdapter = fakeChromeAdapter; const isAvailableStub = stub(chromeAdapter, 'isAvailable').resolves(true); const mockResponse = getMockResponse( 'vertexAI', diff --git a/packages/ai/src/models/generative-model.test.ts b/packages/ai/src/models/generative-model.test.ts index 6ea7470ef5f..68f1565b26a 100644 --- a/packages/ai/src/models/generative-model.test.ts +++ b/packages/ai/src/models/generative-model.test.ts @@ -16,7 +16,7 @@ */ import { use, expect } from 'chai'; import { GenerativeModel } from './generative-model'; -import { FunctionCallingMode, AI } from '../public-types'; +import { FunctionCallingMode, AI, InferenceMode } from '../public-types'; import * as request from '../requests/request'; import { match, restore, stub } from 'sinon'; import { getMockResponse } from '../../test-utils/mock-response'; @@ -40,6 +40,12 @@ const fakeAI: AI = { location: 'us-central1' }; +const fakeChromeAdapter = new ChromeAdapterImpl( + // @ts-expect-error + undefined, + InferenceMode.PREFER_ON_DEVICE +); + describe('GenerativeModel', () => { it('passes params through to generateContent', async () => { const genModel = new GenerativeModel( @@ -62,7 +68,7 @@ describe('GenerativeModel', () => { systemInstruction: { role: 'system', parts: [{ text: 'be friendly' }] } }, {}, - new ChromeAdapterImpl() + fakeChromeAdapter ); expect(genModel.tools?.length).to.equal(1); expect(genModel.toolConfig?.functionCallingConfig?.mode).to.equal( @@ -101,7 +107,7 @@ describe('GenerativeModel', () => { systemInstruction: 'be friendly' }, {}, - new ChromeAdapterImpl() + fakeChromeAdapter ); expect(genModel.systemInstruction?.parts[0].text).to.equal('be friendly'); const mockResponse = getMockResponse( @@ -145,7 +151,7 @@ describe('GenerativeModel', () => { systemInstruction: { role: 'system', parts: [{ text: 'be friendly' }] } }, {}, - new ChromeAdapterImpl() + fakeChromeAdapter ); expect(genModel.tools?.length).to.equal(1); expect(genModel.toolConfig?.functionCallingConfig?.mode).to.equal( @@ -197,7 +203,7 @@ describe('GenerativeModel', () => { } }, {}, - new ChromeAdapterImpl() + fakeChromeAdapter ); const chatSession = genModel.startChat(); expect(chatSession.params?.generationConfig).to.deep.equal({ @@ -215,7 +221,7 @@ describe('GenerativeModel', () => { } }, {}, - new ChromeAdapterImpl() + fakeChromeAdapter ); const chatSession = genModel.startChat({ generationConfig: { @@ -243,7 +249,7 @@ describe('GenerativeModel', () => { } }, {}, - new ChromeAdapterImpl() + fakeChromeAdapter ); expect(genModel.tools?.length).to.equal(1); expect(genModel.toolConfig?.functionCallingConfig?.mode).to.equal( @@ -283,7 +289,7 @@ describe('GenerativeModel', () => { systemInstruction: 'be friendly' }, {}, - new ChromeAdapterImpl() + fakeChromeAdapter ); expect(genModel.systemInstruction?.parts[0].text).to.equal('be friendly'); const mockResponse = getMockResponse( @@ -323,7 +329,7 @@ describe('GenerativeModel', () => { } }, {}, - new ChromeAdapterImpl() + fakeChromeAdapter ); expect(genModel.tools?.length).to.equal(1); expect(genModel.toolConfig?.functionCallingConfig?.mode).to.equal( @@ -378,7 +384,7 @@ describe('GenerativeModel', () => { fakeAI, { model: 'my-model' }, {}, - new ChromeAdapterImpl() + fakeChromeAdapter ); const mockResponse = getMockResponse( 'vertexAI', diff --git a/packages/ai/src/types/chrome-adapter.ts b/packages/ai/src/types/chrome-adapter.ts index 5657302a60f..8cddf9960fb 100644 --- a/packages/ai/src/types/chrome-adapter.ts +++ b/packages/ai/src/types/chrome-adapter.ts @@ -15,11 +15,12 @@ * limitations under the License. */ +import { InferenceMode } from './enums'; import { CountTokensRequest, GenerateContentRequest } from './requests'; /** * (EXPERIMENTAL) - * + * * Defines an inference "backend" that uses Chrome's on-device model, * and encapsulates logic for detecting when on-device is possible. * @@ -56,7 +57,7 @@ export interface ChromeAdapter { *

This is comparable to {@link GenerativeModel.generateContent} for generating content in * Cloud.

* @param request - a standard Firebase AI {@link GenerateContentRequest} - * @returns {@link Response}, so we can reuse common response formatting. + * @returns Response, so we can reuse common response formatting. */ generateContent(request: GenerateContentRequest): Promise; @@ -66,7 +67,7 @@ export interface ChromeAdapter { *

This is comparable to {@link GenerativeModel.generateContentStream} for generating content in * Cloud.

* @param request - a standard Firebase AI {@link GenerateContentRequest} - * @returns {@link Response}, so we can reuse common response formatting. + * @returns Response, so we can reuse common response formatting. */ generateContentStream(request: GenerateContentRequest): Promise; } diff --git a/packages/ai/src/types/language-model.ts b/packages/ai/src/types/language-model.ts index 0d1db8d4971..555680aa0dd 100644 --- a/packages/ai/src/types/language-model.ts +++ b/packages/ai/src/types/language-model.ts @@ -43,10 +43,10 @@ export interface LanguageModel extends EventTarget { * @internal */ export enum Availability { - 'unavailable' = 'unavailable', - 'downloadable' = 'downloadable', - 'downloading' = 'downloading', - 'available' = 'available' + 'UNAVAILABLE' = 'unavailable', + 'DOWNLOADABLE' = 'downloadable', + 'DOWNLOADING' = 'downloading', + 'AVAILABLE' = 'available' } /** From 4f469cfa8ae5358b4c80ae8c40bb07f917125009 Mon Sep 17 00:00:00 2001 From: Christina Holland Date: Thu, 31 Jul 2025 18:05:59 -0700 Subject: [PATCH 13/20] update docs --- common/api-review/ai.api.md | 2 -- docs-devsite/ai.chromeadapter.md | 14 -------------- 2 files changed, 16 deletions(-) diff --git a/common/api-review/ai.api.md b/common/api-review/ai.api.md index 695f0cd7cca..fff19dfecbb 100644 --- a/common/api-review/ai.api.md +++ b/common/api-review/ai.api.md @@ -143,8 +143,6 @@ export interface ChromeAdapter { generateContent(request: GenerateContentRequest): Promise; generateContentStream(request: GenerateContentRequest): Promise; isAvailable(request: GenerateContentRequest): Promise; - // (undocumented) - mode: InferenceMode; } // @public diff --git a/docs-devsite/ai.chromeadapter.md b/docs-devsite/ai.chromeadapter.md index afa6ce46ff6..86fe20a298f 100644 --- a/docs-devsite/ai.chromeadapter.md +++ b/docs-devsite/ai.chromeadapter.md @@ -22,12 +22,6 @@ These methods should not be called directly by the user. export interface ChromeAdapter ``` -## Properties - -| Property | Type | Description | -| --- | --- | --- | -| [mode](./ai.chromeadapter.md#chromeadaptermode) | [InferenceMode](./ai.md#inferencemode) | | - ## Methods | Method | Description | @@ -37,14 +31,6 @@ export interface ChromeAdapter | [generateContentStream(request)](./ai.chromeadapter.md#chromeadaptergeneratecontentstream) | Generates content stream on device.

This is comparable to [GenerativeModel.generateContentStream()](./ai.generativemodel.md#generativemodelgeneratecontentstream) for generating content in Cloud.

| | [isAvailable(request)](./ai.chromeadapter.md#chromeadapterisavailable) | Checks if a given request can be made on-device.
    Encapsulates a few concerns:
  1. the mode
  2. API existence
  3. prompt formatting
  4. model availability, including triggering download if necessary

Pros: callers needn't be concerned with details of on-device availability.

Cons: this method spans a few concerns and splits request validation from usage. If instance variables weren't already part of the API, we could consider a better separation of concerns.

| -## ChromeAdapter.mode - -Signature: - -```typescript -mode: InferenceMode; -``` - ## ChromeAdapter.countTokens() Stub - not yet available for on-device. From d32c661ce162d2fbb2ce112b7f3aab5231604a6c Mon Sep 17 00:00:00 2001 From: Christina Holland Date: Thu, 31 Jul 2025 18:18:14 -0700 Subject: [PATCH 14/20] add some tests for on-device availability cases --- .../ai/src/methods/chrome-adapter.test.ts | 50 +++++++++++++++++-- packages/ai/src/types/chrome-adapter.ts | 1 - 2 files changed, 47 insertions(+), 4 deletions(-) diff --git a/packages/ai/src/methods/chrome-adapter.test.ts b/packages/ai/src/methods/chrome-adapter.test.ts index 0392e5bba3f..83610f3dcd6 100644 --- a/packages/ai/src/methods/chrome-adapter.test.ts +++ b/packages/ai/src/methods/chrome-adapter.test.ts @@ -112,7 +112,7 @@ describe('ChromeAdapter', () => { it('returns false if mode is only cloud', async () => { const adapter = new ChromeAdapterImpl( {} as LanguageModel, - 'only_in_cloud' + InferenceMode.ONLY_IN_CLOUD ); expect( await adapter.isAvailable({ @@ -120,6 +120,47 @@ describe('ChromeAdapter', () => { }) ).to.be.false; }); + it('returns true if mode is only on device and is available', async () => { + const adapter = new ChromeAdapterImpl( + { + availability: async () => Availability.AVAILABLE + } as LanguageModel, + InferenceMode.ONLY_ON_DEVICE + ); + expect( + await adapter.isAvailable({ + contents: [] + }) + ).to.be.true; + }); + it('throws if mode is only on device and is unavailable', async () => { + const adapter = new ChromeAdapterImpl( + { + availability: async () => Availability.UNAVAILABLE + } as LanguageModel, + InferenceMode.ONLY_ON_DEVICE + ); + await expect( + adapter.isAvailable({ + contents: [] + }) + ).to.be.rejected; + }); + it('returns true after waiting for download if mode is only on device', async () => { + const adapter = new ChromeAdapterImpl( + { + availability: async () => Availability.DOWNLOADING, + create: ({}: LanguageModelCreateOptions) => + Promise.resolve({} as LanguageModel) + } as LanguageModel, + InferenceMode.ONLY_ON_DEVICE + ); + expect( + await adapter.isAvailable({ + contents: [] + }) + ).to.be.true; + }); it('returns false if LanguageModel API is undefined', async () => { const adapter = new ChromeAdapterImpl( // @ts-expect-error @@ -301,8 +342,11 @@ describe('ChromeAdapter', () => { }); describe('generateContent', () => { it('throws if Chrome API is undefined', async () => { - // @ts-expect-error - const adapter = new ChromeAdapterImpl(undefined, 'only_on_device'); + const adapter = new ChromeAdapterImpl( + // @ts-expect-error + undefined, + InferenceMode.ONLY_ON_DEVICE + ); await expect( adapter.generateContent({ contents: [] diff --git a/packages/ai/src/types/chrome-adapter.ts b/packages/ai/src/types/chrome-adapter.ts index 8cddf9960fb..ffc8186669e 100644 --- a/packages/ai/src/types/chrome-adapter.ts +++ b/packages/ai/src/types/chrome-adapter.ts @@ -15,7 +15,6 @@ * limitations under the License. */ -import { InferenceMode } from './enums'; import { CountTokensRequest, GenerateContentRequest } from './requests'; /** From 0efb0d03f219187d565f7d8ddc9c20c8ff56299f Mon Sep 17 00:00:00 2001 From: Christina Holland Date: Tue, 5 Aug 2025 15:33:40 -0700 Subject: [PATCH 15/20] Address doc comments --- common/api-review/ai.api.md | 1 - docs-devsite/ai.chromeadapter.md | 51 ++++--------------- docs-devsite/ai.hybridparams.md | 2 +- .../ai.languagemodelcreatecoreoptions.md | 2 +- docs-devsite/ai.languagemodelcreateoptions.md | 2 +- docs-devsite/ai.languagemodelexpected.md | 2 +- docs-devsite/ai.languagemodelmessage.md | 2 +- .../ai.languagemodelmessagecontent.md | 2 +- docs-devsite/ai.languagemodelpromptoptions.md | 2 +- docs-devsite/ai.md | 38 +++++++------- docs-devsite/ai.ondeviceparams.md | 2 +- packages/ai/src/methods/chrome-adapter.ts | 3 +- packages/ai/src/types/chrome-adapter.ts | 44 +++++----------- packages/ai/src/types/enums.ts | 4 +- packages/ai/src/types/language-model.ts | 26 +++++----- packages/ai/src/types/requests.ts | 4 +- 16 files changed, 69 insertions(+), 118 deletions(-) diff --git a/common/api-review/ai.api.md b/common/api-review/ai.api.md index fff19dfecbb..1074cdb8d14 100644 --- a/common/api-review/ai.api.md +++ b/common/api-review/ai.api.md @@ -139,7 +139,6 @@ export class ChatSession { // @public export interface ChromeAdapter { - countTokens(_request: CountTokensRequest): Promise; generateContent(request: GenerateContentRequest): Promise; generateContentStream(request: GenerateContentRequest): Promise; isAvailable(request: GenerateContentRequest): Promise; diff --git a/docs-devsite/ai.chromeadapter.md b/docs-devsite/ai.chromeadapter.md index 86fe20a298f..8d019f686da 100644 --- a/docs-devsite/ai.chromeadapter.md +++ b/docs-devsite/ai.chromeadapter.md @@ -10,9 +10,7 @@ https://github.com/firebase/firebase-js-sdk {% endcomment %} # ChromeAdapter interface -(EXPERIMENTAL) - -Defines an inference "backend" that uses Chrome's on-device model, and encapsulates logic for detecting when on-device is possible. +(EXPERIMENTAL) Defines an inference "backend" that uses Chrome's on-device model, and encapsulates logic for detecting when on-device inference is possible. These methods should not be called directly by the user. @@ -26,36 +24,15 @@ export interface ChromeAdapter | Method | Description | | --- | --- | -| [countTokens(\_request)](./ai.chromeadapter.md#chromeadaptercounttokens) | Stub - not yet available for on-device. | -| [generateContent(request)](./ai.chromeadapter.md#chromeadaptergeneratecontent) | Generates content on device.

This is comparable to [GenerativeModel.generateContent()](./ai.generativemodel.md#generativemodelgeneratecontent) for generating content in Cloud.

| -| [generateContentStream(request)](./ai.chromeadapter.md#chromeadaptergeneratecontentstream) | Generates content stream on device.

This is comparable to [GenerativeModel.generateContentStream()](./ai.generativemodel.md#generativemodelgeneratecontentstream) for generating content in Cloud.

| -| [isAvailable(request)](./ai.chromeadapter.md#chromeadapterisavailable) | Checks if a given request can be made on-device.
    Encapsulates a few concerns:
  1. the mode
  2. API existence
  3. prompt formatting
  4. model availability, including triggering download if necessary

Pros: callers needn't be concerned with details of on-device availability.

Cons: this method spans a few concerns and splits request validation from usage. If instance variables weren't already part of the API, we could consider a better separation of concerns.

| - -## ChromeAdapter.countTokens() - -Stub - not yet available for on-device. - -Signature: - -```typescript -countTokens(_request: CountTokensRequest): Promise; -``` - -#### Parameters - -| Parameter | Type | Description | -| --- | --- | --- | -| \_request | [CountTokensRequest](./ai.counttokensrequest.md#counttokensrequest_interface) | | - -Returns: - -Promise<Response> +| [generateContent(request)](./ai.chromeadapter.md#chromeadaptergeneratecontent) | Generates content using on-device inference.

This is comparable to [GenerativeModel.generateContent()](./ai.generativemodel.md#generativemodelgeneratecontent) for generating content using in-cloud inference.

| +| [generateContentStream(request)](./ai.chromeadapter.md#chromeadaptergeneratecontentstream) | Generates a content stream using on-device inference.

This is comparable to [GenerativeModel.generateContentStream()](./ai.generativemodel.md#generativemodelgeneratecontentstream) for generating content using in-cloud inference.

| +| [isAvailable(request)](./ai.chromeadapter.md#chromeadapterisavailable) | Checks if the on-device model is capable of handling a given request. | ## ChromeAdapter.generateContent() -Generates content on device. +Generates content using on-device inference. -

This is comparable to [GenerativeModel.generateContent()](./ai.generativemodel.md#generativemodelgeneratecontent) for generating content in Cloud.

+

This is comparable to [GenerativeModel.generateContent()](./ai.generativemodel.md#generativemodelgeneratecontent) for generating content using in-cloud inference.

Signature: @@ -73,13 +50,11 @@ generateContent(request: GenerateContentRequest): Promise; Promise<Response> -Response, so we can reuse common response formatting. - ## ChromeAdapter.generateContentStream() -Generates content stream on device. +Generates a content stream using on-device inference. -

This is comparable to [GenerativeModel.generateContentStream()](./ai.generativemodel.md#generativemodelgeneratecontentstream) for generating content in Cloud.

+

This is comparable to [GenerativeModel.generateContentStream()](./ai.generativemodel.md#generativemodelgeneratecontentstream) for generating content using in-cloud inference.

Signature: @@ -97,15 +72,9 @@ generateContentStream(request: GenerateContentRequest): Promise; Promise<Response> -Response, so we can reuse common response formatting. - ## ChromeAdapter.isAvailable() -Checks if a given request can be made on-device. - -
    Encapsulates a few concerns:
  1. the mode
  2. API existence
  3. prompt formatting
  4. model availability, including triggering download if necessary
- -

Pros: callers needn't be concerned with details of on-device availability.

Cons: this method spans a few concerns and splits request validation from usage. If instance variables weren't already part of the API, we could consider a better separation of concerns.

+Checks if the on-device model is capable of handling a given request. Signature: @@ -117,7 +86,7 @@ isAvailable(request: GenerateContentRequest): Promise; | Parameter | Type | Description | | --- | --- | --- | -| request | [GenerateContentRequest](./ai.generatecontentrequest.md#generatecontentrequest_interface) | | +| request | [GenerateContentRequest](./ai.generatecontentrequest.md#generatecontentrequest_interface) | A potential request to be passed to the model. | Returns: diff --git a/docs-devsite/ai.hybridparams.md b/docs-devsite/ai.hybridparams.md index 383e9baafa5..baf568217d3 100644 --- a/docs-devsite/ai.hybridparams.md +++ b/docs-devsite/ai.hybridparams.md @@ -10,7 +10,7 @@ https://github.com/firebase/firebase-js-sdk {% endcomment %} # HybridParams interface -(EXPERIMENTAL) Configures hybrid inference. +(EXPERIMENTAL) Configures hybrid inference. Signature: diff --git a/docs-devsite/ai.languagemodelcreatecoreoptions.md b/docs-devsite/ai.languagemodelcreatecoreoptions.md index e6432543958..3b221933034 100644 --- a/docs-devsite/ai.languagemodelcreatecoreoptions.md +++ b/docs-devsite/ai.languagemodelcreatecoreoptions.md @@ -10,7 +10,7 @@ https://github.com/firebase/firebase-js-sdk {% endcomment %} # LanguageModelCreateCoreOptions interface -(EXPERIMENTAL) Used to configure the creation of an on-device language model session. +(EXPERIMENTAL) Configures the creation of an on-device language model session. Signature: diff --git a/docs-devsite/ai.languagemodelcreateoptions.md b/docs-devsite/ai.languagemodelcreateoptions.md index 22e88cbfdcd..5d2ec9c69ad 100644 --- a/docs-devsite/ai.languagemodelcreateoptions.md +++ b/docs-devsite/ai.languagemodelcreateoptions.md @@ -10,7 +10,7 @@ https://github.com/firebase/firebase-js-sdk {% endcomment %} # LanguageModelCreateOptions interface -(EXPERIMENTAL) Used to configure the creation of an on-device language model session. +(EXPERIMENTAL) Configures the creation of an on-device language model session. Signature: diff --git a/docs-devsite/ai.languagemodelexpected.md b/docs-devsite/ai.languagemodelexpected.md index ffec338c773..d27e718e1eb 100644 --- a/docs-devsite/ai.languagemodelexpected.md +++ b/docs-devsite/ai.languagemodelexpected.md @@ -10,7 +10,7 @@ https://github.com/firebase/firebase-js-sdk {% endcomment %} # LanguageModelExpected interface -(EXPERIMENTAL) Options for an on-device language model expected inputs. +(EXPERIMENTAL) Options for the expected inputs for an on-device language model. Signature: diff --git a/docs-devsite/ai.languagemodelmessage.md b/docs-devsite/ai.languagemodelmessage.md index 5f9e62a8ab0..228a31c8521 100644 --- a/docs-devsite/ai.languagemodelmessage.md +++ b/docs-devsite/ai.languagemodelmessage.md @@ -10,7 +10,7 @@ https://github.com/firebase/firebase-js-sdk {% endcomment %} # LanguageModelMessage interface -(EXPERIMENTAL) An on-device language model message. +(EXPERIMENTAL) An on-device language model message. Signature: diff --git a/docs-devsite/ai.languagemodelmessagecontent.md b/docs-devsite/ai.languagemodelmessagecontent.md index 1600d312c89..71d2ce9919b 100644 --- a/docs-devsite/ai.languagemodelmessagecontent.md +++ b/docs-devsite/ai.languagemodelmessagecontent.md @@ -10,7 +10,7 @@ https://github.com/firebase/firebase-js-sdk {% endcomment %} # LanguageModelMessageContent interface -(EXPERIMENTAL) An on-device language model content object. +(EXPERIMENTAL) An on-device language model content object. Signature: diff --git a/docs-devsite/ai.languagemodelpromptoptions.md b/docs-devsite/ai.languagemodelpromptoptions.md index 9ff8c5d5941..35a22c3d1a6 100644 --- a/docs-devsite/ai.languagemodelpromptoptions.md +++ b/docs-devsite/ai.languagemodelpromptoptions.md @@ -10,7 +10,7 @@ https://github.com/firebase/firebase-js-sdk {% endcomment %} # LanguageModelPromptOptions interface -(EXPERIMENTAL) Options for an on-device language model prompt. +(EXPERIMENTAL) Options for an on-device language model prompt. Signature: diff --git a/docs-devsite/ai.md b/docs-devsite/ai.md index 0f5dd4628bd..29b3f73f86e 100644 --- a/docs-devsite/ai.md +++ b/docs-devsite/ai.md @@ -51,7 +51,7 @@ The Firebase AI Web SDK. | [AI](./ai.ai.md#ai_interface) | An instance of the Firebase AI SDK.Do not create this instance directly. Instead, use [getAI()](./ai.md#getai_a94a413). | | [AIOptions](./ai.aioptions.md#aioptions_interface) | Options for initializing the AI service using [getAI()](./ai.md#getai_a94a413). This allows specifying which backend to use (Vertex AI Gemini API or Gemini Developer API) and configuring its specific options (like location for Vertex AI). | | [BaseParams](./ai.baseparams.md#baseparams_interface) | Base parameters for a number of methods. | -| [ChromeAdapter](./ai.chromeadapter.md#chromeadapter_interface) | (EXPERIMENTAL)Defines an inference "backend" that uses Chrome's on-device model, and encapsulates logic for detecting when on-device is possible.These methods should not be called directly by the user. | +| [ChromeAdapter](./ai.chromeadapter.md#chromeadapter_interface) | (EXPERIMENTAL) Defines an inference "backend" that uses Chrome's on-device model, and encapsulates logic for detecting when on-device inference is possible.These methods should not be called directly by the user. | | [Citation](./ai.citation.md#citation_interface) | A single citation. | | [CitationMetadata](./ai.citationmetadata.md#citationmetadata_interface) | Citation metadata that may be found on a [GenerateContentCandidate](./ai.generatecontentcandidate.md#generatecontentcandidate_interface). | | [Content](./ai.content.md#content_interface) | Content type for both prompts and response candidates. | @@ -82,7 +82,7 @@ The Firebase AI Web SDK. | [GroundingChunk](./ai.groundingchunk.md#groundingchunk_interface) | Represents a chunk of retrieved data that supports a claim in the model's response. This is part of the grounding information provided when grounding is enabled. | | [GroundingMetadata](./ai.groundingmetadata.md#groundingmetadata_interface) | Metadata returned when grounding is enabled.Currently, only Grounding with Google Search is supported (see [GoogleSearchTool](./ai.googlesearchtool.md#googlesearchtool_interface)).Important: If using Grounding with Google Search, you are required to comply with the "Grounding with Google Search" usage requirements for your chosen API provider: [Gemini Developer API](https://ai.google.dev/gemini-api/terms#grounding-with-google-search) or Vertex AI Gemini API (see [Service Terms](https://cloud.google.com/terms/service-terms) section within the Service Specific Terms). | | [GroundingSupport](./ai.groundingsupport.md#groundingsupport_interface) | Provides information about how a specific segment of the model's response is supported by the retrieved grounding chunks. | -| [HybridParams](./ai.hybridparams.md#hybridparams_interface) | (EXPERIMENTAL) Configures hybrid inference. | +| [HybridParams](./ai.hybridparams.md#hybridparams_interface) | (EXPERIMENTAL) Configures hybrid inference. | | [ImagenGCSImage](./ai.imagengcsimage.md#imagengcsimage_interface) | (Public Preview) An image generated by Imagen, stored in a Cloud Storage for Firebase bucket.This feature is not available yet. | | [ImagenGenerationConfig](./ai.imagengenerationconfig.md#imagengenerationconfig_interface) | (Public Preview) Configuration options for generating images with Imagen.See the [documentation](http://firebase.google.com/docs/vertex-ai/generate-images-imagen) for more details. | | [ImagenGenerationResponse](./ai.imagengenerationresponse.md#imagengenerationresponse_interface) | (Public Preview) The response from a request to generate images with Imagen. | @@ -90,16 +90,16 @@ The Firebase AI Web SDK. | [ImagenModelParams](./ai.imagenmodelparams.md#imagenmodelparams_interface) | (Public Preview) Parameters for configuring an [ImagenModel](./ai.imagenmodel.md#imagenmodel_class). | | [ImagenSafetySettings](./ai.imagensafetysettings.md#imagensafetysettings_interface) | (Public Preview) Settings for controlling the aggressiveness of filtering out sensitive content.See the [documentation](http://firebase.google.com/docs/vertex-ai/generate-images) for more details. | | [InlineDataPart](./ai.inlinedatapart.md#inlinedatapart_interface) | Content part interface if the part represents an image. | -| [LanguageModelCreateCoreOptions](./ai.languagemodelcreatecoreoptions.md#languagemodelcreatecoreoptions_interface) | (EXPERIMENTAL) Used to configure the creation of an on-device language model session. | -| [LanguageModelCreateOptions](./ai.languagemodelcreateoptions.md#languagemodelcreateoptions_interface) | (EXPERIMENTAL) Used to configure the creation of an on-device language model session. | -| [LanguageModelExpected](./ai.languagemodelexpected.md#languagemodelexpected_interface) | (EXPERIMENTAL) Options for an on-device language model expected inputs. | -| [LanguageModelMessage](./ai.languagemodelmessage.md#languagemodelmessage_interface) | (EXPERIMENTAL) An on-device language model message. | -| [LanguageModelMessageContent](./ai.languagemodelmessagecontent.md#languagemodelmessagecontent_interface) | (EXPERIMENTAL) An on-device language model content object. | -| [LanguageModelPromptOptions](./ai.languagemodelpromptoptions.md#languagemodelpromptoptions_interface) | (EXPERIMENTAL) Options for an on-device language model prompt. | +| [LanguageModelCreateCoreOptions](./ai.languagemodelcreatecoreoptions.md#languagemodelcreatecoreoptions_interface) | (EXPERIMENTAL) Configures the creation of an on-device language model session. | +| [LanguageModelCreateOptions](./ai.languagemodelcreateoptions.md#languagemodelcreateoptions_interface) | (EXPERIMENTAL) Configures the creation of an on-device language model session. | +| [LanguageModelExpected](./ai.languagemodelexpected.md#languagemodelexpected_interface) | (EXPERIMENTAL) Options for the expected inputs for an on-device language model. | +| [LanguageModelMessage](./ai.languagemodelmessage.md#languagemodelmessage_interface) | (EXPERIMENTAL) An on-device language model message. | +| [LanguageModelMessageContent](./ai.languagemodelmessagecontent.md#languagemodelmessagecontent_interface) | (EXPERIMENTAL) An on-device language model content object. | +| [LanguageModelPromptOptions](./ai.languagemodelpromptoptions.md#languagemodelpromptoptions_interface) | (EXPERIMENTAL) Options for an on-device language model prompt. | | [ModalityTokenCount](./ai.modalitytokencount.md#modalitytokencount_interface) | Represents token counting info for a single modality. | | [ModelParams](./ai.modelparams.md#modelparams_interface) | Params passed to [getGenerativeModel()](./ai.md#getgenerativemodel_c63f46a). | | [ObjectSchemaRequest](./ai.objectschemarequest.md#objectschemarequest_interface) | Interface for JSON parameters in a schema of [SchemaType](./ai.md#schematype) "object" when not using the Schema.object() helper. | -| [OnDeviceParams](./ai.ondeviceparams.md#ondeviceparams_interface) | (EXPERIMENTAL) Encapsulates configuration for on-device inference. | +| [OnDeviceParams](./ai.ondeviceparams.md#ondeviceparams_interface) | (EXPERIMENTAL) Encapsulates configuration for on-device inference. | | [PromptFeedback](./ai.promptfeedback.md#promptfeedback_interface) | If the prompt was blocked, this will be populated with blockReason and the relevant safetyRatings. | | [RequestOptions](./ai.requestoptions.md#requestoptions_interface) | Params passed to [getGenerativeModel()](./ai.md#getgenerativemodel_c63f46a). | | [RetrievedContextAttribution](./ai.retrievedcontextattribution.md#retrievedcontextattribution_interface) | | @@ -137,7 +137,7 @@ The Firebase AI Web SDK. | [ImagenAspectRatio](./ai.md#imagenaspectratio) | (Public Preview) Aspect ratios for Imagen images.To specify an aspect ratio for generated images, set the aspectRatio property in your [ImagenGenerationConfig](./ai.imagengenerationconfig.md#imagengenerationconfig_interface).See the [documentation](http://firebase.google.com/docs/vertex-ai/generate-images) for more details and examples of the supported aspect ratios. | | [ImagenPersonFilterLevel](./ai.md#imagenpersonfilterlevel) | (Public Preview) A filter level controlling whether generation of images containing people or faces is allowed.See the personGeneration documentation for more details. | | [ImagenSafetyFilterLevel](./ai.md#imagensafetyfilterlevel) | (Public Preview) A filter level controlling how aggressively to filter sensitive content.Text prompts provided as inputs and images (generated or uploaded) through Imagen on Vertex AI are assessed against a list of safety filters, which include 'harmful categories' (for example, violence, sexual, derogatory, and toxic). This filter level controls how aggressively to filter out potentially harmful content from responses. See the [documentation](http://firebase.google.com/docs/vertex-ai/generate-images) and the [Responsible AI and usage guidelines](https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen#safety-filters) for more details. | -| [InferenceMode](./ai.md#inferencemode) | (EXPERIMENTAL) Determines whether inference happens on-device or in-cloud. | +| [InferenceMode](./ai.md#inferencemode) | (EXPERIMENTAL) Determines whether inference happens on-device or in-cloud. | | [Modality](./ai.md#modality) | Content part modality. | | [POSSIBLE\_ROLES](./ai.md#possible_roles) | Possible roles. | | [ResponseModality](./ai.md#responsemodality) | (Public Preview) Generation modalities to be returned in generation responses. | @@ -160,10 +160,10 @@ The Firebase AI Web SDK. | [ImagenAspectRatio](./ai.md#imagenaspectratio) | (Public Preview) Aspect ratios for Imagen images.To specify an aspect ratio for generated images, set the aspectRatio property in your [ImagenGenerationConfig](./ai.imagengenerationconfig.md#imagengenerationconfig_interface).See the [documentation](http://firebase.google.com/docs/vertex-ai/generate-images) for more details and examples of the supported aspect ratios. | | [ImagenPersonFilterLevel](./ai.md#imagenpersonfilterlevel) | (Public Preview) A filter level controlling whether generation of images containing people or faces is allowed.See the personGeneration documentation for more details. | | [ImagenSafetyFilterLevel](./ai.md#imagensafetyfilterlevel) | (Public Preview) A filter level controlling how aggressively to filter sensitive content.Text prompts provided as inputs and images (generated or uploaded) through Imagen on Vertex AI are assessed against a list of safety filters, which include 'harmful categories' (for example, violence, sexual, derogatory, and toxic). This filter level controls how aggressively to filter out potentially harmful content from responses. See the [documentation](http://firebase.google.com/docs/vertex-ai/generate-images) and the [Responsible AI and usage guidelines](https://cloud.google.com/vertex-ai/generative-ai/docs/image/responsible-ai-imagen#safety-filters) for more details. | -| [InferenceMode](./ai.md#inferencemode) | (EXPERIMENTAL) Determines whether inference happens on-device or in-cloud. | -| [LanguageModelMessageContentValue](./ai.md#languagemodelmessagecontentvalue) | (EXPERIMENTAL) Content formats that can be provided as on-device message content. | -| [LanguageModelMessageRole](./ai.md#languagemodelmessagerole) | (EXPERIMENTAL) Allowable roles for on-device language model usage. | -| [LanguageModelMessageType](./ai.md#languagemodelmessagetype) | (EXPERIMENTAL) Allowable types for on-device language model messages. | +| [InferenceMode](./ai.md#inferencemode) | (EXPERIMENTAL) Determines whether inference happens on-device or in-cloud. | +| [LanguageModelMessageContentValue](./ai.md#languagemodelmessagecontentvalue) | (EXPERIMENTAL) Content formats that can be provided as on-device message content. | +| [LanguageModelMessageRole](./ai.md#languagemodelmessagerole) | (EXPERIMENTAL) Allowable roles for on-device language model usage. | +| [LanguageModelMessageType](./ai.md#languagemodelmessagetype) | (EXPERIMENTAL) Allowable types for on-device language model messages. | | [Modality](./ai.md#modality) | Content part modality. | | [Part](./ai.md#part) | Content part - includes text, image/video, or function call/response part types. | | [ResponseModality](./ai.md#responsemodality) | (Public Preview) Generation modalities to be returned in generation responses. | @@ -504,7 +504,7 @@ ImagenSafetyFilterLevel: { ## InferenceMode -(EXPERIMENTAL) Determines whether inference happens on-device or in-cloud. +(EXPERIMENTAL) Determines whether inference happens on-device or in-cloud. Signature: @@ -724,7 +724,7 @@ export type ImagenSafetyFilterLevel = (typeof ImagenSafetyFilterLevel)[keyof typ ## InferenceMode -(EXPERIMENTAL) Determines whether inference happens on-device or in-cloud. +(EXPERIMENTAL) Determines whether inference happens on-device or in-cloud. Signature: @@ -734,7 +734,7 @@ export type InferenceMode = (typeof InferenceMode)[keyof typeof InferenceMode]; ## LanguageModelMessageContentValue -(EXPERIMENTAL) Content formats that can be provided as on-device message content. +(EXPERIMENTAL) Content formats that can be provided as on-device message content. Signature: @@ -744,7 +744,7 @@ export type LanguageModelMessageContentValue = ImageBitmapSource | AudioBuffer | ## LanguageModelMessageRole -(EXPERIMENTAL) Allowable roles for on-device language model usage. +(EXPERIMENTAL) Allowable roles for on-device language model usage. Signature: @@ -754,7 +754,7 @@ export type LanguageModelMessageRole = 'system' | 'user' | 'assistant'; ## LanguageModelMessageType -(EXPERIMENTAL) Allowable types for on-device language model messages. +(EXPERIMENTAL) Allowable types for on-device language model messages. Signature: diff --git a/docs-devsite/ai.ondeviceparams.md b/docs-devsite/ai.ondeviceparams.md index 77a4b8aab85..bce68ff8174 100644 --- a/docs-devsite/ai.ondeviceparams.md +++ b/docs-devsite/ai.ondeviceparams.md @@ -10,7 +10,7 @@ https://github.com/firebase/firebase-js-sdk {% endcomment %} # OnDeviceParams interface -(EXPERIMENTAL) Encapsulates configuration for on-device inference. +(EXPERIMENTAL) Encapsulates configuration for on-device inference. Signature: diff --git a/packages/ai/src/methods/chrome-adapter.ts b/packages/ai/src/methods/chrome-adapter.ts index a3c13e514d7..4dea4170c0d 100644 --- a/packages/ai/src/methods/chrome-adapter.ts +++ b/packages/ai/src/methods/chrome-adapter.ts @@ -38,7 +38,8 @@ import { /** * Defines an inference "backend" that uses Chrome's on-device model, - * and encapsulates logic for detecting when on-device is possible. + * and encapsulates logic for detecting when on-device inference is + * possible. */ export class ChromeAdapterImpl implements ChromeAdapter { // Visible for testing diff --git a/packages/ai/src/types/chrome-adapter.ts b/packages/ai/src/types/chrome-adapter.ts index ffc8186669e..74df8b387c4 100644 --- a/packages/ai/src/types/chrome-adapter.ts +++ b/packages/ai/src/types/chrome-adapter.ts @@ -15,13 +15,12 @@ * limitations under the License. */ -import { CountTokensRequest, GenerateContentRequest } from './requests'; +import { GenerateContentRequest } from './requests'; /** - * (EXPERIMENTAL) - * - * Defines an inference "backend" that uses Chrome's on-device model, - * and encapsulates logic for detecting when on-device is possible. + * (EXPERIMENTAL) Defines an inference "backend" that uses Chrome's on-device model, + * and encapsulates logic for detecting when on-device inference is + * possible. * * These methods should not be called directly by the user. * @@ -29,44 +28,27 @@ import { CountTokensRequest, GenerateContentRequest } from './requests'; */ export interface ChromeAdapter { /** - * Checks if a given request can be made on-device. - * - *
    Encapsulates a few concerns: - *
  1. the mode
    2. - *
  2. API existence
    3. - *
  3. prompt formatting
    4. - *
  4. model availability, including triggering download if necessary
    5. - *
- * - *

Pros: callers needn't be concerned with details of on-device availability.

- *

Cons: this method spans a few concerns and splits request validation from usage. - * If instance variables weren't already part of the API, we could consider a better - * separation of concerns.

+ * Checks if the on-device model is capable of handling a given + * request. + * @param request - A potential request to be passed to the model. */ isAvailable(request: GenerateContentRequest): Promise; /** - * Stub - not yet available for on-device. - */ - countTokens(_request: CountTokensRequest): Promise; - - /** - * Generates content on device. + * Generates content using on-device inference. * - *

This is comparable to {@link GenerativeModel.generateContent} for generating content in - * Cloud.

+ *

This is comparable to {@link GenerativeModel.generateContent} for generating + * content using in-cloud inference.

* @param request - a standard Firebase AI {@link GenerateContentRequest} - * @returns Response, so we can reuse common response formatting. */ generateContent(request: GenerateContentRequest): Promise; /** - * Generates content stream on device. + * Generates a content stream using on-device inference. * - *

This is comparable to {@link GenerativeModel.generateContentStream} for generating content in - * Cloud.

+ *

This is comparable to {@link GenerativeModel.generateContentStream} for generating + * content using in-cloud inference.

* @param request - a standard Firebase AI {@link GenerateContentRequest} - * @returns Response, so we can reuse common response formatting. */ generateContentStream(request: GenerateContentRequest): Promise; } diff --git a/packages/ai/src/types/enums.ts b/packages/ai/src/types/enums.ts index d02b3be4ebe..956be64ba75 100644 --- a/packages/ai/src/types/enums.ts +++ b/packages/ai/src/types/enums.ts @@ -340,7 +340,7 @@ export type ResponseModality = (typeof ResponseModality)[keyof typeof ResponseModality]; /** - * (EXPERIMENTAL) + * (EXPERIMENTAL) * Determines whether inference happens on-device or in-cloud. * @public */ @@ -351,7 +351,7 @@ export const InferenceMode = { } as const; /** - * (EXPERIMENTAL) + * (EXPERIMENTAL) * Determines whether inference happens on-device or in-cloud. * @public */ diff --git a/packages/ai/src/types/language-model.ts b/packages/ai/src/types/language-model.ts index 555680aa0dd..4157e6d05e6 100644 --- a/packages/ai/src/types/language-model.ts +++ b/packages/ai/src/types/language-model.ts @@ -50,8 +50,8 @@ export enum Availability { } /** - * (EXPERIMENTAL) - * Used to configure the creation of an on-device language model session. + * (EXPERIMENTAL) + * Configures the creation of an on-device language model session. * @public */ export interface LanguageModelCreateCoreOptions { @@ -61,8 +61,8 @@ export interface LanguageModelCreateCoreOptions { } /** - * (EXPERIMENTAL) - * Used to configure the creation of an on-device language model session. + * (EXPERIMENTAL) + * Configures the creation of an on-device language model session. * @public */ export interface LanguageModelCreateOptions @@ -72,7 +72,7 @@ export interface LanguageModelCreateOptions } /** - * (EXPERIMENTAL) + * (EXPERIMENTAL) * Options for an on-device language model prompt. * @public */ @@ -82,8 +82,8 @@ export interface LanguageModelPromptOptions { } /** - * (EXPERIMENTAL) - * Options for an on-device language model expected inputs. + * (EXPERIMENTAL) + * Options for the expected inputs for an on-device language model. * @public */ export interface LanguageModelExpected { type: LanguageModelMessageType; @@ -91,14 +91,14 @@ export interface LanguageModelPromptOptions { } /** - * (EXPERIMENTAL) + * (EXPERIMENTAL) * An on-device language model prompt. * @public */ export type LanguageModelPrompt = LanguageModelMessage[]; /** - * (EXPERIMENTAL) + * (EXPERIMENTAL) * An on-device language model message. * @public */ @@ -108,7 +108,7 @@ export interface LanguageModelMessage { } /** - * (EXPERIMENTAL) + * (EXPERIMENTAL) * An on-device language model content object. * @public */ @@ -118,21 +118,21 @@ export interface LanguageModelMessageContent { } /** - * (EXPERIMENTAL) + * (EXPERIMENTAL) * Allowable roles for on-device language model usage. * @public */ export type LanguageModelMessageRole = 'system' | 'user' | 'assistant'; /** - * (EXPERIMENTAL) + * (EXPERIMENTAL) * Allowable types for on-device language model messages. * @public */ export type LanguageModelMessageType = 'text' | 'image' | 'audio'; /** - * (EXPERIMENTAL) + * (EXPERIMENTAL) * Content formats that can be provided as on-device message content. * @public */ diff --git a/packages/ai/src/types/requests.ts b/packages/ai/src/types/requests.ts index 0673db17497..b1bde0bc290 100644 --- a/packages/ai/src/types/requests.ts +++ b/packages/ai/src/types/requests.ts @@ -277,7 +277,7 @@ export interface FunctionCallingConfig { } /** - * (EXPERIMENTAL) + * (EXPERIMENTAL) * Encapsulates configuration for on-device inference. * * @public @@ -288,7 +288,7 @@ export interface OnDeviceParams { } /** - * (EXPERIMENTAL) + * (EXPERIMENTAL) * Configures hybrid inference. * @public */ From a27968c729ca095ac902e653700ba704b2f30d15 Mon Sep 17 00:00:00 2001 From: Christina Holland Date: Tue, 5 Aug 2025 15:47:21 -0700 Subject: [PATCH 16/20] Put countTokens back in the interface but mark it internal --- common/api-review/ai.api.md | 2 ++ packages/ai/src/types/chrome-adapter.ts | 7 ++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/common/api-review/ai.api.md b/common/api-review/ai.api.md index 1074cdb8d14..e0eac35996a 100644 --- a/common/api-review/ai.api.md +++ b/common/api-review/ai.api.md @@ -139,6 +139,8 @@ export class ChatSession { // @public export interface ChromeAdapter { + // @internal (undocumented) + countTokens(request: CountTokensRequest): Promise; generateContent(request: GenerateContentRequest): Promise; generateContentStream(request: GenerateContentRequest): Promise; isAvailable(request: GenerateContentRequest): Promise; diff --git a/packages/ai/src/types/chrome-adapter.ts b/packages/ai/src/types/chrome-adapter.ts index 74df8b387c4..d04b87c2026 100644 --- a/packages/ai/src/types/chrome-adapter.ts +++ b/packages/ai/src/types/chrome-adapter.ts @@ -15,7 +15,7 @@ * limitations under the License. */ -import { GenerateContentRequest } from './requests'; +import { CountTokensRequest, GenerateContentRequest } from './requests'; /** * (EXPERIMENTAL) Defines an inference "backend" that uses Chrome's on-device model, @@ -51,4 +51,9 @@ export interface ChromeAdapter { * @param request - a standard Firebase AI {@link GenerateContentRequest} */ generateContentStream(request: GenerateContentRequest): Promise; + + /** + * @internal + */ + countTokens(request: CountTokensRequest): Promise; } From c2d7baed1bc9954c0a9e30fddba25afc5d7ba431 Mon Sep 17 00:00:00 2001 From: Christina Holland Date: Wed, 6 Aug 2025 10:52:58 -0700 Subject: [PATCH 17/20] Fix stream comment --- docs-devsite/ai.chromeadapter.md | 4 ++-- packages/ai/src/types/chrome-adapter.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs-devsite/ai.chromeadapter.md b/docs-devsite/ai.chromeadapter.md index 8d019f686da..f497312c609 100644 --- a/docs-devsite/ai.chromeadapter.md +++ b/docs-devsite/ai.chromeadapter.md @@ -25,7 +25,7 @@ export interface ChromeAdapter | Method | Description | | --- | --- | | [generateContent(request)](./ai.chromeadapter.md#chromeadaptergeneratecontent) | Generates content using on-device inference.

This is comparable to [GenerativeModel.generateContent()](./ai.generativemodel.md#generativemodelgeneratecontent) for generating content using in-cloud inference.

| -| [generateContentStream(request)](./ai.chromeadapter.md#chromeadaptergeneratecontentstream) | Generates a content stream using on-device inference.

This is comparable to [GenerativeModel.generateContentStream()](./ai.generativemodel.md#generativemodelgeneratecontentstream) for generating content using in-cloud inference.

| +| [generateContentStream(request)](./ai.chromeadapter.md#chromeadaptergeneratecontentstream) | Generates a content stream using on-device inference.

This is comparable to [GenerativeModel.generateContentStream()](./ai.generativemodel.md#generativemodelgeneratecontentstream) for generating a content stream using in-cloud inference.

| | [isAvailable(request)](./ai.chromeadapter.md#chromeadapterisavailable) | Checks if the on-device model is capable of handling a given request. | ## ChromeAdapter.generateContent() @@ -54,7 +54,7 @@ Promise<Response> Generates a content stream using on-device inference. -

This is comparable to [GenerativeModel.generateContentStream()](./ai.generativemodel.md#generativemodelgeneratecontentstream) for generating content using in-cloud inference.

+

This is comparable to [GenerativeModel.generateContentStream()](./ai.generativemodel.md#generativemodelgeneratecontentstream) for generating a content stream using in-cloud inference.

Signature: diff --git a/packages/ai/src/types/chrome-adapter.ts b/packages/ai/src/types/chrome-adapter.ts index d04b87c2026..77c52eb9391 100644 --- a/packages/ai/src/types/chrome-adapter.ts +++ b/packages/ai/src/types/chrome-adapter.ts @@ -47,7 +47,7 @@ export interface ChromeAdapter { * Generates a content stream using on-device inference. * *

This is comparable to {@link GenerativeModel.generateContentStream} for generating - * content using in-cloud inference.

+ * a content stream using in-cloud inference.

* @param request - a standard Firebase AI {@link GenerateContentRequest} */ generateContentStream(request: GenerateContentRequest): Promise; From 254b257d77a75e9dfdc65adecc631f67fe7d39b8 Mon Sep 17 00:00:00 2001 From: Christina Holland Date: Wed, 6 Aug 2025 13:48:42 -0700 Subject: [PATCH 18/20] Add changeset --- .changeset/purple-chairs-wait.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/purple-chairs-wait.md diff --git a/.changeset/purple-chairs-wait.md b/.changeset/purple-chairs-wait.md new file mode 100644 index 00000000000..be9e1452876 --- /dev/null +++ b/.changeset/purple-chairs-wait.md @@ -0,0 +1,5 @@ +--- +'@firebase/ai': minor +--- + +Add hybrid inference to the Firebase AI SDK. From 2d81e0082ce002df799344183b55af5cfbf11726 Mon Sep 17 00:00:00 2001 From: Christina Holland Date: Wed, 6 Aug 2025 13:48:55 -0700 Subject: [PATCH 19/20] Add changeset --- .changeset/purple-chairs-wait.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/purple-chairs-wait.md b/.changeset/purple-chairs-wait.md index be9e1452876..26d164c6d0e 100644 --- a/.changeset/purple-chairs-wait.md +++ b/.changeset/purple-chairs-wait.md @@ -2,4 +2,4 @@ '@firebase/ai': minor --- -Add hybrid inference to the Firebase AI SDK. +Add hybrid inference options to the Firebase AI SDK. From 67ad2830fe9015300419740d1f352f1822e50c1e Mon Sep 17 00:00:00 2001 From: Christina Holland Date: Wed, 6 Aug 2025 13:57:04 -0700 Subject: [PATCH 20/20] Add firebase to changeset --- .changeset/purple-chairs-wait.md | 1 + 1 file changed, 1 insertion(+) diff --git a/.changeset/purple-chairs-wait.md b/.changeset/purple-chairs-wait.md index 26d164c6d0e..e29595537cd 100644 --- a/.changeset/purple-chairs-wait.md +++ b/.changeset/purple-chairs-wait.md @@ -1,5 +1,6 @@ --- '@firebase/ai': minor +'firebase': minor --- Add hybrid inference options to the Firebase AI SDK.