From 2e86688f41c5b28156292f791e64120059239c79 Mon Sep 17 00:00:00 2001 From: Erik Eldridge Date: Tue, 13 May 2025 10:22:55 -0700 Subject: [PATCH 1/2] Migrate to LanguageModelMessage --- .../ai/src/methods/chrome-adapter.test.ts | 144 ++++++++++++++---- packages/ai/src/methods/chrome-adapter.ts | 49 ++++-- packages/ai/src/types/language-model.ts | 10 +- 3 files changed, 160 insertions(+), 43 deletions(-) diff --git a/packages/ai/src/methods/chrome-adapter.test.ts b/packages/ai/src/methods/chrome-adapter.test.ts index adb9ae47d87..a94d414684d 100644 --- a/packages/ai/src/methods/chrome-adapter.test.ts +++ b/packages/ai/src/methods/chrome-adapter.test.ts @@ -24,7 +24,7 @@ import { Availability, LanguageModel, LanguageModelCreateOptions, - LanguageModelMessageContent + LanguageModelMessage } from '../types/language-model'; import { match, stub } from 'sinon'; import { GenerateContentRequest, AIErrorCode } from '../types'; @@ -146,7 +146,7 @@ describe('ChromeAdapter', () => { }) ).to.be.false; }); - it('returns false if request content has non-user role', async () => { + it('returns false if request content has "function" role', async () => { const adapter = new ChromeAdapter( { availability: async () => Availability.available @@ -157,7 +157,7 @@ describe('ChromeAdapter', () => { await adapter.isAvailable({ contents: [ { - role: 'model', + role: 'function', parts: [] } ] @@ -320,7 +320,7 @@ describe('ChromeAdapter', () => { } as LanguageModel; const languageModel = { // eslint-disable-next-line @typescript-eslint/no-unused-vars - prompt: (p: LanguageModelMessageContent[]) => Promise.resolve('') + prompt: (p: LanguageModelMessage[]) => Promise.resolve('') } as LanguageModel; const createStub = stub(languageModelProvider, 'create').resolves( languageModel @@ -345,8 +345,13 @@ describe('ChromeAdapter', () => { // Asserts Vertex input type is mapped to Chrome type. expect(promptStub).to.have.been.calledOnceWith([ { - type: 'text', - content: request.contents[0].parts[0].text + role: request.contents[0].role, + content: [ + { + type: 'text', + content: request.contents[0].parts[0].text + } + ] } ]); // Asserts expected output. @@ -366,7 +371,7 @@ describe('ChromeAdapter', () => { } as LanguageModel; const languageModel = { // eslint-disable-next-line @typescript-eslint/no-unused-vars - prompt: (p: LanguageModelMessageContent[]) => Promise.resolve('') + prompt: (p: LanguageModelMessage[]) => Promise.resolve('') } as LanguageModel; const createStub = stub(languageModelProvider, 'create').resolves( languageModel @@ -404,12 +409,17 @@ describe('ChromeAdapter', () => { // Asserts Vertex input type is mapped to Chrome type. expect(promptStub).to.have.been.calledOnceWith([ { - type: 'text', - content: request.contents[0].parts[0].text - }, - { - type: 'image', - content: match.instanceOf(ImageBitmap) + role: request.contents[0].role, + content: [ + { + type: 'text', + content: request.contents[0].parts[0].text + }, + { + type: 'image', + content: match.instanceOf(ImageBitmap) + } + ] } ]); // Asserts expected output. @@ -426,7 +436,7 @@ describe('ChromeAdapter', () => { it('honors prompt options', async () => { const languageModel = { // eslint-disable-next-line @typescript-eslint/no-unused-vars - prompt: (p: LanguageModelMessageContent[]) => Promise.resolve('') + prompt: (p: LanguageModelMessage[]) => Promise.resolve('') } as LanguageModel; const languageModelProvider = { create: () => Promise.resolve(languageModel) @@ -450,13 +460,48 @@ describe('ChromeAdapter', () => { expect(promptStub).to.have.been.calledOnceWith( [ { - type: 'text', - content: request.contents[0].parts[0].text + role: request.contents[0].role, + content: [ + { + type: 'text', + content: request.contents[0].parts[0].text + } + ] } ], promptOptions ); }); + it('normalizes roles', async () => { + const languageModel = { + // eslint-disable-next-line @typescript-eslint/no-unused-vars + prompt: (p: LanguageModelMessage[]) => Promise.resolve('unused') + } as LanguageModel; + const promptStub = stub(languageModel, 'prompt').resolves('unused'); + const languageModelProvider = { + create: () => Promise.resolve(languageModel) + } as LanguageModel; + const adapter = new ChromeAdapter( + languageModelProvider, + 'prefer_on_device' + ); + const request = { + contents: [{ role: 'model', parts: [{ text: 'unused' }] }] + } as GenerateContentRequest; + await adapter.generateContent(request); + expect(promptStub).to.have.been.calledOnceWith([ + { + // Asserts Vertex's "model" role normalized to Chrome's "assistant" role. + role: 'assistant', + content: [ + { + type: 'text', + content: request.contents[0].parts[0].text + } + ] + } + ]); + }); }); describe('countTokens', () => { it('counts tokens is not yet available', async () => { @@ -528,8 +573,13 @@ describe('ChromeAdapter', () => { expect(createStub).to.have.been.calledOnceWith(createOptions); expect(promptStub).to.have.been.calledOnceWith([ { - type: 'text', - content: request.contents[0].parts[0].text + role: request.contents[0].role, + content: [ + { + type: 'text', + content: request.contents[0].parts[0].text + } + ] } ]); const actual = await toStringArray(response.body!); @@ -584,12 +634,17 @@ describe('ChromeAdapter', () => { expect(createStub).to.have.been.calledOnceWith(createOptions); expect(promptStub).to.have.been.calledOnceWith([ { - type: 'text', - content: request.contents[0].parts[0].text - }, - { - type: 'image', - content: match.instanceOf(ImageBitmap) + role: request.contents[0].role, + content: [ + { + type: 'text', + content: request.contents[0].parts[0].text + }, + { + type: 'image', + content: match.instanceOf(ImageBitmap) + } + ] } ]); const actual = await toStringArray(response.body!); @@ -625,13 +680,50 @@ describe('ChromeAdapter', () => { expect(promptStub).to.have.been.calledOnceWith( [ { - type: 'text', - content: request.contents[0].parts[0].text + role: request.contents[0].role, + content: [ + { + type: 'text', + content: request.contents[0].parts[0].text + } + ] } ], promptOptions ); }); + it('normalizes roles', async () => { + const languageModel = { + // eslint-disable-next-line @typescript-eslint/no-unused-vars + promptStreaming: p => new ReadableStream() + } as LanguageModel; + const promptStub = stub(languageModel, 'promptStreaming').returns( + new ReadableStream() + ); + const languageModelProvider = { + create: () => Promise.resolve(languageModel) + } as LanguageModel; + const adapter = new ChromeAdapter( + languageModelProvider, + 'prefer_on_device' + ); + const request = { + contents: [{ role: 'model', parts: [{ text: 'unused' }] }] + } as GenerateContentRequest; + await adapter.generateContentStream(request); + expect(promptStub).to.have.been.calledOnceWith([ + { + // Asserts Vertex's "model" role normalized to Chrome's "assistant" role. + role: 'assistant', + content: [ + { + type: 'text', + content: request.contents[0].parts[0].text + } + ] + } + ]); + }); }); }); diff --git a/packages/ai/src/methods/chrome-adapter.ts b/packages/ai/src/methods/chrome-adapter.ts index b61ad9b5f09..75ffafe795f 100644 --- a/packages/ai/src/methods/chrome-adapter.ts +++ b/packages/ai/src/methods/chrome-adapter.ts @@ -23,12 +23,16 @@ import { InferenceMode, Part, AIErrorCode, - OnDeviceParams + OnDeviceParams, + Content, + Role } from '../types'; import { Availability, LanguageModel, - LanguageModelMessageContent + LanguageModelMessage, + LanguageModelMessageContent, + LanguageModelMessageRole } from '../types/language-model'; /** @@ -115,10 +119,8 @@ export class ChromeAdapter { */ async generateContent(request: GenerateContentRequest): Promise { const session = await this.createSession(); - // TODO: support multiple content objects when Chrome supports - // sequence const contents = await Promise.all( - request.contents[0].parts.map(ChromeAdapter.toLanguageModelMessageContent) + request.contents.map(ChromeAdapter.toLanguageModelMessage) ); const text = await session.prompt( contents, @@ -139,10 +141,8 @@ export class ChromeAdapter { request: GenerateContentRequest ): Promise { const session = await this.createSession(); - // TODO: support multiple content objects when Chrome supports - // sequence const contents = await Promise.all( - request.contents[0].parts.map(ChromeAdapter.toLanguageModelMessageContent) + request.contents.map(ChromeAdapter.toLanguageModelMessage) ); const stream = await session.promptStreaming( contents, @@ -169,12 +169,8 @@ export class ChromeAdapter { } for (const content of request.contents) { - // Returns false if the request contains multiple roles, eg a chat history. - // TODO: remove this guard once LanguageModelMessage is supported. - if (content.role !== 'user') { - logger.debug( - `Non-user role "${content.role}" rejected for on-device inference.` - ); + if (content.role === 'function') { + logger.debug(`"Function" role rejected for on-device inference.`); return false; } @@ -233,6 +229,21 @@ export class ChromeAdapter { }); } + /** + * Converts Vertex {@link Content} object to a Chrome {@link LanguageModelMessage} object. + */ + private static async toLanguageModelMessage( + content: Content + ): Promise { + const languageModelMessageContents = await Promise.all( + content.parts.map(ChromeAdapter.toLanguageModelMessageContent) + ); + return { + role: ChromeAdapter.toLanguageModelMessageRole(content.role), + content: languageModelMessageContents + }; + } + /** * Converts a Vertex Part object to a Chrome LanguageModelMessageContent object. */ @@ -260,6 +271,16 @@ export class ChromeAdapter { throw new Error('Not yet implemented'); } + /** + * Converts a Vertex {@link Role} string to a {@link LanguageModelMessageRole} string. + */ + private static toLanguageModelMessageRole( + role: Role + ): LanguageModelMessageRole { + // Assumes 'function' rule has been filtered by isOnDeviceRequest + return role === 'model' ? 'assistant' : 'user'; + } + /** * Abstracts Chrome session creation. * diff --git a/packages/ai/src/types/language-model.ts b/packages/ai/src/types/language-model.ts index de4020f66bf..97baf1455f0 100644 --- a/packages/ai/src/types/language-model.ts +++ b/packages/ai/src/types/language-model.ts @@ -14,7 +14,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - +/** + * {@see https://github.com/webmachinelearning/prompt-api#full-api-surface-in-web-idl} + */ export interface LanguageModel extends EventTarget { create(options?: LanguageModelCreateOptions): Promise; availability(options?: LanguageModelCreateCoreOptions): Promise; @@ -57,8 +59,10 @@ export interface LanguageModelExpectedInput { type: LanguageModelMessageType; languages?: string[]; } -// TODO: revert to type from Prompt API explainer once it's supported. -export type LanguageModelPrompt = LanguageModelMessageContent[]; +export type LanguageModelPrompt = + | LanguageModelMessage[] + | LanguageModelMessageShorthand[] + | string; export type LanguageModelInitialPrompts = | LanguageModelMessage[] | LanguageModelMessageShorthand[]; From e96b9924d4ea5531afbc98d1ed5209b3ba39df76 Mon Sep 17 00:00:00 2001 From: Erik Eldridge Date: Tue, 27 May 2025 15:53:35 -0700 Subject: [PATCH 2/2] Rename field: content->value --- common/api-review/ai.api.md | 4 ++-- .../ai/src/methods/chrome-adapter.test.ts | 20 +++++++++---------- packages/ai/src/methods/chrome-adapter.ts | 4 ++-- packages/ai/src/types/language-model.ts | 2 +- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/common/api-review/ai.api.md b/common/api-review/ai.api.md index a7da6210ada..97d25b9e03d 100644 --- a/common/api-review/ai.api.md +++ b/common/api-review/ai.api.md @@ -705,10 +705,10 @@ export interface LanguageModelMessage { // @public (undocumented) export interface LanguageModelMessageContent { - // (undocumented) - content: LanguageModelMessageContentValue; // (undocumented) type: LanguageModelMessageType; + // (undocumented) + value: LanguageModelMessageContentValue; } // @public (undocumented) diff --git a/packages/ai/src/methods/chrome-adapter.test.ts b/packages/ai/src/methods/chrome-adapter.test.ts index a94d414684d..f8ea80b0e09 100644 --- a/packages/ai/src/methods/chrome-adapter.test.ts +++ b/packages/ai/src/methods/chrome-adapter.test.ts @@ -349,7 +349,7 @@ describe('ChromeAdapter', () => { content: [ { type: 'text', - content: request.contents[0].parts[0].text + value: request.contents[0].parts[0].text } ] } @@ -413,11 +413,11 @@ describe('ChromeAdapter', () => { content: [ { type: 'text', - content: request.contents[0].parts[0].text + value: request.contents[0].parts[0].text }, { type: 'image', - content: match.instanceOf(ImageBitmap) + value: match.instanceOf(ImageBitmap) } ] } @@ -464,7 +464,7 @@ describe('ChromeAdapter', () => { content: [ { type: 'text', - content: request.contents[0].parts[0].text + value: request.contents[0].parts[0].text } ] } @@ -496,7 +496,7 @@ describe('ChromeAdapter', () => { content: [ { type: 'text', - content: request.contents[0].parts[0].text + value: request.contents[0].parts[0].text } ] } @@ -577,7 +577,7 @@ describe('ChromeAdapter', () => { content: [ { type: 'text', - content: request.contents[0].parts[0].text + value: request.contents[0].parts[0].text } ] } @@ -638,11 +638,11 @@ describe('ChromeAdapter', () => { content: [ { type: 'text', - content: request.contents[0].parts[0].text + value: request.contents[0].parts[0].text }, { type: 'image', - content: match.instanceOf(ImageBitmap) + value: match.instanceOf(ImageBitmap) } ] } @@ -684,7 +684,7 @@ describe('ChromeAdapter', () => { content: [ { type: 'text', - content: request.contents[0].parts[0].text + value: request.contents[0].parts[0].text } ] } @@ -718,7 +718,7 @@ describe('ChromeAdapter', () => { content: [ { type: 'text', - content: request.contents[0].parts[0].text + value: request.contents[0].parts[0].text } ] } diff --git a/packages/ai/src/methods/chrome-adapter.ts b/packages/ai/src/methods/chrome-adapter.ts index 75ffafe795f..e7bb39c34c8 100644 --- a/packages/ai/src/methods/chrome-adapter.ts +++ b/packages/ai/src/methods/chrome-adapter.ts @@ -253,7 +253,7 @@ export class ChromeAdapter { if (part.text) { return { type: 'text', - content: part.text + value: part.text }; } else if (part.inlineData) { const formattedImageContent = await fetch( @@ -263,7 +263,7 @@ export class ChromeAdapter { const imageBitmap = await createImageBitmap(imageBlob); return { type: 'image', - content: imageBitmap + value: imageBitmap }; } // Assumes contents have been verified to contain only a single TextPart. diff --git a/packages/ai/src/types/language-model.ts b/packages/ai/src/types/language-model.ts index 97baf1455f0..503f3d49d05 100644 --- a/packages/ai/src/types/language-model.ts +++ b/packages/ai/src/types/language-model.ts @@ -76,7 +76,7 @@ export interface LanguageModelMessageShorthand { } export interface LanguageModelMessageContent { type: LanguageModelMessageType; - content: LanguageModelMessageContentValue; + value: LanguageModelMessageContentValue; } export type LanguageModelMessageRole = 'system' | 'user' | 'assistant'; export type LanguageModelMessageType = 'text' | 'image' | 'audio';