From a6eea14fa0e45a6e2b2aeacc364442cc777ed4c6 Mon Sep 17 00:00:00 2001 From: Daniel La Rocque Date: Tue, 16 Sep 2025 11:50:16 -0400 Subject: [PATCH 1/9] feat(ai): Add suppport for URL Context --- .changeset/poor-rings-admire.md | 5 + common/api-review/ai.api.md | 40 ++++- docs-devsite/_toc.yaml | 8 + docs-devsite/ai.generatecontentcandidate.md | 9 ++ docs-devsite/ai.md | 36 ++++- docs-devsite/ai.urlcontext.md | 21 +++ docs-devsite/ai.urlcontextmetadata.md | 35 +++++ docs-devsite/ai.urlcontexttool.md | 33 +++++ docs-devsite/ai.urlmetadata.md | 46 ++++++ docs-devsite/ai.usagemetadata.md | 22 +++ .../ai/integration/generate-content.test.ts | 139 +++++++++++++++++- packages/ai/src/googleai-mappers.ts | 3 +- .../ai/src/methods/generate-content.test.ts | 51 +++++++ packages/ai/src/requests/stream-reader.ts | 2 + packages/ai/src/types/googleai.ts | 4 +- packages/ai/src/types/requests.ts | 25 +++- packages/ai/src/types/responses.ts | 86 +++++++++++ 17 files changed, 555 insertions(+), 10 deletions(-) create mode 100644 .changeset/poor-rings-admire.md create mode 100644 docs-devsite/ai.urlcontext.md create mode 100644 docs-devsite/ai.urlcontextmetadata.md create mode 100644 docs-devsite/ai.urlcontexttool.md create mode 100644 docs-devsite/ai.urlmetadata.md diff --git a/.changeset/poor-rings-admire.md b/.changeset/poor-rings-admire.md new file mode 100644 index 00000000000..c6b3b7dd029 --- /dev/null +++ b/.changeset/poor-rings-admire.md @@ -0,0 +1,5 @@ +--- +'@firebase/ai': minor +--- + +Add support for the URL Context tool. diff --git a/common/api-review/ai.api.md b/common/api-review/ai.api.md index 5a8e5df6ab9..ba4e7959e30 100644 --- a/common/api-review/ai.api.md +++ b/common/api-review/ai.api.md @@ -375,6 +375,8 @@ export interface GenerateContentCandidate { index: number; // (undocumented) safetyRatings?: SafetyRating[]; + // (undocumented) + urlContextMetadata?: URLContextMetadata; } // @public @@ -523,6 +525,8 @@ export interface GoogleAIGenerateContentCandidate { index: number; // (undocumented) safetyRatings?: SafetyRating[]; + // (undocumented) + urlContextMetadata?: URLContextMetadata; } // Warning: (ae-internal-missing-underscore) The name "GoogleAIGenerateContentResponse" should be prefixed with an underscore because the declaration is marked as @internal @@ -1199,7 +1203,7 @@ export interface ThinkingConfig { } // @public -export type Tool = FunctionDeclarationsTool | GoogleSearchTool; +export type Tool = FunctionDeclarationsTool | GoogleSearchTool | URLContextTool; // @public export interface ToolConfig { @@ -1210,6 +1214,38 @@ export interface ToolConfig { // @public export type TypedSchema = IntegerSchema | NumberSchema | StringSchema | BooleanSchema | ObjectSchema | ArraySchema | AnyOfSchema; +// @public +export interface URLContext { +} + +// @public +export interface URLContextMetadata { + urlMetadata: URLMetadata[]; +} + +// @public +export interface URLContextTool { + urlContext: URLContext; +} + +// @public +export interface URLMetadata { + retrievedUrl?: string; + urlRetrievalStatus?: URLRetrievalStatus; +} + +// @public +export const URLRetrievalStatus: { + URL_RETRIEVAL_STATUS_UNSPECIFIED: string; + URL_RETRIEVAL_STATUS_SUCCESS: string; + URL_RETRIEVAL_STATUS_ERROR: string; + URL_RETRIEVAL_STATUS_PAYWALL: string; + URL_RETRIEVAL_STATUS_UNSAFE: string; +}; + +// @public +export type URLRetrievalStatus = (typeof URLRetrievalStatus)[keyof typeof URLRetrievalStatus]; + // @public export interface UsageMetadata { // (undocumented) @@ -1221,6 +1257,8 @@ export interface UsageMetadata { // (undocumented) promptTokensDetails?: ModalityTokenCount[]; thoughtsTokenCount?: number; + toolUsePromptTokenCount?: number; + toolUsePromptTokensDetails?: ModalityTokenCount[]; // (undocumented) totalTokenCount: number; } diff --git a/docs-devsite/_toc.yaml b/docs-devsite/_toc.yaml index 9161f501aa3..cf5d00bdb3b 100644 --- a/docs-devsite/_toc.yaml +++ b/docs-devsite/_toc.yaml @@ -192,6 +192,14 @@ toc: path: /docs/reference/js/ai.thinkingconfig.md - title: ToolConfig path: /docs/reference/js/ai.toolconfig.md + - title: URLContext + path: /docs/reference/js/ai.urlcontext.md + - title: URLContextMetadata + path: /docs/reference/js/ai.urlcontextmetadata.md + - title: URLContextTool + path: /docs/reference/js/ai.urlcontexttool.md + - title: URLMetadata + path: /docs/reference/js/ai.urlmetadata.md - title: UsageMetadata path: /docs/reference/js/ai.usagemetadata.md - title: VertexAIBackend diff --git a/docs-devsite/ai.generatecontentcandidate.md b/docs-devsite/ai.generatecontentcandidate.md index ca0383549a7..1691442ecfa 100644 --- a/docs-devsite/ai.generatecontentcandidate.md +++ b/docs-devsite/ai.generatecontentcandidate.md @@ -29,6 +29,7 @@ export interface GenerateContentCandidate | [groundingMetadata](./ai.generatecontentcandidate.md#generatecontentcandidategroundingmetadata) | [GroundingMetadata](./ai.groundingmetadata.md#groundingmetadata_interface) | | | [index](./ai.generatecontentcandidate.md#generatecontentcandidateindex) | number | | | [safetyRatings](./ai.generatecontentcandidate.md#generatecontentcandidatesafetyratings) | [SafetyRating](./ai.safetyrating.md#safetyrating_interface)\[\] | | +| [urlContextMetadata](./ai.generatecontentcandidate.md#generatecontentcandidateurlcontextmetadata) | [URLContextMetadata](./ai.urlcontextmetadata.md#urlcontextmetadata_interface) | | ## GenerateContentCandidate.citationMetadata @@ -85,3 +86,11 @@ index: number; ```typescript safetyRatings?: SafetyRating[]; ``` + +## GenerateContentCandidate.urlContextMetadata + +Signature: + +```typescript +urlContextMetadata?: URLContextMetadata; +``` diff --git a/docs-devsite/ai.md b/docs-devsite/ai.md index d70b381d6fe..baf441cdd89 100644 --- a/docs-devsite/ai.md +++ b/docs-devsite/ai.md @@ -129,6 +129,10 @@ The Firebase AI Web SDK. | [TextPart](./ai.textpart.md#textpart_interface) | Content part interface if the part represents a text string. | | [ThinkingConfig](./ai.thinkingconfig.md#thinkingconfig_interface) | Configuration for "thinking" behavior of compatible Gemini models.Certain models utilize a thinking process before generating a response. This allows them to reason through complex problems and plan a more coherent and accurate answer. | | [ToolConfig](./ai.toolconfig.md#toolconfig_interface) | Tool config. This config is shared for all tools provided in the request. | +| [URLContext](./ai.urlcontext.md#urlcontext_interface) | Specifies the URL Context configuration. | +| [URLContextMetadata](./ai.urlcontextmetadata.md#urlcontextmetadata_interface) | Metadata related to [URLContextTool](./ai.urlcontexttool.md#urlcontexttool_interface). | +| [URLContextTool](./ai.urlcontexttool.md#urlcontexttool_interface) | A tool that allows you to provide additional context to the models in the form of URLs. By including URLs in your request, the Gemini model will access the content from those pages to inform and enhances it's response. At most 20 URLs can be consumed per request. | +| [URLMetadata](./ai.urlmetadata.md#urlmetadata_interface) | Metadata for a URL that was used to provide context to the Gemini model. | | [UsageMetadata](./ai.usagemetadata.md#usagemetadata_interface) | Usage metadata about a [GenerateContentResponse](./ai.generatecontentresponse.md#generatecontentresponse_interface). | | [VideoMetadata](./ai.videometadata.md#videometadata_interface) | Describes the input video content. | | [VoiceConfig](./ai.voiceconfig.md#voiceconfig_interface) | (Public Preview) Configuration for the voice to used in speech synthesis. | @@ -158,6 +162,7 @@ The Firebase AI Web SDK. | [POSSIBLE\_ROLES](./ai.md#possible_roles) | Possible roles. | | [ResponseModality](./ai.md#responsemodality) | (Public Preview) Generation modalities to be returned in generation responses. | | [SchemaType](./ai.md#schematype) | Contains the list of OpenAPI data types as defined by the [OpenAPI specification](https://swagger.io/docs/specification/data-models/data-types/) | +| [URLRetrievalStatus](./ai.md#urlretrievalstatus) | The status of a URL retrieval. | ## Type Aliases @@ -188,6 +193,7 @@ The Firebase AI Web SDK. | [SchemaType](./ai.md#schematype) | Contains the list of OpenAPI data types as defined by the [OpenAPI specification](https://swagger.io/docs/specification/data-models/data-types/) | | [Tool](./ai.md#tool) | Defines a tool that model can call to access external knowledge. | | [TypedSchema](./ai.md#typedschema) | A type that includes all specific Schema types. | +| [URLRetrievalStatus](./ai.md#urlretrievalstatus) | The status of a URL retrieval. | ## function(app, ...) @@ -712,6 +718,22 @@ SchemaType: { } ``` +## URLRetrievalStatus + +The status of a URL retrieval. + +Signature: + +```typescript +URLRetrievalStatus: { + URL_RETRIEVAL_STATUS_UNSPECIFIED: string; + URL_RETRIEVAL_STATUS_SUCCESS: string; + URL_RETRIEVAL_STATUS_ERROR: string; + URL_RETRIEVAL_STATUS_PAYWALL: string; + URL_RETRIEVAL_STATUS_UNSAFE: string; +} +``` + ## AIErrorCode Standardized error codes that [AIError](./ai.aierror.md#aierror_class) can have. @@ -971,7 +993,7 @@ Defines a tool that model can call to access external knowledge. Signature: ```typescript -export type Tool = FunctionDeclarationsTool | GoogleSearchTool; +export type Tool = FunctionDeclarationsTool | GoogleSearchTool | URLContextTool; ``` ## TypedSchema @@ -983,3 +1005,15 @@ A type that includes all specific Schema types. ```typescript export type TypedSchema = IntegerSchema | NumberSchema | StringSchema | BooleanSchema | ObjectSchema | ArraySchema | AnyOfSchema; ``` + +## URLRetrievalStatus + +The status of a URL retrieval. + +URL\_RETRIEVAL\_STATUS\_UNSPECIFIED: Unspecified retrieval status.
URL\_RETRIEVAL\_STATUS\_SUCCESS: The URL retrieval was successful.
URL\_RETRIEVAL\_STATUS\_ERROR: The URL retrieval failed due to an error.
URL\_RETRIEVAL\_STATUS\_PAYWALL: The URL retrieval failed because the content is behind a paywall.
URL\_RETRIEVAL\_STATUS\_UNSAFE: The URL retrieval failed because the content is unsafe.
+ +Signature: + +```typescript +export type URLRetrievalStatus = (typeof URLRetrievalStatus)[keyof typeof URLRetrievalStatus]; +``` diff --git a/docs-devsite/ai.urlcontext.md b/docs-devsite/ai.urlcontext.md new file mode 100644 index 00000000000..02dbcd49f52 --- /dev/null +++ b/docs-devsite/ai.urlcontext.md @@ -0,0 +1,21 @@ +Project: /docs/reference/js/_project.yaml +Book: /docs/reference/_book.yaml +page_type: reference + +{% comment %} +DO NOT EDIT THIS FILE! +This is generated by the JS SDK team, and any local changes will be +overwritten. Changes should be made in the source code at +https://github.com/firebase/firebase-js-sdk +{% endcomment %} + +# URLContext interface +Specifies the URL Context configuration. + +Currently, this is an empty object, but it's reserved for future configuration. + +Signature: + +```typescript +export interface URLContext +``` diff --git a/docs-devsite/ai.urlcontextmetadata.md b/docs-devsite/ai.urlcontextmetadata.md new file mode 100644 index 00000000000..4155c1e9a6a --- /dev/null +++ b/docs-devsite/ai.urlcontextmetadata.md @@ -0,0 +1,35 @@ +Project: /docs/reference/js/_project.yaml +Book: /docs/reference/_book.yaml +page_type: reference + +{% comment %} +DO NOT EDIT THIS FILE! +This is generated by the JS SDK team, and any local changes will be +overwritten. Changes should be made in the source code at +https://github.com/firebase/firebase-js-sdk +{% endcomment %} + +# URLContextMetadata interface +Metadata related to [URLContextTool](./ai.urlcontexttool.md#urlcontexttool_interface). + +Signature: + +```typescript +export interface URLContextMetadata +``` + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [urlMetadata](./ai.urlcontextmetadata.md#urlcontextmetadataurlmetadata) | [URLMetadata](./ai.urlmetadata.md#urlmetadata_interface)\[\] | List of URL metadata were used to provide context to the Gemini model. | + +## URLContextMetadata.urlMetadata + +List of URL metadata were used to provide context to the Gemini model. + +Signature: + +```typescript +urlMetadata: URLMetadata[]; +``` diff --git a/docs-devsite/ai.urlcontexttool.md b/docs-devsite/ai.urlcontexttool.md new file mode 100644 index 00000000000..bb9087ab4b8 --- /dev/null +++ b/docs-devsite/ai.urlcontexttool.md @@ -0,0 +1,33 @@ +Project: /docs/reference/js/_project.yaml +Book: /docs/reference/_book.yaml +page_type: reference + +{% comment %} +DO NOT EDIT THIS FILE! +This is generated by the JS SDK team, and any local changes will be +overwritten. Changes should be made in the source code at +https://github.com/firebase/firebase-js-sdk +{% endcomment %} + +# URLContextTool interface +A tool that allows you to provide additional context to the models in the form of URLs. By including URLs in your request, the Gemini model will access the content from those pages to inform and enhances it's response. At most 20 URLs can be consumed per request. + +Signature: + +```typescript +export interface URLContextTool +``` + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [urlContext](./ai.urlcontexttool.md#urlcontexttoolurlcontext) | [URLContext](./ai.urlcontext.md#urlcontext_interface) | | + +## URLContextTool.urlContext + +Signature: + +```typescript +urlContext: URLContext; +``` diff --git a/docs-devsite/ai.urlmetadata.md b/docs-devsite/ai.urlmetadata.md new file mode 100644 index 00000000000..8e4aec5333f --- /dev/null +++ b/docs-devsite/ai.urlmetadata.md @@ -0,0 +1,46 @@ +Project: /docs/reference/js/_project.yaml +Book: /docs/reference/_book.yaml +page_type: reference + +{% comment %} +DO NOT EDIT THIS FILE! +This is generated by the JS SDK team, and any local changes will be +overwritten. Changes should be made in the source code at +https://github.com/firebase/firebase-js-sdk +{% endcomment %} + +# URLMetadata interface +Metadata for a URL that was used to provide context to the Gemini model. + +Signature: + +```typescript +export interface URLMetadata +``` + +## Properties + +| Property | Type | Description | +| --- | --- | --- | +| [retrievedUrl](./ai.urlmetadata.md#urlmetadataretrievedurl) | string | The retrieved URL. | +| [urlRetrievalStatus](./ai.urlmetadata.md#urlmetadataurlretrievalstatus) | [URLRetrievalStatus](./ai.md#urlretrievalstatus) | The status of the URL retrieval. | + +## URLMetadata.retrievedUrl + +The retrieved URL. + +Signature: + +```typescript +retrievedUrl?: string; +``` + +## URLMetadata.urlRetrievalStatus + +The status of the URL retrieval. + +Signature: + +```typescript +urlRetrievalStatus?: URLRetrievalStatus; +``` diff --git a/docs-devsite/ai.usagemetadata.md b/docs-devsite/ai.usagemetadata.md index 954fcc6e530..c82593b978a 100644 --- a/docs-devsite/ai.usagemetadata.md +++ b/docs-devsite/ai.usagemetadata.md @@ -27,6 +27,8 @@ export interface UsageMetadata | [promptTokenCount](./ai.usagemetadata.md#usagemetadataprompttokencount) | number | | | [promptTokensDetails](./ai.usagemetadata.md#usagemetadataprompttokensdetails) | [ModalityTokenCount](./ai.modalitytokencount.md#modalitytokencount_interface)\[\] | | | [thoughtsTokenCount](./ai.usagemetadata.md#usagemetadatathoughtstokencount) | number | The number of tokens used by the model's internal "thinking" process. | +| [toolUsePromptTokenCount](./ai.usagemetadata.md#usagemetadatatooluseprompttokencount) | number | The number of tokens in the results from tool executions, which are provided back to the model as input. | +| [toolUsePromptTokensDetails](./ai.usagemetadata.md#usagemetadatatooluseprompttokensdetails) | [ModalityTokenCount](./ai.modalitytokencount.md#modalitytokencount_interface)\[\] | A list of tokens used by tools whose usage was triggered from a prompt, broken down by modality. | | [totalTokenCount](./ai.usagemetadata.md#usagemetadatatotaltokencount) | number | | ## UsageMetadata.candidatesTokenCount @@ -71,6 +73,26 @@ The number of tokens used by the model's internal "thinking" process. thoughtsTokenCount?: number; ``` +## UsageMetadata.toolUsePromptTokenCount + +The number of tokens in the results from tool executions, which are provided back to the model as input. + +Signature: + +```typescript +toolUsePromptTokenCount?: number; +``` + +## UsageMetadata.toolUsePromptTokensDetails + +A list of tokens used by tools whose usage was triggered from a prompt, broken down by modality. + +Signature: + +```typescript +toolUsePromptTokensDetails?: ModalityTokenCount[]; +``` + ## UsageMetadata.totalTokenCount Signature: diff --git a/packages/ai/integration/generate-content.test.ts b/packages/ai/integration/generate-content.test.ts index 0b83df38ecb..9c922ff8d89 100644 --- a/packages/ai/integration/generate-content.test.ts +++ b/packages/ai/integration/generate-content.test.ts @@ -17,17 +17,20 @@ import { expect } from 'chai'; import { + BackendType, Content, GenerationConfig, HarmBlockThreshold, HarmCategory, Modality, SafetySetting, + URLRetrievalStatus, getGenerativeModel } from '../src'; import { testConfigs, TOKEN_COUNT_DELTA } from './constants'; -describe('Generate Content', () => { +describe('Generate Content', function () { + this.timeout(20_000); testConfigs.forEach(testConfig => { describe(`${testConfig.toString()}`, () => { const commonGenerationConfig: GenerationConfig = { @@ -39,19 +42,19 @@ describe('Generate Content', () => { const commonSafetySettings: SafetySetting[] = [ { category: HarmCategory.HARM_CATEGORY_HARASSMENT, - threshold: HarmBlockThreshold.BLOCK_LOW_AND_ABOVE + threshold: HarmBlockThreshold.BLOCK_NONE }, { category: HarmCategory.HARM_CATEGORY_HATE_SPEECH, - threshold: HarmBlockThreshold.BLOCK_LOW_AND_ABOVE + threshold: HarmBlockThreshold.BLOCK_NONE }, { category: HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT, - threshold: HarmBlockThreshold.BLOCK_LOW_AND_ABOVE + threshold: HarmBlockThreshold.BLOCK_NONE }, { category: HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT, - threshold: HarmBlockThreshold.BLOCK_LOW_AND_ABOVE + threshold: HarmBlockThreshold.BLOCK_NONE } ]; @@ -188,6 +191,132 @@ describe('Generate Content', () => { }); }); + describe('URL Context', async () => { + // URL Context is not supported in Google AI for gemini-2.0-flash + if ( + testConfig.ai.backend.backendType === BackendType.GOOGLE_AI && + testConfig.model === 'gemini-2.0-flash' + ) { + return; + } + + it('generateContent: url context', async () => { + const model = getGenerativeModel(testConfig.ai, { + model: testConfig.model, + generationConfig: commonGenerationConfig, + safetySettings: commonSafetySettings, + tools: [{ urlContext: {} }] + }); + + const result = await model.generateContent( + 'Summarize this website https://berkshirehathaway.com' + ); + const response = result.response; + const urlContextMetadata = + response.candidates?.[0].urlContextMetadata; + expect(urlContextMetadata?.urlMetadata).to.exist; + expect( + urlContextMetadata?.urlMetadata.length + ).to.be.greaterThanOrEqual(1); + expect(urlContextMetadata?.urlMetadata[0].retrievedUrl).to.exist; + expect(urlContextMetadata?.urlMetadata[0].retrievedUrl).to.equal( + 'https://berkshirehathaway.com' + ); + expect( + urlContextMetadata?.urlMetadata[0].urlRetrievalStatus + ).to.equal(URLRetrievalStatus.URL_RETRIEVAL_STATUS_SUCCESS); + + const usageMetadata = response.usageMetadata; + expect(usageMetadata).to.exist; + expect(usageMetadata?.toolUsePromptTokenCount).to.exist; + expect(usageMetadata?.toolUsePromptTokenCount).to.be.greaterThan(0); + }); + + it('generateContent: url context and google search grounding', async () => { + const model = getGenerativeModel(testConfig.ai, { + model: testConfig.model, + generationConfig: commonGenerationConfig, + safetySettings: commonSafetySettings, + tools: [{ urlContext: {} }, { googleSearch: {} }] + }); + + const result = await model.generateContent( + 'According to https://info.cern.ch/hypertext/WWW/TheProject.html, what is the WorldWideWeb? Search the web for other definitions.' + ); + const response = result.response; + const trimmedText = response.text().trim(); + const urlContextMetadata = + response.candidates?.[0].urlContextMetadata; + const groundingMetadata = response.candidates?.[0].groundingMetadata; + expect(trimmedText).to.contain( + 'hypermedia information retrieval initiative' + ); + expect(urlContextMetadata?.urlMetadata).to.exist; + expect( + urlContextMetadata?.urlMetadata.length + ).to.be.greaterThanOrEqual(1); + expect(urlContextMetadata?.urlMetadata[0].retrievedUrl).to.exist; + expect(urlContextMetadata?.urlMetadata[0].retrievedUrl).to.equal( + 'https://info.cern.ch/hypertext/WWW/TheProject.html' + ); + expect( + urlContextMetadata?.urlMetadata[0].urlRetrievalStatus + ).to.equal(URLRetrievalStatus.URL_RETRIEVAL_STATUS_SUCCESS); + expect(groundingMetadata).to.exist; + expect(groundingMetadata?.groundingChunks).to.exist; + expect( + groundingMetadata?.groundingChunks!.length + ).to.be.greaterThanOrEqual(1); + expect( + groundingMetadata?.groundingSupports!.length + ).to.be.greaterThanOrEqual(1); + + const usageMetadata = response.usageMetadata; + expect(usageMetadata).to.exist; + expect(usageMetadata?.toolUsePromptTokenCount).to.exist; + expect(usageMetadata?.toolUsePromptTokenCount).to.be.greaterThan(0); + }); + + it('generateContent: url context and google search grounding without URLs in prompt', async () => { + const model = getGenerativeModel(testConfig.ai, { + model: testConfig.model, + generationConfig: commonGenerationConfig, + safetySettings: commonSafetySettings, + tools: [{ urlContext: {} }, { googleSearch: {} }] + }); + + const result = await model.generateContent( + 'Recommend 3 books for beginners to read to learn more about the latest advancements in Quantum Computing.' + ); + const response = result.response; + const urlContextMetadata = + response.candidates?.[0].urlContextMetadata; + const groundingMetadata = response.candidates?.[0].groundingMetadata; + if (testConfig.ai.backend.backendType === BackendType.GOOGLE_AI) { + expect(urlContextMetadata?.urlMetadata).to.exist; + expect( + urlContextMetadata?.urlMetadata.length + ).to.be.greaterThanOrEqual(1); + expect(urlContextMetadata?.urlMetadata[0].retrievedUrl).to.exist; + expect( + urlContextMetadata?.urlMetadata[0].urlRetrievalStatus + ).to.equal(URLRetrievalStatus.URL_RETRIEVAL_STATUS_SUCCESS); + expect(groundingMetadata).to.exist; + expect(groundingMetadata?.groundingChunks).to.exist; + + const usageMetadata = response.usageMetadata; + expect(usageMetadata).to.exist; + expect(usageMetadata?.toolUsePromptTokenCount).to.exist; + expect(usageMetadata?.toolUsePromptTokenCount).to.be.greaterThan(0); + } else { + // URL Context does not integrate with Google Search Grounding in Vertex AI + expect(urlContextMetadata?.urlMetadata).to.not.exist; + expect(groundingMetadata).to.exist; + expect(groundingMetadata?.groundingChunks).to.exist; + } + }); + }); + it('generateContentStream: text input, text output', async () => { const model = getGenerativeModel(testConfig.ai, { model: testConfig.model, diff --git a/packages/ai/src/googleai-mappers.ts b/packages/ai/src/googleai-mappers.ts index 23c238c1e3b..b3e4206f33c 100644 --- a/packages/ai/src/googleai-mappers.ts +++ b/packages/ai/src/googleai-mappers.ts @@ -193,7 +193,8 @@ export function mapGenerateContentCandidates( finishMessage: candidate.finishMessage, safetyRatings: mappedSafetyRatings, citationMetadata, - groundingMetadata: candidate.groundingMetadata + groundingMetadata: candidate.groundingMetadata, + urlContextMetadata: candidate.urlContextMetadata }; mappedCandidates.push(mappedCandidate); }); diff --git a/packages/ai/src/methods/generate-content.test.ts b/packages/ai/src/methods/generate-content.test.ts index 19c0761949a..1c3aafd3258 100644 --- a/packages/ai/src/methods/generate-content.test.ts +++ b/packages/ai/src/methods/generate-content.test.ts @@ -247,6 +247,57 @@ describe('generateContent()', () => { false, match.any ); + + it('url context', async () => { + const mockResponse = getMockResponse( + 'vertexAI', + 'unary-success-url-context.json' + ); + const makeRequestStub = stub(request, 'makeRequest').resolves( + mockResponse as Response + ); + const result = await generateContent( + fakeApiSettings, + 'model', + fakeRequestParams + ); + expect(result.response.text()).to.include( + 'The temperature is 67°F (19°C)' + ); + const groundingMetadata = + result.response.candidates?.[0].groundingMetadata; + expect(groundingMetadata).to.not.be.undefined; + expect(groundingMetadata!.searchEntryPoint?.renderedContent).to.contain( + 'div' + ); + expect(groundingMetadata!.groundingChunks?.length).to.equal(2); + expect(groundingMetadata!.groundingChunks?.[0].web?.uri).to.contain( + 'https://vertexaisearch.cloud.google.com' + ); + expect(groundingMetadata!.groundingChunks?.[0].web?.title).to.equal( + 'accuweather.com' + ); + expect(groundingMetadata!.groundingSupports?.length).to.equal(3); + expect( + groundingMetadata!.groundingSupports?.[0].groundingChunkIndices + ).to.deep.equal([0]); + expect(groundingMetadata!.groundingSupports?.[0].segment).to.deep.equal({ + endIndex: 56, + text: 'The current weather in London, United Kingdom is cloudy.' + }); + expect(groundingMetadata!.groundingSupports?.[0].segment?.partIndex).to.be + .undefined; + expect(groundingMetadata!.groundingSupports?.[0].segment?.startIndex).to + .be.undefined; + + expect(makeRequestStub).to.be.calledWith( + 'model', + Task.GENERATE_CONTENT, + fakeApiSettings, + false, + match.any + ); + }); }); it('blocked prompt', async () => { const mockResponse = getMockResponse( diff --git a/packages/ai/src/requests/stream-reader.ts b/packages/ai/src/requests/stream-reader.ts index c3a35b1da4a..de64c37522a 100644 --- a/packages/ai/src/requests/stream-reader.ts +++ b/packages/ai/src/requests/stream-reader.ts @@ -192,6 +192,8 @@ export function aggregateResponses( candidate.safetyRatings; aggregatedResponse.candidates[i].groundingMetadata = candidate.groundingMetadata; + aggregatedResponse.candidates[i].urlContextMetadata = + candidate.urlContextMetadata; /** * Candidates should always have content and parts, but this handles diff --git a/packages/ai/src/types/googleai.ts b/packages/ai/src/types/googleai.ts index 38c27b3fe8b..eb282b094fc 100644 --- a/packages/ai/src/types/googleai.ts +++ b/packages/ai/src/types/googleai.ts @@ -23,7 +23,8 @@ import { GroundingMetadata, PromptFeedback, SafetyRating, - UsageMetadata + UsageMetadata, + URLContextMetadata } from '../public-types'; import { Content, Part } from './content'; @@ -60,6 +61,7 @@ export interface GoogleAIGenerateContentCandidate { safetyRatings?: SafetyRating[]; citationMetadata?: GoogleAICitationMetadata; groundingMetadata?: GroundingMetadata; + urlContextMetadata?: URLContextMetadata; } /** diff --git a/packages/ai/src/types/requests.ts b/packages/ai/src/types/requests.ts index 21ccce8bd11..815986e4cc4 100644 --- a/packages/ai/src/types/requests.ts +++ b/packages/ai/src/types/requests.ts @@ -239,7 +239,7 @@ export interface RequestOptions { * Defines a tool that model can call to access external knowledge. * @public */ -export type Tool = FunctionDeclarationsTool | GoogleSearchTool; +export type Tool = FunctionDeclarationsTool | GoogleSearchTool | URLContextTool; /** * Structured representation of a function declaration as defined by the @@ -305,6 +305,29 @@ export interface GoogleSearchTool { */ export interface GoogleSearch {} +/** + * A tool that allows you to provide additional context to the models in the form of URLs. + * By including URLs in your request, the Gemini model will access the content from those pages + * to inform and enhance its response. + * + * @public + */ +export interface URLContextTool { + /** + * Specifies the URL Context configuration. + */ + urlContext: URLContext; +} + +/** + * Specifies the URL Context configuration. + * + * @remarks Currently, this is an empty object, but it's reserved for future configuration. + * + * @public + */ +export interface URLContext {} + /** * A `FunctionDeclarationsTool` is a piece of code that enables the system to * interact with external systems to perform an action, or set of actions, diff --git a/packages/ai/src/types/responses.ts b/packages/ai/src/types/responses.ts index 4a01e79a77c..ec03d2e201f 100644 --- a/packages/ai/src/types/responses.ts +++ b/packages/ai/src/types/responses.ts @@ -116,8 +116,17 @@ export interface UsageMetadata { */ thoughtsTokenCount?: number; totalTokenCount: number; + /** + * The number of tokens in the results from tool executions, which are provided back to the + * model as input. + */ + toolUsePromptTokenCount?: number; promptTokensDetails?: ModalityTokenCount[]; candidatesTokensDetails?: ModalityTokenCount[]; + /** + * A list of tokens used by tools whose usage was triggered from a prompt, broken down by modality. + */ + toolUsePromptTokensDetails?: ModalityTokenCount[]; } /** @@ -160,6 +169,7 @@ export interface GenerateContentCandidate { safetyRatings?: SafetyRating[]; citationMetadata?: CitationMetadata; groundingMetadata?: GroundingMetadata; + urlContextMetadata?: URLContextMetadata; } /** @@ -349,6 +359,82 @@ export interface Segment { text: string; } +/** + * Metadata related to {@link URLContextTool}. + * + * @public + */ +export interface URLContextMetadata { + /** + * List of URL metadata were used to provide context to the Gemini model. + */ + urlMetadata: URLMetadata[]; +} + +/** + * Metadata for a URL that was used to provide context to the Gemini model. + * + * @public + */ +export interface URLMetadata { + /** + * The retrieved URL. + */ + retrievedUrl?: string; + /** + * The status of the URL retrieval. + */ + urlRetrievalStatus?: URLRetrievalStatus; +} + +/** + * The status of a URL retrieval. + * + * @public + */ +export const URLRetrievalStatus = { + /** + * Unspecified retrieval status. + */ + URL_RETRIEVAL_STATUS_UNSPECIFIED: 'URL_RETRIEVAL_STATUS_UNSPECIFIED', + /** + * The URL retrieval was successful. + */ + URL_RETRIEVAL_STATUS_SUCCESS: 'URL_RETRIEVAL_STATUS_SUCCESS', + /** + * The URL retrieval failed due to an error. + */ + URL_RETRIEVAL_STATUS_ERROR: 'URL_RETRIEVAL_STATUS_ERROR', + /** + * The URL retrieval failed because the content is behind a paywall. + */ + URL_RETRIEVAL_STATUS_PAYWALL: 'URL_RETRIEVAL_STATUS_PAYWALL', + /** + * The URL retrieval failed because the content is unsafe. + */ + URL_RETRIEVAL_STATUS_UNSAFE: 'URL_RETRIEVAL_STATUS_UNSAFE' +}; + +/** + * The status of a URL retrieval. + * + * @remarks + * URL_RETRIEVAL_STATUS_UNSPECIFIED: Unspecified retrieval status. + *
+ * URL_RETRIEVAL_STATUS_SUCCESS: The URL retrieval was successful. + *
+ * URL_RETRIEVAL_STATUS_ERROR: The URL retrieval failed due to an error. + *
+ * URL_RETRIEVAL_STATUS_PAYWALL: The URL retrieval failed because the content is behind a paywall. + *
+ * URL_RETRIEVAL_STATUS_UNSAFE: The URL retrieval failed because the content is unsafe. + *
+ * + * @public + */ +export type URLRetrievalStatus = + (typeof URLRetrievalStatus)[keyof typeof URLRetrievalStatus]; + /** * @public */ From b7633db34e6f11795db793bb3c10ce09b09e5daf Mon Sep 17 00:00:00 2001 From: Daniel La Rocque Date: Tue, 16 Sep 2025 11:53:59 -0400 Subject: [PATCH 2/9] Fix API review conflict --- common/api-review/ai.api.md | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/common/api-review/ai.api.md b/common/api-review/ai.api.md index eec3a9fd095..3aad5eda94c 100644 --- a/common/api-review/ai.api.md +++ b/common/api-review/ai.api.md @@ -1305,13 +1305,7 @@ export interface ThinkingConfig { } // @public -<<<<<<< HEAD -export type Tool = FunctionDeclarationsTool | GoogleSearchTool | URLContextTool; -||||||| a4848b401 -export type Tool = FunctionDeclarationsTool | GoogleSearchTool; -======= -export type Tool = FunctionDeclarationsTool | GoogleSearchTool | CodeExecutionTool; ->>>>>>> main +export type Tool = FunctionDeclarationsTool | GoogleSearchTool | CodeExecutionTool | URLContextTool; // @public export interface ToolConfig { From 286c9c617cd44c58d02194bca51989df8567ad45 Mon Sep 17 00:00:00 2001 From: Daniel La Rocque Date: Tue, 16 Sep 2025 11:56:08 -0400 Subject: [PATCH 3/9] Fix merge conflict in test --- packages/ai/integration/generate-content.test.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/packages/ai/integration/generate-content.test.ts b/packages/ai/integration/generate-content.test.ts index 65d53e3aa09..ffb1ecca698 100644 --- a/packages/ai/integration/generate-content.test.ts +++ b/packages/ai/integration/generate-content.test.ts @@ -193,7 +193,6 @@ describe('Generate Content', function () { }); }); -<<<<<<< HEAD describe('URL Context', async () => { // URL Context is not supported in Google AI for gemini-2.0-flash if ( @@ -318,8 +317,8 @@ describe('Generate Content', function () { expect(groundingMetadata?.groundingChunks).to.exist; } }); -||||||| a4848b401 -======= + }); + it('generateContent: code execution', async () => { const model = getGenerativeModel(testConfig.ai, { model: testConfig.model, @@ -344,7 +343,6 @@ describe('Generate Content', function () { .true; expect(parts?.some(part => part.codeExecutionResult?.output != null)).to .be.true; ->>>>>>> main }); it('generateContentStream: text input, text output', async () => { From bbb9d4a6961c47f04e32795679459b5b19ca2449 Mon Sep 17 00:00:00 2001 From: Daniel La Rocque Date: Tue, 16 Sep 2025 11:58:59 -0400 Subject: [PATCH 4/9] fix docs --- docs-devsite/ai.md | 10 ++-------- docs-devsite/ai.urlcontexttool.md | 6 ++++-- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/docs-devsite/ai.md b/docs-devsite/ai.md index 6291c392593..e011ad17754 100644 --- a/docs-devsite/ai.md +++ b/docs-devsite/ai.md @@ -136,7 +136,7 @@ The Firebase AI Web SDK. | [ToolConfig](./ai.toolconfig.md#toolconfig_interface) | Tool config. This config is shared for all tools provided in the request. | | [URLContext](./ai.urlcontext.md#urlcontext_interface) | Specifies the URL Context configuration. | | [URLContextMetadata](./ai.urlcontextmetadata.md#urlcontextmetadata_interface) | Metadata related to [URLContextTool](./ai.urlcontexttool.md#urlcontexttool_interface). | -| [URLContextTool](./ai.urlcontexttool.md#urlcontexttool_interface) | A tool that allows you to provide additional context to the models in the form of URLs. By including URLs in your request, the Gemini model will access the content from those pages to inform and enhances it's response. At most 20 URLs can be consumed per request. | +| [URLContextTool](./ai.urlcontexttool.md#urlcontexttool_interface) | A tool that allows you to provide additional context to the models in the form of URLs. By including URLs in your request, the Gemini model will access the content from those pages to inform and enhance its response. | | [URLMetadata](./ai.urlmetadata.md#urlmetadata_interface) | Metadata for a URL that was used to provide context to the Gemini model. | | [UsageMetadata](./ai.usagemetadata.md#usagemetadata_interface) | Usage metadata about a [GenerateContentResponse](./ai.generatecontentresponse.md#generatecontentresponse_interface). | | [VideoMetadata](./ai.videometadata.md#videometadata_interface) | Describes the input video content. | @@ -1053,13 +1053,7 @@ Defines a tool that model can call to access external knowledge. Signature: ```typescript -<<<<<<< HEAD -export type Tool = FunctionDeclarationsTool | GoogleSearchTool | URLContextTool; -||||||| a4848b401 -export type Tool = FunctionDeclarationsTool | GoogleSearchTool; -======= -export type Tool = FunctionDeclarationsTool | GoogleSearchTool | CodeExecutionTool; ->>>>>>> main +export type Tool = FunctionDeclarationsTool | GoogleSearchTool | CodeExecutionTool | URLContextTool; ``` ## TypedSchema diff --git a/docs-devsite/ai.urlcontexttool.md b/docs-devsite/ai.urlcontexttool.md index bb9087ab4b8..212ccb5286c 100644 --- a/docs-devsite/ai.urlcontexttool.md +++ b/docs-devsite/ai.urlcontexttool.md @@ -10,7 +10,7 @@ https://github.com/firebase/firebase-js-sdk {% endcomment %} # URLContextTool interface -A tool that allows you to provide additional context to the models in the form of URLs. By including URLs in your request, the Gemini model will access the content from those pages to inform and enhances it's response. At most 20 URLs can be consumed per request. +A tool that allows you to provide additional context to the models in the form of URLs. By including URLs in your request, the Gemini model will access the content from those pages to inform and enhance its response. Signature: @@ -22,10 +22,12 @@ export interface URLContextTool | Property | Type | Description | | --- | --- | --- | -| [urlContext](./ai.urlcontexttool.md#urlcontexttoolurlcontext) | [URLContext](./ai.urlcontext.md#urlcontext_interface) | | +| [urlContext](./ai.urlcontexttool.md#urlcontexttoolurlcontext) | [URLContext](./ai.urlcontext.md#urlcontext_interface) | Specifies the URL Context configuration. | ## URLContextTool.urlContext +Specifies the URL Context configuration. + Signature: ```typescript From 58d4cad81589077226cba2085d66b4143eab2c08 Mon Sep 17 00:00:00 2001 From: Daniel La Rocque Date: Tue, 16 Sep 2025 13:10:24 -0400 Subject: [PATCH 5/9] Fix stream overwrite issue --- packages/ai/src/requests/stream-reader.ts | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/packages/ai/src/requests/stream-reader.ts b/packages/ai/src/requests/stream-reader.ts index de64c37522a..5ff76d75250 100644 --- a/packages/ai/src/requests/stream-reader.ts +++ b/packages/ai/src/requests/stream-reader.ts @@ -28,7 +28,7 @@ import { createEnhancedContentResponse } from './response-helpers'; import * as GoogleAIMapper from '../googleai-mappers'; import { GoogleAIGenerateContentResponse } from '../types/googleai'; import { ApiSettings } from '../types/internal'; -import { BackendType } from '../public-types'; +import { BackendType, URLContextMetadata } from '../public-types'; const responseLineRE = /^data\: (.*)(?:\n\n|\r\r|\r\n\r\n)/; @@ -192,8 +192,15 @@ export function aggregateResponses( candidate.safetyRatings; aggregatedResponse.candidates[i].groundingMetadata = candidate.groundingMetadata; - aggregatedResponse.candidates[i].urlContextMetadata = - candidate.urlContextMetadata; + + // The urlContextMetadata object is defined in the first chunk of the response stream. + // In all subsequent chunks, the urlContextMetadata object will be undefined. We need to + // make sure that we don't overwrite the first value urlContextMetadata object with undefined. + // FIXME: What happens if we receive a second, valid urlContextMetadata object? + const urlContextMetadata = candidate.urlContextMetadata as unknown; + if (typeof urlContextMetadata === 'object' && urlContextMetadata !== null && Object.keys(urlContextMetadata).length > 0) { + aggregatedResponse.candidates[i].urlContextMetadata = urlContextMetadata as URLContextMetadata; + } /** * Candidates should always have content and parts, but this handles From 05633c82223627efbcadc6ed2e62afba7934ee2a Mon Sep 17 00:00:00 2001 From: Daniel La Rocque Date: Wed, 17 Sep 2025 16:33:47 -0400 Subject: [PATCH 6/9] update changeset --- .changeset/poor-rings-admire.md | 1 + 1 file changed, 1 insertion(+) diff --git a/.changeset/poor-rings-admire.md b/.changeset/poor-rings-admire.md index c6b3b7dd029..c491e39f68d 100644 --- a/.changeset/poor-rings-admire.md +++ b/.changeset/poor-rings-admire.md @@ -1,4 +1,5 @@ --- +'firebase': minor '@firebase/ai': minor --- From 45a014f1e0460abf129d0be7fe32d79448b77d3b Mon Sep 17 00:00:00 2001 From: Daniel La Rocque Date: Mon, 22 Sep 2025 11:25:02 -0400 Subject: [PATCH 7/9] Update docs --- docs-devsite/ai.md | 8 +++++--- docs-devsite/ai.urlcontext.md | 2 -- docs-devsite/ai.urlcontextmetadata.md | 4 ++-- docs-devsite/ai.urlcontexttool.md | 2 +- docs-devsite/ai.urlmetadata.md | 2 +- docs-devsite/ai.usagemetadata.md | 8 ++++---- packages/ai/src/requests/stream-reader.ts | 9 ++++++-- packages/ai/src/types/requests.ts | 8 +++----- packages/ai/src/types/responses.ts | 25 ++++++++++++++++------- 9 files changed, 41 insertions(+), 27 deletions(-) diff --git a/docs-devsite/ai.md b/docs-devsite/ai.md index e011ad17754..7163fa9f71a 100644 --- a/docs-devsite/ai.md +++ b/docs-devsite/ai.md @@ -136,8 +136,8 @@ The Firebase AI Web SDK. | [ToolConfig](./ai.toolconfig.md#toolconfig_interface) | Tool config. This config is shared for all tools provided in the request. | | [URLContext](./ai.urlcontext.md#urlcontext_interface) | Specifies the URL Context configuration. | | [URLContextMetadata](./ai.urlcontextmetadata.md#urlcontextmetadata_interface) | Metadata related to [URLContextTool](./ai.urlcontexttool.md#urlcontexttool_interface). | -| [URLContextTool](./ai.urlcontexttool.md#urlcontexttool_interface) | A tool that allows you to provide additional context to the models in the form of URLs. By including URLs in your request, the Gemini model will access the content from those pages to inform and enhance its response. | -| [URLMetadata](./ai.urlmetadata.md#urlmetadata_interface) | Metadata for a URL that was used to provide context to the Gemini model. | +| [URLContextTool](./ai.urlcontexttool.md#urlcontexttool_interface) | A tool that allows you to provide additional context to the models in the form of public web URLs. By including URLs in your request, the Gemini model will access the content from those pages to inform and enhance its response. | +| [URLMetadata](./ai.urlmetadata.md#urlmetadata_interface) | Metadata for a single URL retrieved by the [URLContextTool](./ai.urlcontexttool.md#urlcontexttool_interface) tool. | | [UsageMetadata](./ai.usagemetadata.md#usagemetadata_interface) | Usage metadata about a [GenerateContentResponse](./ai.generatecontentresponse.md#generatecontentresponse_interface). | | [VideoMetadata](./ai.videometadata.md#videometadata_interface) | Describes the input video content. | | [VoiceConfig](./ai.voiceconfig.md#voiceconfig_interface) | (Public Preview) Configuration for the voice to used in speech synthesis. | @@ -762,6 +762,8 @@ SchemaType: { The status of a URL retrieval. +URL\_RETRIEVAL\_STATUS\_UNSPECIFIED: Unspecified retrieval status.
URL\_RETRIEVAL\_STATUS\_SUCCESS: The URL retrieval was successful.
URL\_RETRIEVAL\_STATUS\_ERROR: The URL retrieval failed.
URL\_RETRIEVAL\_STATUS\_PAYWALL: The URL retrieval failed because the content is behind a paywall.
URL\_RETRIEVAL\_STATUS\_UNSAFE: The URL retrieval failed because the content is unsafe.
+ Signature: ```typescript @@ -1070,7 +1072,7 @@ export type TypedSchema = IntegerSchema | NumberSchema | StringSchema | BooleanS The status of a URL retrieval. -URL\_RETRIEVAL\_STATUS\_UNSPECIFIED: Unspecified retrieval status.
URL\_RETRIEVAL\_STATUS\_SUCCESS: The URL retrieval was successful.
URL\_RETRIEVAL\_STATUS\_ERROR: The URL retrieval failed due to an error.
URL\_RETRIEVAL\_STATUS\_PAYWALL: The URL retrieval failed because the content is behind a paywall.
URL\_RETRIEVAL\_STATUS\_UNSAFE: The URL retrieval failed because the content is unsafe.
+URL\_RETRIEVAL\_STATUS\_UNSPECIFIED: Unspecified retrieval status.
URL\_RETRIEVAL\_STATUS\_SUCCESS: The URL retrieval was successful.
URL\_RETRIEVAL\_STATUS\_ERROR: The URL retrieval failed.
URL\_RETRIEVAL\_STATUS\_PAYWALL: The URL retrieval failed because the content is behind a paywall.
URL\_RETRIEVAL\_STATUS\_UNSAFE: The URL retrieval failed because the content is unsafe.
Signature: diff --git a/docs-devsite/ai.urlcontext.md b/docs-devsite/ai.urlcontext.md index 02dbcd49f52..7fec0481e65 100644 --- a/docs-devsite/ai.urlcontext.md +++ b/docs-devsite/ai.urlcontext.md @@ -12,8 +12,6 @@ https://github.com/firebase/firebase-js-sdk # URLContext interface Specifies the URL Context configuration. -Currently, this is an empty object, but it's reserved for future configuration. - Signature: ```typescript diff --git a/docs-devsite/ai.urlcontextmetadata.md b/docs-devsite/ai.urlcontextmetadata.md index 4155c1e9a6a..dcbf34103eb 100644 --- a/docs-devsite/ai.urlcontextmetadata.md +++ b/docs-devsite/ai.urlcontextmetadata.md @@ -22,11 +22,11 @@ export interface URLContextMetadata | Property | Type | Description | | --- | --- | --- | -| [urlMetadata](./ai.urlcontextmetadata.md#urlcontextmetadataurlmetadata) | [URLMetadata](./ai.urlmetadata.md#urlmetadata_interface)\[\] | List of URL metadata were used to provide context to the Gemini model. | +| [urlMetadata](./ai.urlcontextmetadata.md#urlcontextmetadataurlmetadata) | [URLMetadata](./ai.urlmetadata.md#urlmetadata_interface)\[\] | List of URL metadata used to provide context to the Gemini model. | ## URLContextMetadata.urlMetadata -List of URL metadata were used to provide context to the Gemini model. +List of URL metadata used to provide context to the Gemini model. Signature: diff --git a/docs-devsite/ai.urlcontexttool.md b/docs-devsite/ai.urlcontexttool.md index 212ccb5286c..cbe94d45b76 100644 --- a/docs-devsite/ai.urlcontexttool.md +++ b/docs-devsite/ai.urlcontexttool.md @@ -10,7 +10,7 @@ https://github.com/firebase/firebase-js-sdk {% endcomment %} # URLContextTool interface -A tool that allows you to provide additional context to the models in the form of URLs. By including URLs in your request, the Gemini model will access the content from those pages to inform and enhance its response. +A tool that allows you to provide additional context to the models in the form of public web URLs. By including URLs in your request, the Gemini model will access the content from those pages to inform and enhance its response. Signature: diff --git a/docs-devsite/ai.urlmetadata.md b/docs-devsite/ai.urlmetadata.md index 8e4aec5333f..c60a32db83e 100644 --- a/docs-devsite/ai.urlmetadata.md +++ b/docs-devsite/ai.urlmetadata.md @@ -10,7 +10,7 @@ https://github.com/firebase/firebase-js-sdk {% endcomment %} # URLMetadata interface -Metadata for a URL that was used to provide context to the Gemini model. +Metadata for a single URL retrieved by the [URLContextTool](./ai.urlcontexttool.md#urlcontexttool_interface) tool. Signature: diff --git a/docs-devsite/ai.usagemetadata.md b/docs-devsite/ai.usagemetadata.md index c82593b978a..bf45610f4a1 100644 --- a/docs-devsite/ai.usagemetadata.md +++ b/docs-devsite/ai.usagemetadata.md @@ -27,8 +27,8 @@ export interface UsageMetadata | [promptTokenCount](./ai.usagemetadata.md#usagemetadataprompttokencount) | number | | | [promptTokensDetails](./ai.usagemetadata.md#usagemetadataprompttokensdetails) | [ModalityTokenCount](./ai.modalitytokencount.md#modalitytokencount_interface)\[\] | | | [thoughtsTokenCount](./ai.usagemetadata.md#usagemetadatathoughtstokencount) | number | The number of tokens used by the model's internal "thinking" process. | -| [toolUsePromptTokenCount](./ai.usagemetadata.md#usagemetadatatooluseprompttokencount) | number | The number of tokens in the results from tool executions, which are provided back to the model as input. | -| [toolUsePromptTokensDetails](./ai.usagemetadata.md#usagemetadatatooluseprompttokensdetails) | [ModalityTokenCount](./ai.modalitytokencount.md#modalitytokencount_interface)\[\] | A list of tokens used by tools whose usage was triggered from a prompt, broken down by modality. | +| [toolUsePromptTokenCount](./ai.usagemetadata.md#usagemetadatatooluseprompttokencount) | number | The number of tokens used by tools. | +| [toolUsePromptTokensDetails](./ai.usagemetadata.md#usagemetadatatooluseprompttokensdetails) | [ModalityTokenCount](./ai.modalitytokencount.md#modalitytokencount_interface)\[\] | A list of tokens used by tools, broken down by modality. | | [totalTokenCount](./ai.usagemetadata.md#usagemetadatatotaltokencount) | number | | ## UsageMetadata.candidatesTokenCount @@ -75,7 +75,7 @@ thoughtsTokenCount?: number; ## UsageMetadata.toolUsePromptTokenCount -The number of tokens in the results from tool executions, which are provided back to the model as input. +The number of tokens used by tools. Signature: @@ -85,7 +85,7 @@ toolUsePromptTokenCount?: number; ## UsageMetadata.toolUsePromptTokensDetails -A list of tokens used by tools whose usage was triggered from a prompt, broken down by modality. +A list of tokens used by tools, broken down by modality. Signature: diff --git a/packages/ai/src/requests/stream-reader.ts b/packages/ai/src/requests/stream-reader.ts index 5ff76d75250..3b205efabf2 100644 --- a/packages/ai/src/requests/stream-reader.ts +++ b/packages/ai/src/requests/stream-reader.ts @@ -198,8 +198,13 @@ export function aggregateResponses( // make sure that we don't overwrite the first value urlContextMetadata object with undefined. // FIXME: What happens if we receive a second, valid urlContextMetadata object? const urlContextMetadata = candidate.urlContextMetadata as unknown; - if (typeof urlContextMetadata === 'object' && urlContextMetadata !== null && Object.keys(urlContextMetadata).length > 0) { - aggregatedResponse.candidates[i].urlContextMetadata = urlContextMetadata as URLContextMetadata; + if ( + typeof urlContextMetadata === 'object' && + urlContextMetadata !== null && + Object.keys(urlContextMetadata).length > 0 + ) { + aggregatedResponse.candidates[i].urlContextMetadata = + urlContextMetadata as URLContextMetadata; } /** diff --git a/packages/ai/src/types/requests.ts b/packages/ai/src/types/requests.ts index 3055a493cf8..c1bbe1e22d7 100644 --- a/packages/ai/src/types/requests.ts +++ b/packages/ai/src/types/requests.ts @@ -321,9 +321,9 @@ export interface CodeExecutionTool { export interface GoogleSearch {} /** - * A tool that allows you to provide additional context to the models in the form of URLs. - * By including URLs in your request, the Gemini model will access the content from those pages - * to inform and enhance its response. + * A tool that allows you to provide additional context to the models in the form of public web + * URLs. By including URLs in your request, the Gemini model will access the content from those + * pages to inform and enhance its response. * * @public */ @@ -337,8 +337,6 @@ export interface URLContextTool { /** * Specifies the URL Context configuration. * - * @remarks Currently, this is an empty object, but it's reserved for future configuration. - * * @public */ export interface URLContext {} diff --git a/packages/ai/src/types/responses.ts b/packages/ai/src/types/responses.ts index ec03d2e201f..1fa6da3cb3f 100644 --- a/packages/ai/src/types/responses.ts +++ b/packages/ai/src/types/responses.ts @@ -117,14 +117,13 @@ export interface UsageMetadata { thoughtsTokenCount?: number; totalTokenCount: number; /** - * The number of tokens in the results from tool executions, which are provided back to the - * model as input. + * The number of tokens used by tools. */ toolUsePromptTokenCount?: number; promptTokensDetails?: ModalityTokenCount[]; candidatesTokensDetails?: ModalityTokenCount[]; /** - * A list of tokens used by tools whose usage was triggered from a prompt, broken down by modality. + * A list of tokens used by tools, broken down by modality. */ toolUsePromptTokensDetails?: ModalityTokenCount[]; } @@ -366,13 +365,13 @@ export interface Segment { */ export interface URLContextMetadata { /** - * List of URL metadata were used to provide context to the Gemini model. + * List of URL metadata used to provide context to the Gemini model. */ urlMetadata: URLMetadata[]; } /** - * Metadata for a URL that was used to provide context to the Gemini model. + * Metadata for a single URL retrieved by the {@link URLContextTool} tool. * * @public */ @@ -390,6 +389,18 @@ export interface URLMetadata { /** * The status of a URL retrieval. * + * @remarks + * URL_RETRIEVAL_STATUS_UNSPECIFIED: Unspecified retrieval status. + *
+ * URL_RETRIEVAL_STATUS_SUCCESS: The URL retrieval was successful. + *
+ * URL_RETRIEVAL_STATUS_ERROR: The URL retrieval failed. + *
+ * URL_RETRIEVAL_STATUS_PAYWALL: The URL retrieval failed because the content is behind a paywall. + *
+ * URL_RETRIEVAL_STATUS_UNSAFE: The URL retrieval failed because the content is unsafe. + *
+ * * @public */ export const URLRetrievalStatus = { @@ -402,7 +413,7 @@ export const URLRetrievalStatus = { */ URL_RETRIEVAL_STATUS_SUCCESS: 'URL_RETRIEVAL_STATUS_SUCCESS', /** - * The URL retrieval failed due to an error. + * The URL retrieval failed. */ URL_RETRIEVAL_STATUS_ERROR: 'URL_RETRIEVAL_STATUS_ERROR', /** @@ -423,7 +434,7 @@ export const URLRetrievalStatus = { *
* URL_RETRIEVAL_STATUS_SUCCESS: The URL retrieval was successful. *
- * URL_RETRIEVAL_STATUS_ERROR: The URL retrieval failed due to an error. + * URL_RETRIEVAL_STATUS_ERROR: The URL retrieval failed. *
* URL_RETRIEVAL_STATUS_PAYWALL: The URL retrieval failed because the content is behind a paywall. *
From 6e7753a7fcd40dd66ece50937682f94544ed7988 Mon Sep 17 00:00:00 2001 From: Daniel La Rocque Date: Tue, 23 Sep 2025 18:15:46 -0400 Subject: [PATCH 8/9] update changeset --- .changeset/poor-rings-admire.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/poor-rings-admire.md b/.changeset/poor-rings-admire.md index c491e39f68d..1b63c0138d0 100644 --- a/.changeset/poor-rings-admire.md +++ b/.changeset/poor-rings-admire.md @@ -3,4 +3,4 @@ '@firebase/ai': minor --- -Add support for the URL Context tool. +Added support for the URL context tool, which allows the model to access content from provided public web URLs to inform and enhance its responses. From 13b4b6e6c8d7117faaa16750e3ac25f459d821a5 Mon Sep 17 00:00:00 2001 From: Daniel La Rocque Date: Wed, 24 Sep 2025 10:29:40 -0400 Subject: [PATCH 9/9] Add public preview annotations --- common/api-review/ai.api.md | 16 ++++++++++------ docs-devsite/ai.md | 18 ++++++++++++------ docs-devsite/ai.urlcontext.md | 3 +++ docs-devsite/ai.urlcontextmetadata.md | 8 +++++++- docs-devsite/ai.urlcontexttool.md | 8 +++++++- docs-devsite/ai.urlmetadata.md | 13 +++++++++++-- packages/ai/src/types/requests.ts | 4 ++-- packages/ai/src/types/responses.ts | 8 ++++---- 8 files changed, 56 insertions(+), 22 deletions(-) diff --git a/common/api-review/ai.api.md b/common/api-review/ai.api.md index 3aad5eda94c..a9187215465 100644 --- a/common/api-review/ai.api.md +++ b/common/api-review/ai.api.md @@ -448,6 +448,8 @@ export interface GenerateContentCandidate { index: number; // (undocumented) safetyRatings?: SafetyRating[]; + // Warning: (ae-incompatible-release-tags) The symbol "urlContextMetadata" is marked as @public, but its signature references "URLContextMetadata" which is marked as @beta + // // (undocumented) urlContextMetadata?: URLContextMetadata; } @@ -1304,6 +1306,8 @@ export interface ThinkingConfig { thinkingBudget?: number; } +// Warning: (ae-incompatible-release-tags) The symbol "Tool" is marked as @public, but its signature references "URLContextTool" which is marked as @beta +// // @public export type Tool = FunctionDeclarationsTool | GoogleSearchTool | CodeExecutionTool | URLContextTool; @@ -1316,27 +1320,27 @@ export interface ToolConfig { // @public export type TypedSchema = IntegerSchema | NumberSchema | StringSchema | BooleanSchema | ObjectSchema | ArraySchema | AnyOfSchema; -// @public +// @beta export interface URLContext { } -// @public +// @beta export interface URLContextMetadata { urlMetadata: URLMetadata[]; } -// @public +// @beta export interface URLContextTool { urlContext: URLContext; } -// @public +// @beta export interface URLMetadata { retrievedUrl?: string; urlRetrievalStatus?: URLRetrievalStatus; } -// @public +// @beta export const URLRetrievalStatus: { URL_RETRIEVAL_STATUS_UNSPECIFIED: string; URL_RETRIEVAL_STATUS_SUCCESS: string; @@ -1345,7 +1349,7 @@ export const URLRetrievalStatus: { URL_RETRIEVAL_STATUS_UNSAFE: string; }; -// @public +// @beta export type URLRetrievalStatus = (typeof URLRetrievalStatus)[keyof typeof URLRetrievalStatus]; // @public diff --git a/docs-devsite/ai.md b/docs-devsite/ai.md index 7163fa9f71a..e47d35f3a24 100644 --- a/docs-devsite/ai.md +++ b/docs-devsite/ai.md @@ -134,10 +134,10 @@ The Firebase AI Web SDK. | [TextPart](./ai.textpart.md#textpart_interface) | Content part interface if the part represents a text string. | | [ThinkingConfig](./ai.thinkingconfig.md#thinkingconfig_interface) | Configuration for "thinking" behavior of compatible Gemini models.Certain models utilize a thinking process before generating a response. This allows them to reason through complex problems and plan a more coherent and accurate answer. | | [ToolConfig](./ai.toolconfig.md#toolconfig_interface) | Tool config. This config is shared for all tools provided in the request. | -| [URLContext](./ai.urlcontext.md#urlcontext_interface) | Specifies the URL Context configuration. | -| [URLContextMetadata](./ai.urlcontextmetadata.md#urlcontextmetadata_interface) | Metadata related to [URLContextTool](./ai.urlcontexttool.md#urlcontexttool_interface). | -| [URLContextTool](./ai.urlcontexttool.md#urlcontexttool_interface) | A tool that allows you to provide additional context to the models in the form of public web URLs. By including URLs in your request, the Gemini model will access the content from those pages to inform and enhance its response. | -| [URLMetadata](./ai.urlmetadata.md#urlmetadata_interface) | Metadata for a single URL retrieved by the [URLContextTool](./ai.urlcontexttool.md#urlcontexttool_interface) tool. | +| [URLContext](./ai.urlcontext.md#urlcontext_interface) | (Public Preview) Specifies the URL Context configuration. | +| [URLContextMetadata](./ai.urlcontextmetadata.md#urlcontextmetadata_interface) | (Public Preview) Metadata related to [URLContextTool](./ai.urlcontexttool.md#urlcontexttool_interface). | +| [URLContextTool](./ai.urlcontexttool.md#urlcontexttool_interface) | (Public Preview) A tool that allows you to provide additional context to the models in the form of public web URLs. By including URLs in your request, the Gemini model will access the content from those pages to inform and enhance its response. | +| [URLMetadata](./ai.urlmetadata.md#urlmetadata_interface) | (Public Preview) Metadata for a single URL retrieved by the [URLContextTool](./ai.urlcontexttool.md#urlcontexttool_interface) tool. | | [UsageMetadata](./ai.usagemetadata.md#usagemetadata_interface) | Usage metadata about a [GenerateContentResponse](./ai.generatecontentresponse.md#generatecontentresponse_interface). | | [VideoMetadata](./ai.videometadata.md#videometadata_interface) | Describes the input video content. | | [VoiceConfig](./ai.voiceconfig.md#voiceconfig_interface) | (Public Preview) Configuration for the voice to used in speech synthesis. | @@ -169,7 +169,7 @@ The Firebase AI Web SDK. | [POSSIBLE\_ROLES](./ai.md#possible_roles) | Possible roles. | | [ResponseModality](./ai.md#responsemodality) | (Public Preview) Generation modalities to be returned in generation responses. | | [SchemaType](./ai.md#schematype) | Contains the list of OpenAPI data types as defined by the [OpenAPI specification](https://swagger.io/docs/specification/data-models/data-types/) | -| [URLRetrievalStatus](./ai.md#urlretrievalstatus) | The status of a URL retrieval. | +| [URLRetrievalStatus](./ai.md#urlretrievalstatus) | (Public Preview) The status of a URL retrieval. | ## Type Aliases @@ -202,7 +202,7 @@ The Firebase AI Web SDK. | [SchemaType](./ai.md#schematype) | Contains the list of OpenAPI data types as defined by the [OpenAPI specification](https://swagger.io/docs/specification/data-models/data-types/) | | [Tool](./ai.md#tool) | Defines a tool that model can call to access external knowledge. | | [TypedSchema](./ai.md#typedschema) | A type that includes all specific Schema types. | -| [URLRetrievalStatus](./ai.md#urlretrievalstatus) | The status of a URL retrieval. | +| [URLRetrievalStatus](./ai.md#urlretrievalstatus) | (Public Preview) The status of a URL retrieval. | ## function(app, ...) @@ -760,6 +760,9 @@ SchemaType: { ## URLRetrievalStatus +> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment. +> + The status of a URL retrieval. URL\_RETRIEVAL\_STATUS\_UNSPECIFIED: Unspecified retrieval status.
URL\_RETRIEVAL\_STATUS\_SUCCESS: The URL retrieval was successful.
URL\_RETRIEVAL\_STATUS\_ERROR: The URL retrieval failed.
URL\_RETRIEVAL\_STATUS\_PAYWALL: The URL retrieval failed because the content is behind a paywall.
URL\_RETRIEVAL\_STATUS\_UNSAFE: The URL retrieval failed because the content is unsafe.
@@ -1070,6 +1073,9 @@ export type TypedSchema = IntegerSchema | NumberSchema | StringSchema | BooleanS ## URLRetrievalStatus +> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment. +> + The status of a URL retrieval. URL\_RETRIEVAL\_STATUS\_UNSPECIFIED: Unspecified retrieval status.
URL\_RETRIEVAL\_STATUS\_SUCCESS: The URL retrieval was successful.
URL\_RETRIEVAL\_STATUS\_ERROR: The URL retrieval failed.
URL\_RETRIEVAL\_STATUS\_PAYWALL: The URL retrieval failed because the content is behind a paywall.
URL\_RETRIEVAL\_STATUS\_UNSAFE: The URL retrieval failed because the content is unsafe.
diff --git a/docs-devsite/ai.urlcontext.md b/docs-devsite/ai.urlcontext.md index 7fec0481e65..435d278e4d1 100644 --- a/docs-devsite/ai.urlcontext.md +++ b/docs-devsite/ai.urlcontext.md @@ -10,6 +10,9 @@ https://github.com/firebase/firebase-js-sdk {% endcomment %} # URLContext interface +> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment. +> + Specifies the URL Context configuration. Signature: diff --git a/docs-devsite/ai.urlcontextmetadata.md b/docs-devsite/ai.urlcontextmetadata.md index dcbf34103eb..bc260b997ad 100644 --- a/docs-devsite/ai.urlcontextmetadata.md +++ b/docs-devsite/ai.urlcontextmetadata.md @@ -10,6 +10,9 @@ https://github.com/firebase/firebase-js-sdk {% endcomment %} # URLContextMetadata interface +> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment. +> + Metadata related to [URLContextTool](./ai.urlcontexttool.md#urlcontexttool_interface). Signature: @@ -22,10 +25,13 @@ export interface URLContextMetadata | Property | Type | Description | | --- | --- | --- | -| [urlMetadata](./ai.urlcontextmetadata.md#urlcontextmetadataurlmetadata) | [URLMetadata](./ai.urlmetadata.md#urlmetadata_interface)\[\] | List of URL metadata used to provide context to the Gemini model. | +| [urlMetadata](./ai.urlcontextmetadata.md#urlcontextmetadataurlmetadata) | [URLMetadata](./ai.urlmetadata.md#urlmetadata_interface)\[\] | (Public Preview) List of URL metadata used to provide context to the Gemini model. | ## URLContextMetadata.urlMetadata +> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment. +> + List of URL metadata used to provide context to the Gemini model. Signature: diff --git a/docs-devsite/ai.urlcontexttool.md b/docs-devsite/ai.urlcontexttool.md index cbe94d45b76..6ecc2a323c1 100644 --- a/docs-devsite/ai.urlcontexttool.md +++ b/docs-devsite/ai.urlcontexttool.md @@ -10,6 +10,9 @@ https://github.com/firebase/firebase-js-sdk {% endcomment %} # URLContextTool interface +> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment. +> + A tool that allows you to provide additional context to the models in the form of public web URLs. By including URLs in your request, the Gemini model will access the content from those pages to inform and enhance its response. Signature: @@ -22,10 +25,13 @@ export interface URLContextTool | Property | Type | Description | | --- | --- | --- | -| [urlContext](./ai.urlcontexttool.md#urlcontexttoolurlcontext) | [URLContext](./ai.urlcontext.md#urlcontext_interface) | Specifies the URL Context configuration. | +| [urlContext](./ai.urlcontexttool.md#urlcontexttoolurlcontext) | [URLContext](./ai.urlcontext.md#urlcontext_interface) | (Public Preview) Specifies the URL Context configuration. | ## URLContextTool.urlContext +> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment. +> + Specifies the URL Context configuration. Signature: diff --git a/docs-devsite/ai.urlmetadata.md b/docs-devsite/ai.urlmetadata.md index c60a32db83e..3cbd27632d0 100644 --- a/docs-devsite/ai.urlmetadata.md +++ b/docs-devsite/ai.urlmetadata.md @@ -10,6 +10,9 @@ https://github.com/firebase/firebase-js-sdk {% endcomment %} # URLMetadata interface +> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment. +> + Metadata for a single URL retrieved by the [URLContextTool](./ai.urlcontexttool.md#urlcontexttool_interface) tool. Signature: @@ -22,11 +25,14 @@ export interface URLMetadata | Property | Type | Description | | --- | --- | --- | -| [retrievedUrl](./ai.urlmetadata.md#urlmetadataretrievedurl) | string | The retrieved URL. | -| [urlRetrievalStatus](./ai.urlmetadata.md#urlmetadataurlretrievalstatus) | [URLRetrievalStatus](./ai.md#urlretrievalstatus) | The status of the URL retrieval. | +| [retrievedUrl](./ai.urlmetadata.md#urlmetadataretrievedurl) | string | (Public Preview) The retrieved URL. | +| [urlRetrievalStatus](./ai.urlmetadata.md#urlmetadataurlretrievalstatus) | [URLRetrievalStatus](./ai.md#urlretrievalstatus) | (Public Preview) The status of the URL retrieval. | ## URLMetadata.retrievedUrl +> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment. +> + The retrieved URL. Signature: @@ -37,6 +43,9 @@ retrievedUrl?: string; ## URLMetadata.urlRetrievalStatus +> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment. +> + The status of the URL retrieval. Signature: diff --git a/packages/ai/src/types/requests.ts b/packages/ai/src/types/requests.ts index c1bbe1e22d7..143d772a506 100644 --- a/packages/ai/src/types/requests.ts +++ b/packages/ai/src/types/requests.ts @@ -325,7 +325,7 @@ export interface GoogleSearch {} * URLs. By including URLs in your request, the Gemini model will access the content from those * pages to inform and enhance its response. * - * @public + * @beta */ export interface URLContextTool { /** @@ -337,7 +337,7 @@ export interface URLContextTool { /** * Specifies the URL Context configuration. * - * @public + * @beta */ export interface URLContext {} diff --git a/packages/ai/src/types/responses.ts b/packages/ai/src/types/responses.ts index 1fa6da3cb3f..8b8e1351675 100644 --- a/packages/ai/src/types/responses.ts +++ b/packages/ai/src/types/responses.ts @@ -361,7 +361,7 @@ export interface Segment { /** * Metadata related to {@link URLContextTool}. * - * @public + * @beta */ export interface URLContextMetadata { /** @@ -373,7 +373,7 @@ export interface URLContextMetadata { /** * Metadata for a single URL retrieved by the {@link URLContextTool} tool. * - * @public + * @beta */ export interface URLMetadata { /** @@ -401,7 +401,7 @@ export interface URLMetadata { * URL_RETRIEVAL_STATUS_UNSAFE: The URL retrieval failed because the content is unsafe. *
* - * @public + * @beta */ export const URLRetrievalStatus = { /** @@ -441,7 +441,7 @@ export const URLRetrievalStatus = { * URL_RETRIEVAL_STATUS_UNSAFE: The URL retrieval failed because the content is unsafe. *
* - * @public + * @beta */ export type URLRetrievalStatus = (typeof URLRetrievalStatus)[keyof typeof URLRetrievalStatus];