Skip to content

Commit 0ffcb26

Browse files
dlarocquedlarocque
authored
feat(ai): add support for URL context (#9254)
1 parent 1bcf83d commit 0ffcb26

17 files changed

+615
-11
lines changed

‎.changeset/poor-rings-admire.md‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
---
2+
'firebase': minor
3+
'@firebase/ai': minor
4+
---
5+
6+
Added support for the URL context tool, which allows the model to access content from provided public web URLs to inform and enhance its responses.

‎common/api-review/ai.api.md‎

Lines changed: 43 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -449,6 +449,10 @@ export interface GenerateContentCandidate {
449449
index: number;
450450
// (undocumented)
451451
safetyRatings?: SafetyRating[];
452+
// Warning: (ae-incompatible-release-tags) The symbol "urlContextMetadata" is marked as @public, but its signature references "URLContextMetadata" which is marked as @beta
453+
//
454+
// (undocumented)
455+
urlContextMetadata?: URLContextMetadata;
452456
}
453457

454458
// @public
@@ -600,6 +604,8 @@ export interface GoogleAIGenerateContentCandidate {
600604
index: number;
601605
// (undocumented)
602606
safetyRatings?: SafetyRating[];
607+
// (undocumented)
608+
urlContextMetadata?: URLContextMetadata;
603609
}
604610

605611
// Warning: (ae-internal-missing-underscore) The name "GoogleAIGenerateContentResponse" should be prefixed with an underscore because the declaration is marked as @internal
@@ -1304,8 +1310,10 @@ export interface ThinkingConfig {
13041310
thinkingBudget?: number;
13051311
}
13061312

1313+
// Warning: (ae-incompatible-release-tags) The symbol "Tool" is marked as @public, but its signature references "URLContextTool" which is marked as @beta
1314+
//
13071315
// @public
1308-
export type Tool = FunctionDeclarationsTool | GoogleSearchTool | CodeExecutionTool;
1316+
export type Tool = FunctionDeclarationsTool | GoogleSearchTool | CodeExecutionTool | URLContextTool;
13091317

13101318
// @public
13111319
export interface ToolConfig {
@@ -1316,6 +1324,38 @@ export interface ToolConfig {
13161324
// @public
13171325
export type TypedSchema = IntegerSchema | NumberSchema | StringSchema | BooleanSchema | ObjectSchema | ArraySchema | AnyOfSchema;
13181326

1327+
// @beta
1328+
export interface URLContext {
1329+
}
1330+
1331+
// @beta
1332+
export interface URLContextMetadata {
1333+
urlMetadata: URLMetadata[];
1334+
}
1335+
1336+
// @beta
1337+
export interface URLContextTool {
1338+
urlContext: URLContext;
1339+
}
1340+
1341+
// @beta
1342+
export interface URLMetadata {
1343+
retrievedUrl?: string;
1344+
urlRetrievalStatus?: URLRetrievalStatus;
1345+
}
1346+
1347+
// @beta
1348+
export const URLRetrievalStatus: {
1349+
URL_RETRIEVAL_STATUS_UNSPECIFIED: string;
1350+
URL_RETRIEVAL_STATUS_SUCCESS: string;
1351+
URL_RETRIEVAL_STATUS_ERROR: string;
1352+
URL_RETRIEVAL_STATUS_PAYWALL: string;
1353+
URL_RETRIEVAL_STATUS_UNSAFE: string;
1354+
};
1355+
1356+
// @beta
1357+
export type URLRetrievalStatus = (typeof URLRetrievalStatus)[keyof typeof URLRetrievalStatus];
1358+
13191359
// @public
13201360
export interface UsageMetadata {
13211361
// (undocumented)
@@ -1327,6 +1367,8 @@ export interface UsageMetadata {
13271367
// (undocumented)
13281368
promptTokensDetails?: ModalityTokenCount[];
13291369
thoughtsTokenCount?: number;
1370+
toolUsePromptTokenCount?: number;
1371+
toolUsePromptTokensDetails?: ModalityTokenCount[];
13301372
// (undocumented)
13311373
totalTokenCount: number;
13321374
}

‎docs-devsite/_toc.yaml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -202,6 +202,14 @@ toc:
202202
path: /docs/reference/js/ai.thinkingconfig.md
203203
- title: ToolConfig
204204
path: /docs/reference/js/ai.toolconfig.md
205+
- title: URLContext
206+
path: /docs/reference/js/ai.urlcontext.md
207+
- title: URLContextMetadata
208+
path: /docs/reference/js/ai.urlcontextmetadata.md
209+
- title: URLContextTool
210+
path: /docs/reference/js/ai.urlcontexttool.md
211+
- title: URLMetadata
212+
path: /docs/reference/js/ai.urlmetadata.md
205213
- title: UsageMetadata
206214
path: /docs/reference/js/ai.usagemetadata.md
207215
- title: VertexAIBackend

‎docs-devsite/ai.generatecontentcandidate.md‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ export interface GenerateContentCandidate
2929
| [groundingMetadata](./ai.generatecontentcandidate.md#generatecontentcandidategroundingmetadata) | [GroundingMetadata](./ai.groundingmetadata.md#groundingmetadata_interface) | |
3030
| [index](./ai.generatecontentcandidate.md#generatecontentcandidateindex) | number | |
3131
| [safetyRatings](./ai.generatecontentcandidate.md#generatecontentcandidatesafetyratings) | [SafetyRating](./ai.safetyrating.md#safetyrating_interface)<!-- -->\[\] | |
32+
| [urlContextMetadata](./ai.generatecontentcandidate.md#generatecontentcandidateurlcontextmetadata) | [URLContextMetadata](./ai.urlcontextmetadata.md#urlcontextmetadata_interface) | |
3233

3334
## GenerateContentCandidate.citationMetadata
3435

@@ -85,3 +86,11 @@ index: number;
8586
```typescript
8687
safetyRatings?: SafetyRating[];
8788
```
89+
90+
## GenerateContentCandidate.urlContextMetadata
91+
92+
<b>Signature:</b>
93+
94+
```typescript
95+
urlContextMetadata?: URLContextMetadata;
96+
```

‎docs-devsite/ai.md‎

Lines changed: 43 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -134,6 +134,10 @@ The Firebase AI Web SDK.
134134
| [TextPart](./ai.textpart.md#textpart_interface) | Content part interface if the part represents a text string. |
135135
| [ThinkingConfig](./ai.thinkingconfig.md#thinkingconfig_interface) | Configuration for "thinking" behavior of compatible Gemini models.<!-- -->Certain models utilize a thinking process before generating a response. This allows them to reason through complex problems and plan a more coherent and accurate answer. |
136136
| [ToolConfig](./ai.toolconfig.md#toolconfig_interface) | Tool config. This config is shared for all tools provided in the request. |
137+
| [URLContext](./ai.urlcontext.md#urlcontext_interface) | <b><i>(Public Preview)</i></b> Specifies the URL Context configuration. |
138+
| [URLContextMetadata](./ai.urlcontextmetadata.md#urlcontextmetadata_interface) | <b><i>(Public Preview)</i></b> Metadata related to [URLContextTool](./ai.urlcontexttool.md#urlcontexttool_interface)<!-- -->. |
139+
| [URLContextTool](./ai.urlcontexttool.md#urlcontexttool_interface) | <b><i>(Public Preview)</i></b> A tool that allows you to provide additional context to the models in the form of public web URLs. By including URLs in your request, the Gemini model will access the content from those pages to inform and enhance its response. |
140+
| [URLMetadata](./ai.urlmetadata.md#urlmetadata_interface) | <b><i>(Public Preview)</i></b> Metadata for a single URL retrieved by the [URLContextTool](./ai.urlcontexttool.md#urlcontexttool_interface) tool. |
137141
| [UsageMetadata](./ai.usagemetadata.md#usagemetadata_interface) | Usage metadata about a [GenerateContentResponse](./ai.generatecontentresponse.md#generatecontentresponse_interface)<!-- -->. |
138142
| [VideoMetadata](./ai.videometadata.md#videometadata_interface) | Describes the input video content. |
139143
| [VoiceConfig](./ai.voiceconfig.md#voiceconfig_interface) | <b><i>(Public Preview)</i></b> Configuration for the voice to used in speech synthesis. |
@@ -165,6 +169,7 @@ The Firebase AI Web SDK.
165169
| [POSSIBLE\_ROLES](./ai.md#possible_roles) | Possible roles. |
166170
| [ResponseModality](./ai.md#responsemodality) | <b><i>(Public Preview)</i></b> Generation modalities to be returned in generation responses. |
167171
| [SchemaType](./ai.md#schematype) | Contains the list of OpenAPI data types as defined by the [OpenAPI specification](https://swagger.io/docs/specification/data-models/data-types/) |
172+
| [URLRetrievalStatus](./ai.md#urlretrievalstatus) | <b><i>(Public Preview)</i></b> The status of a URL retrieval. |
168173

169174
## Type Aliases
170175

@@ -197,6 +202,7 @@ The Firebase AI Web SDK.
197202
| [SchemaType](./ai.md#schematype) | Contains the list of OpenAPI data types as defined by the [OpenAPI specification](https://swagger.io/docs/specification/data-models/data-types/) |
198203
| [Tool](./ai.md#tool) | Defines a tool that model can call to access external knowledge. |
199204
| [TypedSchema](./ai.md#typedschema) | A type that includes all specific Schema types. |
205+
| [URLRetrievalStatus](./ai.md#urlretrievalstatus) | <b><i>(Public Preview)</i></b> The status of a URL retrieval. |
200206

201207
## function(app, ...)
202208

@@ -755,6 +761,27 @@ SchemaType: {
755761
}
756762
```
757763

764+
## URLRetrievalStatus
765+
766+
> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment.
767+
>
768+
769+
The status of a URL retrieval.
770+
771+
<b>URL\_RETRIEVAL\_STATUS\_UNSPECIFIED:</b> Unspecified retrieval status. <br/> <b>URL\_RETRIEVAL\_STATUS\_SUCCESS:</b> The URL retrieval was successful. <br/> <b>URL\_RETRIEVAL\_STATUS\_ERROR:</b> The URL retrieval failed. <br/> <b>URL\_RETRIEVAL\_STATUS\_PAYWALL:</b> The URL retrieval failed because the content is behind a paywall. <br/> <b>URL\_RETRIEVAL\_STATUS\_UNSAFE:</b> The URL retrieval failed because the content is unsafe. <br/>
772+
773+
<b>Signature:</b>
774+
775+
```typescript
776+
URLRetrievalStatus: {
777+
URL_RETRIEVAL_STATUS_UNSPECIFIED: string;
778+
URL_RETRIEVAL_STATUS_SUCCESS: string;
779+
URL_RETRIEVAL_STATUS_ERROR: string;
780+
URL_RETRIEVAL_STATUS_PAYWALL: string;
781+
URL_RETRIEVAL_STATUS_UNSAFE: string;
782+
}
783+
```
784+
758785
## AIErrorCode
759786

760787
Standardized error codes that [AIError](./ai.aierror.md#aierror_class) can have.
@@ -1046,7 +1073,7 @@ Defines a tool that model can call to access external knowledge.
10461073
<b>Signature:</b>
10471074

10481075
```typescript
1049-
export type Tool = FunctionDeclarationsTool | GoogleSearchTool | CodeExecutionTool;
1076+
export type Tool = FunctionDeclarationsTool | GoogleSearchTool | CodeExecutionTool | URLContextTool;
10501077
```
10511078

10521079
## TypedSchema
@@ -1058,3 +1085,18 @@ A type that includes all specific Schema types.
10581085
```typescript
10591086
export type TypedSchema = IntegerSchema | NumberSchema | StringSchema | BooleanSchema | ObjectSchema | ArraySchema | AnyOfSchema;
10601087
```
1088+
1089+
## URLRetrievalStatus
1090+
1091+
> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment.
1092+
>
1093+
1094+
The status of a URL retrieval.
1095+
1096+
<b>URL\_RETRIEVAL\_STATUS\_UNSPECIFIED:</b> Unspecified retrieval status. <br/> <b>URL\_RETRIEVAL\_STATUS\_SUCCESS:</b> The URL retrieval was successful. <br/> <b>URL\_RETRIEVAL\_STATUS\_ERROR:</b> The URL retrieval failed. <br/> <b>URL\_RETRIEVAL\_STATUS\_PAYWALL:</b> The URL retrieval failed because the content is behind a paywall. <br/> <b>URL\_RETRIEVAL\_STATUS\_UNSAFE:</b> The URL retrieval failed because the content is unsafe. <br/>
1097+
1098+
<b>Signature:</b>
1099+
1100+
```typescript
1101+
export type URLRetrievalStatus = (typeof URLRetrievalStatus)[keyof typeof URLRetrievalStatus];
1102+
```

‎docs-devsite/ai.urlcontext.md‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
Project: /docs/reference/js/_project.yaml
2+
Book: /docs/reference/_book.yaml
3+
page_type: reference
4+
5+
{% comment %}
6+
DO NOT EDIT THIS FILE!
7+
This is generated by the JS SDK team, and any local changes will be
8+
overwritten. Changes should be made in the source code at
9+
https://github.com/firebase/firebase-js-sdk
10+
{% endcomment %}
11+
12+
# URLContext interface
13+
> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment.
14+
>
15+
16+
Specifies the URL Context configuration.
17+
18+
<b>Signature:</b>
19+
20+
```typescript
21+
export interface URLContext
22+
```

‎docs-devsite/ai.urlcontextmetadata.md‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
Project: /docs/reference/js/_project.yaml
2+
Book: /docs/reference/_book.yaml
3+
page_type: reference
4+
5+
{% comment %}
6+
DO NOT EDIT THIS FILE!
7+
This is generated by the JS SDK team, and any local changes will be
8+
overwritten. Changes should be made in the source code at
9+
https://github.com/firebase/firebase-js-sdk
10+
{% endcomment %}
11+
12+
# URLContextMetadata interface
13+
> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment.
14+
>
15+
16+
Metadata related to [URLContextTool](./ai.urlcontexttool.md#urlcontexttool_interface)<!-- -->.
17+
18+
<b>Signature:</b>
19+
20+
```typescript
21+
export interface URLContextMetadata
22+
```
23+
24+
## Properties
25+
26+
| Property | Type | Description |
27+
| --- | --- | --- |
28+
| [urlMetadata](./ai.urlcontextmetadata.md#urlcontextmetadataurlmetadata) | [URLMetadata](./ai.urlmetadata.md#urlmetadata_interface)<!-- -->\[\] | <b><i>(Public Preview)</i></b> List of URL metadata used to provide context to the Gemini model. |
29+
30+
## URLContextMetadata.urlMetadata
31+
32+
> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment.
33+
>
34+
35+
List of URL metadata used to provide context to the Gemini model.
36+
37+
<b>Signature:</b>
38+
39+
```typescript
40+
urlMetadata: URLMetadata[];
41+
```

‎docs-devsite/ai.urlcontexttool.md‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
Project: /docs/reference/js/_project.yaml
2+
Book: /docs/reference/_book.yaml
3+
page_type: reference
4+
5+
{% comment %}
6+
DO NOT EDIT THIS FILE!
7+
This is generated by the JS SDK team, and any local changes will be
8+
overwritten. Changes should be made in the source code at
9+
https://github.com/firebase/firebase-js-sdk
10+
{% endcomment %}
11+
12+
# URLContextTool interface
13+
> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment.
14+
>
15+
16+
A tool that allows you to provide additional context to the models in the form of public web URLs. By including URLs in your request, the Gemini model will access the content from those pages to inform and enhance its response.
17+
18+
<b>Signature:</b>
19+
20+
```typescript
21+
export interface URLContextTool
22+
```
23+
24+
## Properties
25+
26+
| Property | Type | Description |
27+
| --- | --- | --- |
28+
| [urlContext](./ai.urlcontexttool.md#urlcontexttoolurlcontext) | [URLContext](./ai.urlcontext.md#urlcontext_interface) | <b><i>(Public Preview)</i></b> Specifies the URL Context configuration. |
29+
30+
## URLContextTool.urlContext
31+
32+
> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment.
33+
>
34+
35+
Specifies the URL Context configuration.
36+
37+
<b>Signature:</b>
38+
39+
```typescript
40+
urlContext: URLContext;
41+
```

‎docs-devsite/ai.urlmetadata.md‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
Project: /docs/reference/js/_project.yaml
2+
Book: /docs/reference/_book.yaml
3+
page_type: reference
4+
5+
{% comment %}
6+
DO NOT EDIT THIS FILE!
7+
This is generated by the JS SDK team, and any local changes will be
8+
overwritten. Changes should be made in the source code at
9+
https://github.com/firebase/firebase-js-sdk
10+
{% endcomment %}
11+
12+
# URLMetadata interface
13+
> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment.
14+
>
15+
16+
Metadata for a single URL retrieved by the [URLContextTool](./ai.urlcontexttool.md#urlcontexttool_interface) tool.
17+
18+
<b>Signature:</b>
19+
20+
```typescript
21+
export interface URLMetadata
22+
```
23+
24+
## Properties
25+
26+
| Property | Type | Description |
27+
| --- | --- | --- |
28+
| [retrievedUrl](./ai.urlmetadata.md#urlmetadataretrievedurl) | string | <b><i>(Public Preview)</i></b> The retrieved URL. |
29+
| [urlRetrievalStatus](./ai.urlmetadata.md#urlmetadataurlretrievalstatus) | [URLRetrievalStatus](./ai.md#urlretrievalstatus) | <b><i>(Public Preview)</i></b> The status of the URL retrieval. |
30+
31+
## URLMetadata.retrievedUrl
32+
33+
> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment.
34+
>
35+
36+
The retrieved URL.
37+
38+
<b>Signature:</b>
39+
40+
```typescript
41+
retrievedUrl?: string;
42+
```
43+
44+
## URLMetadata.urlRetrievalStatus
45+
46+
> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment.
47+
>
48+
49+
The status of the URL retrieval.
50+
51+
<b>Signature:</b>
52+
53+
```typescript
54+
urlRetrievalStatus?: URLRetrievalStatus;
55+
```

0 commit comments

Comments
 (0)