add playwright arguments to agent

.changeset/icy-toes-obey.md

@@ -0,0 +1,5 @@
+---
+"@browserbasehq/stagehand": patch
+---
+
+Add playwright arguments to agent execute response

lib/agent/tools/act.ts

@@ -1,7 +1,8 @@
 import { tool } from "ai";
 import { z } from "zod/v3";
 import { StagehandPage } from "../../StagehandPage";
-
+import { buildActObservePrompt } from "../../prompt";
+import { SupportedPlaywrightAction } from "@/types/act";
 export const createActTool = (
   stagehandPage: StagehandPage,
   executionModel?: string,
@@ -19,37 +20,91 @@ export const createActTool = (
     }),
     execute: async ({ action }) => {
       try {
-        let result;
-        if (executionModel) {
-          result = await stagehandPage.page.act({
-            action,
-            modelName: executionModel,
-          });
-        } else {
-          result = await stagehandPage.page.act(action);
+        const builtPrompt = buildActObservePrompt(
+          action,
+          Object.values(SupportedPlaywrightAction),
+        );
+
+        const observeOptions = executionModel
+          ? {
+              instruction: builtPrompt,
+              modelName: executionModel,
+            }
+          : {
+              instruction: builtPrompt,
+            };
+
+        const observeResults = await stagehandPage.page.observe(observeOptions);
+
+        if (!observeResults || observeResults.length === 0) {
+          return {
+            success: false,
+            error: "No observable actions found for the given instruction",
+          };
         }
-        const isIframeAction = result.action === "an iframe";
+
+        const observeResult = observeResults[0];
+
+        const isIframeAction = observeResult.description === "an iframe";
 
         if (isIframeAction) {
-          const fallback = await stagehandPage.page.act(
-            executionModel
-              ? { action, modelName: executionModel, iframes: true }
-              : { action, iframes: true },
-          );
+          const iframeObserveOptions = executionModel
+            ? {
+                instruction: builtPrompt,
+                modelName: executionModel,
+                iframes: true,
+              }
+            : {
+                instruction: builtPrompt,
+                iframes: true,
+              };
+
+          const iframeObserveResults =
+            await stagehandPage.page.observe(iframeObserveOptions);
+
+          if (!iframeObserveResults || iframeObserveResults.length === 0) {
+            return {
+              success: false,
+              error: "No observable actions found within iframe context",
+              isIframe: true,
+            };
+          }
+
+          const iframeObserveResult = iframeObserveResults[0];
+          const fallback = await stagehandPage.page.act(iframeObserveResult);
+
           return {
             success: fallback.success,
             action: fallback.action,
             isIframe: true,
+            playwrightArguments: {
+              description: iframeObserveResult.description,
+              method: iframeObserveResult.method,
+              arguments: iframeObserveResult.arguments,
+              selector: iframeObserveResult.selector,
+            },
           };
         }
 
+        const result = await stagehandPage.page.act(observeResult);
+        const playwrightArguments = {
+          description: observeResult.description,
+          method: observeResult.method,
+          arguments: observeResult.arguments,
+          selector: observeResult.selector,
+        };
+
         return {
           success: result.success,
           action: result.action,
           isIframe: false,
+          playwrightArguments,
         };
       } catch (error) {
-        return { success: false, error: error.message };
+        return {
+          success: false,
+          error: error.message,
+        };
       }
     },
   });

lib/handlers/stagehandAgentHandler.ts

@@ -1,4 +1,9 @@
-import { AgentAction, AgentExecuteOptions, AgentResult } from "@/types/agent";
+import {
+  AgentAction,
+  AgentExecuteOptions,
+  AgentResult,
+  ActToolResult,
+} from "@/types/agent";
 import { LogLine } from "@/types/log";
 import { StagehandPage } from "../StagehandPage";
 import { LLMClient } from "../llm/LLMClient";
@@ -99,7 +104,8 @@ export class StagehandAgentHandler {
           });
 
           if (event.toolCalls && event.toolCalls.length > 0) {
-            for (const toolCall of event.toolCalls) {
+            for (let i = 0; i < event.toolCalls.length; i++) {
+              const toolCall = event.toolCalls[i];
               const args = toolCall.args as Record<string, unknown>;
 
               if (event.text.length > 0) {
@@ -122,6 +128,21 @@ export class StagehandAgentHandler {
                 }
               }
 
+              // Get the tool result if available
+              const toolResult = event.toolResults?.[i];
+
+              const getPlaywrightArguments = () => {
+                if (toolCall.toolName !== "act" || !toolResult) {
+                  return {};
+                }
+                const result = toolResult.result as ActToolResult;
+                if (result && result.playwrightArguments) {
+                  return { playwrightArguments: result.playwrightArguments };
+                }
+
+                return {};
+              };
+
               const action: AgentAction = {
                 type: toolCall.toolName,
                 reasoning: event.text || undefined,
@@ -130,6 +151,7 @@ export class StagehandAgentHandler {
                     ? (args?.taskComplete as boolean)
                     : false,
                 ...args,
+                ...getPlaywrightArguments(),
               };
 
               actions.push(action);

types/agent.ts

@@ -1,4 +1,13 @@
 import { LogLine } from "./log";
+import { ObserveResult } from "./stagehand";
+
+export interface ActToolResult {
+  success: boolean;
+  action?: string;
+  error?: string;
+  isIframe?: boolean;
+  playwrightArguments?: ObserveResult | null;
+}
 
 export interface AgentAction {
   type: string;
@@ -10,6 +19,7 @@ export interface AgentAction {
   pageText?: string; // ariaTree tool
   pageUrl?: string; // ariaTree tool
   instruction?: string; // various tools
+  playwrightArguments?: ObserveResult | null; // act tool
   [key: string]: unknown;
 }