diff --git a/src/agent/Agent.zig b/src/agent/Agent.zig index 5207605ec..ae870f162 100644 --- a/src/agent/Agent.zig +++ b/src/agent/Agent.zig @@ -167,6 +167,11 @@ cancel_requested: std.atomic.Value(bool) = .init(false), /// mid-request instead of blocking until the model's full response arrives. http_interrupt: zenai.http.Interrupt = .{}, synthetic_tool_call_id: u32 = 0, +/// Per-turn CSS selector for each tool call the model made, in call order, so +/// `--save` can record a call that addressed its element by `backendNodeId`. +/// Only filled while `capturing_for_save`. +save_selectors: std.ArrayListUnmanaged(?[]const u8) = .empty, +capturing_for_save: bool = false, /// Aggregate Anthropic/OpenAI/Gemini token usage across every model call. /// Printed as a structured `$usage ...` line on stderr at the end of `--task` /// (one-shot) mode so wrappers can capture per-task cost. @@ -355,6 +360,7 @@ pub fn init(allocator: std.mem.Allocator, app: *App, opts: Config.Agent) !*Agent pub fn deinit(self: *Agent) void { self.terminal.uninstallLogSink(); self.save_buffer.deinit(); + self.save_selectors.deinit(self.allocator); if (self.save_path) |p| self.allocator.free(p); self.terminal.deinit(); self.conversation.deinit(); @@ -1328,6 +1334,21 @@ fn logSaveBufferError(self: *Agent, err: anyerror) void { self.terminal.printError("save buffer disabled: {s}", .{@errorName(err)}); } +/// Swap a call's ephemeral `backendNodeId` for the selector the tool layer +/// resolved, so the call can be replayed. Returns `args` untouched when there +/// is nothing to swap. +fn withSelector(arena: std.mem.Allocator, args: ?std.json.Value, selector: ?[]const u8) ?std.json.Value { + const sel = selector orelse return args; + const original = args orelse return args; + if (original != .object) return args; + if (!original.object.contains("backendNodeId")) return args; + + var rewritten = original.object.clone(arena) catch return args; + _ = rewritten.swapRemove("backendNodeId"); + rewritten.put(arena, "selector", .{ .string = sel }) catch return args; + return .{ .object = rewritten }; +} + fn recordSaveCommand(self: *Agent, cmd: Command) void { self.save_buffer.record(cmd) catch |err| self.logSaveBufferError(err); } @@ -1671,6 +1692,10 @@ fn processUserMessage(self: *Agent, input: TurnInput) !?[]const u8 { const provider_client = self.ai_client orelse return error.NoAiClient; self.refreshAuthIfNeeded(); + self.capturing_for_save = input.capture_for_save; + defer self.capturing_for_save = false; + self.save_selectors.clearRetainingCapacity(); + self.terminal.spinner.start(); var result = provider_client.runTools( self.model, @@ -1731,11 +1756,15 @@ fn processUserMessage(self: *Agent, input: TurnInput) !?[]const u8 { const args = browser_tools.normalizeArgKeys(ca, tool, tc.arguments) catch tc.arguments; // Fall back to the navigation a read tool performed, so a // markdown/tree-driven turn isn't lost from `/save`. - const cmd = Command.fromToolCall(tool, args); + // A call that named its element by id is unreplayable as-is; the + // tool layer resolved a selector for it while the node still + // existed. + const replayable = withSelector(ca, args, if (i < self.save_selectors.items.len) self.save_selectors.items[i] else null); + const cmd = Command.fromToolCall(tool, replayable); const to_record = if (cmd.isRecorded()) cmd else - navigationGoto(ca, tool, args) orelse continue; + navigationGoto(ca, tool, replayable) orelse continue; if (!recorded_any) { if (input.record_comment) |c| self.recordSaveComment(c); recorded_any = true; @@ -1888,10 +1917,17 @@ fn handleToolCall(ctx: *anyopaque, allocator: std.mem.Allocator, tool_name: []co self.terminal.spinner.setTool(tool_name, args_str); defer self.terminal.spinner.setThinking(); - const outcome = self.toolOutcome(allocator, tool_name, arguments) catch |err| zenai.provider.Client.ToolHandler.Result{ + var selector: ?[]const u8 = null; + const outcome = self.toolOutcome(allocator, tool_name, arguments, &selector) catch |err| zenai.provider.Client.ToolHandler.Result{ .content = std.fmt.allocPrint(allocator, "Error: {s}", .{browser_tools.errorMessage(err)}) catch "Error: tool execution failed", .is_error = true, }; + if (self.capturing_for_save) { + // One entry per call, errors included, so the index lines up with + // `RunToolsResult.tool_calls_made`. + const kept = if (selector) |sel| self.allocator.dupe(u8, sel) catch null else null; + self.save_selectors.append(self.allocator, kept) catch {}; + } self.terminal.agentToolDone(tool_name, args_str, !outcome.is_error); if (self.terminal.verbosity == .high) self.terminal.printToolOutcome(tool_name, outcome.content, outcome.is_error); @@ -1899,8 +1935,12 @@ fn handleToolCall(ctx: *anyopaque, allocator: std.mem.Allocator, tool_name: []co } /// The text plus the rendered PNG, for backends that can show the model an image. -fn toolOutcome(self: *Agent, allocator: std.mem.Allocator, tool_name: []const u8, arguments: ?std.json.Value) browser_tools.ToolError!zenai.provider.Client.ToolHandler.Result { - const result = try browser_tools.call(allocator, self.ts.session, &self.ts.registry, tool_name, arguments, .{ .inline_image = true }); +fn toolOutcome(self: *Agent, allocator: std.mem.Allocator, tool_name: []const u8, arguments: ?std.json.Value, selector: *?[]const u8) browser_tools.ToolError!zenai.provider.Client.ToolHandler.Result { + const result = try browser_tools.call(allocator, self.ts.session, &self.ts.registry, tool_name, arguments, .{ + .inline_image = true, + .record = self.capturing_for_save, + }); + selector.* = result.selector; const content = capToolOutput(allocator, tool_name, result.text); return .{ .content = content, diff --git a/src/browser/tools.zig b/src/browser/tools.zig index 9b8d31e52..9a7bfcf88 100644 --- a/src/browser/tools.zig +++ b/src/browser/tools.zig @@ -24,6 +24,8 @@ const NodeRegistry = @import("../NodeRegistry.zig"); const DOMNode = @import("webapi/Node.zig"); const Selector = @import("webapi/selector/Selector.zig"); +const SelectorPath = @import("SelectorPath.zig"); +const Element = @import("webapi/Element.zig"); const log = lp.log; const tavily = zenai.search.tavily; @@ -823,6 +825,10 @@ pub const ToolResult = struct { is_error: bool = false, /// Only set when the caller passed `CallOpts.inline_image`. image: ?lp.screenshot.Prepared = null, + /// Only set when the caller passed `CallOpts.record` and the call named its + /// element by `backendNodeId`. Resolved before the action runs, because a + /// navigation takes the node with it. + selector: ?[]const u8 = null, }; const GotoParams = struct { @@ -854,6 +860,11 @@ const NodeAndPage = struct { node: *DOMNode, page: *lp.Frame, target: ActionTarg pub const CallOpts = struct { /// The caller can hand an image to a model. inline_image: bool = false, + /// The caller is recording for `--save`/`/save`. A call that addresses its + /// element by `backendNodeId` gets `ToolResult.selector` filled in, since + /// a registry id means nothing in a later session and the node may be gone + /// by the time the caller wants to record it. + record: bool = false, }; // An inline screenshot is re-sent on every turn; keep it within what models @@ -886,13 +897,39 @@ pub fn call( }; const substituted = try substituteStringArgs(arena, tool, normalized); - return dispatch(arena, session, registry, tool, substituted, opts) catch |err| { + // Before dispatch: after a navigation the node is gone. + const selector = if (opts.record) selectorForArgs(arena, session, registry, substituted) else null; + + var result = dispatch(arena, session, registry, tool, substituted, opts) catch |err| { if (err == error.NavigationFailed) { if (formatNavigationError(arena, session)) |text| return .{ .text = text, .is_error = true }; } return err; }; + result.selector = selector; + return result; +} + +/// The CSS selector for a call's `backendNodeId`, so the call can be recorded +/// in a form that still resolves in a later session. Null when the arguments +/// name no node, already carry a selector, or the node cannot be named. +fn selectorForArgs( + arena: std.mem.Allocator, + session: *lp.Session, + registry: *NodeRegistry, + arguments: ?std.json.Value, +) ?[]const u8 { + const args = arguments orelse return null; + if (args != .object) return null; + if (args.object.contains("selector")) return null; + const id = args.object.get("backendNodeId") orelse return null; + if (id != .integer) return null; + + const node = registry.lookup_by_id.get(std.math.cast(NodeRegistry.Id, id.integer) orelse return null) orelse return null; + const el = node.dom.is(Element) orelse return null; + const frame = session.currentFrame() orelse return null; + return SelectorPath.init(arena, frame).build(el) catch null; } fn dispatch(