agent: resolve the replay selector in the tool layer

`--save` drops any tool call that names its element by `backendNodeId` and
carries no selector (`Command.ToolCall.isRecorded`), because a registry id
means nothing in a later session. That is a silent gap on the chat path:
whenever the model addresses a node by id, the action runs and is then
quietly omitted from the saved script.

The tool layer is the one place that already resolves the node before
acting, so it is where the selector can be taken while the node still
exists -- a navigation takes it away a moment later. `CallOpts.record`
asks for it and `ToolResult.selector` returns it, for every tool, with no
per-tool knowledge anywhere. Recording then swaps `backendNodeId` for the
selector and keeps every other argument, so `scroll`'s `y` and
`waitForState`'s arguments survive.

One `save_selectors` entry per call, errors included, so the index lines
up with `RunToolsResult.tool_calls_made`.
This commit is contained in:
Adrià Arrufat committed 2026-09-22 15:42:35 +02:00
1 parent b84e5b7397
commit 4da152e9aa
2 files changed
+83 -6

No files matched your search

+45 -5
View File
@@ -167,6 +167,11 @@ cancel_requested: std.atomic.Value(bool) = .init(false),
/// mid-request instead of blocking until the model's full response arrives.
http_interrupt: zenai.http.Interrupt = .{},
synthetic_tool_call_id: u32 = 0,
/// Per-turn CSS selector for each tool call the model made, in call order, so
/// `--save` can record a call that addressed its element by `backendNodeId`.
/// Only filled while `capturing_for_save`.
save_selectors: std.ArrayListUnmanaged(?[]const u8) = .empty,
capturing_for_save: bool = false,
/// Aggregate Anthropic/OpenAI/Gemini token usage across every model call.
/// Printed as a structured `$usage ...` line on stderr at the end of `--task`
/// (one-shot) mode so wrappers can capture per-task cost.
@@ -355,6 +360,7 @@ pub fn init(allocator: std.mem.Allocator, app: *App, opts: Config.Agent) !*Agent
pub fn deinit(self: *Agent) void {
self.terminal.uninstallLogSink();
self.save_buffer.deinit();
self.save_selectors.deinit(self.allocator);
if (self.save_path) |p| self.allocator.free(p);
self.terminal.deinit();
self.conversation.deinit();
@@ -1328,6 +1334,21 @@ fn logSaveBufferError(self: *Agent, err: anyerror) void {
self.terminal.printError("save buffer disabled: {s}", .{@errorName(err)});
}
/// Swap a call's ephemeral `backendNodeId` for the selector the tool layer
/// resolved, so the call can be replayed. Returns `args` untouched when there
/// is nothing to swap.
fn withSelector(arena: std.mem.Allocator, args: ?std.json.Value, selector: ?[]const u8) ?std.json.Value {
const sel = selector orelse return args;
const original = args orelse return args;
if (original != .object) return args;
if (!original.object.contains("backendNodeId")) return args;
var rewritten = original.object.clone(arena) catch return args;
_ = rewritten.swapRemove("backendNodeId");
rewritten.put(arena, "selector", .{ .string = sel }) catch return args;
return .{ .object = rewritten };
}
fn recordSaveCommand(self: *Agent, cmd: Command) void {
self.save_buffer.record(cmd) catch |err| self.logSaveBufferError(err);
}
@@ -1671,6 +1692,10 @@ fn processUserMessage(self: *Agent, input: TurnInput) !?[]const u8 {
const provider_client = self.ai_client orelse return error.NoAiClient;
self.refreshAuthIfNeeded();
self.capturing_for_save = input.capture_for_save;
defer self.capturing_for_save = false;
self.save_selectors.clearRetainingCapacity();
self.terminal.spinner.start();
var result = provider_client.runTools(
self.model,
@@ -1731,11 +1756,15 @@ fn processUserMessage(self: *Agent, input: TurnInput) !?[]const u8 {
const args = browser_tools.normalizeArgKeys(ca, tool, tc.arguments) catch tc.arguments;
// Fall back to the navigation a read tool performed, so a
// markdown/tree-driven turn isn't lost from `/save`.
const cmd = Command.fromToolCall(tool, args);
// A call that named its element by id is unreplayable as-is; the
// tool layer resolved a selector for it while the node still
// existed.
const replayable = withSelector(ca, args, if (i < self.save_selectors.items.len) self.save_selectors.items[i] else null);
const cmd = Command.fromToolCall(tool, replayable);
const to_record = if (cmd.isRecorded())
cmd
else
navigationGoto(ca, tool, args) orelse continue;
navigationGoto(ca, tool, replayable) orelse continue;
if (!recorded_any) {
if (input.record_comment) |c| self.recordSaveComment(c);
recorded_any = true;
@@ -1888,10 +1917,17 @@ fn handleToolCall(ctx: *anyopaque, allocator: std.mem.Allocator, tool_name: []co
self.terminal.spinner.setTool(tool_name, args_str);
defer self.terminal.spinner.setThinking();
const outcome = self.toolOutcome(allocator, tool_name, arguments) catch |err| zenai.provider.Client.ToolHandler.Result{
var selector: ?[]const u8 = null;
const outcome = self.toolOutcome(allocator, tool_name, arguments, &selector) catch |err| zenai.provider.Client.ToolHandler.Result{
.content = std.fmt.allocPrint(allocator, "Error: {s}", .{browser_tools.errorMessage(err)}) catch "Error: tool execution failed",
.is_error = true,
};
if (self.capturing_for_save) {
// One entry per call, errors included, so the index lines up with
// `RunToolsResult.tool_calls_made`.
const kept = if (selector) |sel| self.allocator.dupe(u8, sel) catch null else null;
self.save_selectors.append(self.allocator, kept) catch {};
}
self.terminal.agentToolDone(tool_name, args_str, !outcome.is_error);
if (self.terminal.verbosity == .high) self.terminal.printToolOutcome(tool_name, outcome.content, outcome.is_error);
@@ -1899,8 +1935,12 @@ fn handleToolCall(ctx: *anyopaque, allocator: std.mem.Allocator, tool_name: []co
}
/// The text plus the rendered PNG, for backends that can show the model an image.
fn toolOutcome(self: *Agent, allocator: std.mem.Allocator, tool_name: []const u8, arguments: ?std.json.Value) browser_tools.ToolError!zenai.provider.Client.ToolHandler.Result {
const result = try browser_tools.call(allocator, self.ts.session, &self.ts.registry, tool_name, arguments, .{ .inline_image = true });
fn toolOutcome(self: *Agent, allocator: std.mem.Allocator, tool_name: []const u8, arguments: ?std.json.Value, selector: *?[]const u8) browser_tools.ToolError!zenai.provider.Client.ToolHandler.Result {
const result = try browser_tools.call(allocator, self.ts.session, &self.ts.registry, tool_name, arguments, .{
.inline_image = true,
.record = self.capturing_for_save,
});
selector.* = result.selector;
const content = capToolOutput(allocator, tool_name, result.text);
return .{
.content = content,
+38 -1
View File
@@ -24,6 +24,8 @@ const NodeRegistry = @import("../NodeRegistry.zig");
const DOMNode = @import("webapi/Node.zig");
const Selector = @import("webapi/selector/Selector.zig");
const SelectorPath = @import("SelectorPath.zig");
const Element = @import("webapi/Element.zig");
const log = lp.log;
const tavily = zenai.search.tavily;
@@ -823,6 +825,10 @@ pub const ToolResult = struct {
is_error: bool = false,
/// Only set when the caller passed `CallOpts.inline_image`.
image: ?lp.screenshot.Prepared = null,
/// Only set when the caller passed `CallOpts.record` and the call named its
/// element by `backendNodeId`. Resolved before the action runs, because a
/// navigation takes the node with it.
selector: ?[]const u8 = null,
};
const GotoParams = struct {
@@ -854,6 +860,11 @@ const NodeAndPage = struct { node: *DOMNode, page: *lp.Frame, target: ActionTarg
pub const CallOpts = struct {
/// The caller can hand an image to a model.
inline_image: bool = false,
/// The caller is recording for `--save`/`/save`. A call that addresses its
/// element by `backendNodeId` gets `ToolResult.selector` filled in, since
/// a registry id means nothing in a later session and the node may be gone
/// by the time the caller wants to record it.
record: bool = false,
};
// An inline screenshot is re-sent on every turn; keep it within what models
@@ -886,13 +897,39 @@ pub fn call(
};
const substituted = try substituteStringArgs(arena, tool, normalized);
return dispatch(arena, session, registry, tool, substituted, opts) catch |err| {
// Before dispatch: after a navigation the node is gone.
const selector = if (opts.record) selectorForArgs(arena, session, registry, substituted) else null;
var result = dispatch(arena, session, registry, tool, substituted, opts) catch |err| {
if (err == error.NavigationFailed) {
if (formatNavigationError(arena, session)) |text|
return .{ .text = text, .is_error = true };
}
return err;
};
result.selector = selector;
return result;
}
/// The CSS selector for a call's `backendNodeId`, so the call can be recorded
/// in a form that still resolves in a later session. Null when the arguments
/// name no node, already carry a selector, or the node cannot be named.
fn selectorForArgs(
arena: std.mem.Allocator,
session: *lp.Session,
registry: *NodeRegistry,
arguments: ?std.json.Value,
) ?[]const u8 {
const args = arguments orelse return null;
if (args != .object) return null;
if (args.object.contains("selector")) return null;
const id = args.object.get("backendNodeId") orelse return null;
if (id != .integer) return null;
const node = registry.lookup_by_id.get(std.math.cast(NodeRegistry.Id, id.integer) orelse return null) orelse return null;
const el = node.dom.is(Element) orelse return null;
const frame = session.currentFrame() orelse return null;
return SelectorPath.init(arena, frame).build(el) catch null;
}
fn dispatch(