agent: let the model see a REPL /screenshot; fold the tool-result adapters

The slash path's result has two consumers: the terminal, which can't
show an image, and the conversation, which can. Opt in when a model is
attached and forward the image through the same adapter the model-driven
path uses, and stop mapping a failed adapter to a text-only success.
MCP's ImageContent and CallToolResult take their payload type like
TextContent does; resolveScope reuses resolveTarget; needsLocator folds
into replayRequires.
This commit is contained in:
Adrià Arrufat authored and GitHub committed 2026-08-27 15:03:19 +02:00
1 parent e6203ff1fe
commit cff32f6c04
6 files changed
+59 -73

No files matched your search

+20 -14
View File
@@ -1488,7 +1488,8 @@ fn runCommand(self: *Agent, arena: std.mem.Allocator, cmd: Command) browser_tool
.tool_call => |t| t,
else => return .{ .text = "internal: command has no tool mapping", .is_error = true },
};
return browser_tools.call(arena, self.session, &self.node_registry, tc.name(), tc.args, .{}) catch |err| .{
// The terminal can't show an image, but the conversation can.
return browser_tools.call(arena, self.session, &self.node_registry, tc.name(), tc.args, .{ .inline_image = self.ai_client != null }) catch |err| .{
.text = switch (err) {
error.OutOfMemory => "out of memory",
error.FrameNotLoaded => "no page loaded — run /goto <url> first",
@@ -1620,6 +1621,7 @@ fn recordSlashToolCall(
.id = try ma.dupe(u8, tool_calls[0].id),
.name = try ma.dupe(u8, tool_calls[0].name),
.content = content,
.parts = if (result.image) |image| try imageParts(ma, content, &image) else null,
.is_error = result.is_error,
};
@@ -1922,14 +1924,10 @@ fn handleToolCall(ctx: *anyopaque, allocator: std.mem.Allocator, tool_name: []co
self.terminal.spinner.setTool(tool_name, args_str);
defer self.terminal.spinner.setThinking();
const outcome: zenai.provider.Client.ToolHandler.Result = if (browser_tools.call(allocator, self.session, &self.node_registry, tool_name, arguments, .{ .inline_image = true })) |result| blk: {
const content = capToolOutput(allocator, tool_name, result.text);
break :blk .{
.content = content,
.is_error = result.is_error,
.parts = if (result.image) |image| imageParts(allocator, content, &image) catch null else null,
};
} else |err| .{ .content = std.fmt.allocPrint(allocator, "Error: {s}", .{browser_tools.errorMessage(err)}) catch "Error: tool execution failed", .is_error = true };
const outcome = self.toolOutcome(allocator, tool_name, arguments) catch |err| zenai.provider.Client.ToolHandler.Result{
.content = std.fmt.allocPrint(allocator, "Error: {s}", .{browser_tools.errorMessage(err)}) catch "Error: tool execution failed",
.is_error = true,
};
self.terminal.agentToolDone(tool_name, args_str, !outcome.is_error);
if (self.terminal.verbosity == .high) self.terminal.printToolOutcome(tool_name, outcome.content, outcome.is_error);
@@ -1937,11 +1935,19 @@ fn handleToolCall(ctx: *anyopaque, allocator: std.mem.Allocator, tool_name: []co
}
/// The text plus the rendered PNG, for backends that can show the model an image.
fn imageParts(allocator: std.mem.Allocator, text: []const u8, image: *const lp.screenshot.Prepared) ![]const zenai.provider.ContentPart {
const parts = try allocator.alloc(zenai.provider.ContentPart, 2);
parts[0] = .{ .text = text };
parts[1] = .{ .image = .{ .data = try image.base64Alloc(allocator), .mime_type = "image/png" } };
return parts;
fn toolOutcome(self: *Agent, allocator: std.mem.Allocator, tool_name: []const u8, arguments: ?std.json.Value) browser_tools.ToolError!zenai.provider.Client.ToolHandler.Result {
const result = try browser_tools.call(allocator, self.session, &self.node_registry, tool_name, arguments, .{ .inline_image = true });
const content = capToolOutput(allocator, tool_name, result.text);
return .{
.content = content,
.is_error = result.is_error,
.parts = if (result.image) |image| try imageParts(allocator, content, &image) else null,
};
}
fn imageParts(arena: std.mem.Allocator, text: []const u8, image: *const lp.screenshot.Prepared) browser_tools.ToolError![]const zenai.provider.ContentPart {
const data = image.base64Alloc(arena) catch return error.InternalError;
return try arena.dupe(zenai.provider.ContentPart, &.{ .{ .text = text }, .{ .image = .{ .data = data, .mime_type = "image/png" } } });
}
/// One-shot for `--list-models`: resolve provider+key, fetch chat-capable model
+4 -21
View File
@@ -136,11 +136,10 @@ pub const Prepared = struct {
}
/// The PNG as base64, for APIs that want it as one string.
pub fn base64Alloc(self: *const Prepared, allocator: Allocator) ![]const u8 {
var aw: std.Io.Writer.Allocating = .init(allocator);
errdefer aw.deinit();
pub fn base64Alloc(self: *const Prepared, arena: Allocator) ![]const u8 {
var aw: std.Io.Writer.Allocating = .init(arena);
try self.writeBase64(&aw.writer);
return aw.toOwnedSlice();
return aw.written();
}
fn writeBase64(self: *const Prepared, writer: *std.Io.Writer) std.Io.Writer.Error!void {
@@ -676,23 +675,6 @@ test "browser.screenshot: png signature and dimensions" {
try testing.expectEqual(true, height > 60 and height < 200);
}
test "browser.screenshot: base64Alloc is the png, base64 encoded" {
defer testing.test_session.closeAllPages();
const frame = try testing.createFrame();
frame.url = "http://localhost/";
const div = try frame.window._document.createElement("div", null, frame);
try Frame.parse.htmlAsChildren(frame, div.asNode(), "<p>Hello</p>");
const prepared = try prepare(testing.arena_allocator, div.asNode(), .{ .width = 320 }, frame);
const b64 = try prepared.base64Alloc(testing.arena_allocator);
try testing.expect(std.mem.startsWith(u8, b64, "iVBORw0KGgo"));
const decoder = std.base64.standard.Decoder;
const bytes = try testing.arena_allocator.alloc(u8, try decoder.calcSizeForSlice(b64));
try decoder.decode(bytes, b64);
try testing.expectEqual("\x89PNG\r\n\x1a\n", bytes[0..8]);
}
test "browser.screenshot: fixed height, clip and scale" {
defer testing.test_session.closeAllPages();
const frame = try testing.createFrame();
@@ -810,6 +792,7 @@ test "browser.screenshot: json streams base64" {
_ = enc.encode(expected[9 .. expected.len - 2], raw.written());
@memcpy(expected[expected.len - 2 ..], "\"}");
try testing.expectString(expected, json.written());
try testing.expectString(expected[9 .. expected.len - 2], try prepared.base64Alloc(testing.arena_allocator));
}
test "browser.screenshot: block extraction" {
+16 -26
View File
@@ -40,8 +40,7 @@ const Selector = @import("webapi/selector/Selector.zig");
pub const driver_guidance =
\\You are driving Lightpanda, a headless browser, through text tools:
\\you reason over pages as a semantic tree, markdown or HTML. `screenshot`
\\renders that text layout as a PNG (no images, fonts or CSS) — use it
\\for spatial layout, not as a primary read.
\\renders that text layout as a PNG: for spatial layout, not a primary read.
\\
\\Reading pages (cheap → expensive — prefer cheaper):
\\- `tree` → semantic overview (role, name, value, backendNodeId per
@@ -215,14 +214,6 @@ pub const save_script_rules =
/// `..` segment. Operator-controlled symlinks already inside CWD are out
/// of scope — the threat we close here is "client supplies an arbitrary
/// path string".
pub const unsafe_path_message = "path must be relative and must not contain '..' segments";
/// The cwd is the server's, not one the user picked, so report where a file
/// really went.
pub fn absolutePath(arena: std.mem.Allocator, path: []const u8) []const u8 {
return std.Io.Dir.cwd().realPathFileAlloc(lp.io, path, arena) catch path;
}
pub fn isPathSafe(path: []const u8) bool {
if (path.len == 0) return false;
if (std.fs.path.isAbsolute(path)) return false;
@@ -233,6 +224,14 @@ pub fn isPathSafe(path: []const u8) bool {
return true;
}
pub const unsafe_path_message = "path must be relative and must not contain '..' segments";
/// The cwd is the server's, not one the user picked, so report where a file
/// really went.
pub fn absolutePath(arena: std.mem.Allocator, path: []const u8) []const u8 {
return std.Io.Dir.cwd().realPathFileAlloc(lp.io, path, arena) catch path;
}
/// Hand-written so per-tool semantics (record/heal/locator/data) and
/// LLM-facing metadata (`definition`) live as exhaustive switches on the
/// tag — adding a new tool is a compile error until each predicate AND
@@ -320,13 +319,6 @@ pub const Tool = enum {
};
}
pub fn needsLocator(self: Tool) bool {
for (self.replayRequires()) |field| {
if (std.mem.eql(u8, field, "selector")) return true;
}
return false;
}
/// Result is data the caller probably wants on stdout (extracted JSON,
/// markdown, evaluate return value) rather than a status line on stderr.
pub fn producesData(self: Tool) bool {
@@ -1322,9 +1314,8 @@ fn execMarkdown(arena: std.mem.Allocator, session: *lp.Session, registry: *CDPNo
/// The node a read tool works on: the selector match, the registry node, or
/// the whole document. All three live in the current frame.
fn resolveScope(session: *lp.Session, registry: *CDPNode.Registry, page: *lp.Frame, selector: ?[]const u8, node_id: ?CDPNode.Id) ToolError!*DOMNode {
if (selector) |sel| return (try resolveBySelector(session, sel)).node;
if (node_id) |nid| return (try resolveNodeAndPage(session, registry, nid)).node;
return page.document.asNode();
if (selector == null and node_id == null) return page.document.asNode();
return (try resolveTarget(session, registry, selector, node_id)).node;
}
const HtmlParams = struct {
@@ -1345,7 +1336,7 @@ fn execHtml(arena: std.mem.Allocator, session: *lp.Session, registry: *CDPNode.R
if (args.selector == null and args.backendNodeId == null) {
lp.dump.root(page.document, opts, &aw.writer, page) catch return ToolError.InternalError;
} else {
const node = try resolveScope(session, registry, page, args.selector, args.backendNodeId);
const node = (try resolveTarget(session, registry, args.selector, args.backendNodeId)).node;
lp.dump.deep(node, opts, &aw.writer, page) catch return ToolError.InternalError;
}
return aw.written();
@@ -1368,19 +1359,18 @@ fn execScreenshot(arena: std.mem.Allocator, session: *lp.Session, registry: *CDP
}
const page = try ensurePage(session, registry, args.url, args.timeout);
const node = try resolveScope(session, registry, page, args.selector, args.backendNodeId);
const prepared = lp.screenshot.prepare(arena, node, .fromViewport(page._page.getViewport(), args.fullPage), page) catch
return ToolError.InternalError;
const width = prepared.opts.width;
const opts: lp.screenshot.Opts = .fromViewport(page._page.getViewport(), args.fullPage);
const prepared = lp.screenshot.prepare(arena, node, opts, page) catch return ToolError.InternalError;
if (args.path) |path| {
const height = writePng(&prepared, path) catch |err| return .{
.text = std.fmt.allocPrint(arena, "could not write {s}: {s}", .{ path, @errorName(err) }) catch return ToolError.OutOfMemory,
.is_error = true,
};
return .{ .text = std.fmt.allocPrint(arena, "Saved {d}x{d} PNG to {s}", .{ width, height, absolutePath(arena, path) }) catch return ToolError.OutOfMemory };
return .{ .text = std.fmt.allocPrint(arena, "Saved {d}x{d} PNG to {s}", .{ opts.width, height, absolutePath(arena, path) }) catch return ToolError.OutOfMemory };
}
return .{
.text = std.fmt.allocPrint(arena, "PNG, {d}px wide", .{width}) catch return ToolError.OutOfMemory,
.text = std.fmt.allocPrint(arena, "PNG, {d}px wide", .{opts.width}) catch return ToolError.OutOfMemory,
.image = prepared,
};
}
+12 -8
View File
@@ -167,12 +167,6 @@ pub const CallParams = struct {
arguments: ?std.json.Value = null,
};
pub const ImageContent = struct {
type: []const u8 = "image",
data: @import("../browser/screenshot.zig").Prepared,
mimeType: []const u8 = "image/png",
};
pub fn TextContent(comptime T: type) type {
return struct {
type: []const u8 = "text",
@@ -180,9 +174,19 @@ pub fn TextContent(comptime T: type) type {
};
}
pub fn CallToolResult(comptime T: type) type {
/// `T` serializes as the base64 payload.
pub fn ImageContent(comptime T: type) type {
return struct {
content: []const TextContent(T),
type: []const u8 = "image",
data: T,
mimeType: []const u8,
};
}
/// `Content` is the content array: a slice or tuple of `TextContent`/`ImageContent`.
pub fn CallToolResult(comptime Content: type) type {
return struct {
content: Content,
isError: bool = false,
};
}
+4 -3
View File
@@ -173,8 +173,9 @@ fn dispatchBrowserTool(
};
if (result.image) |image| {
return server.sendResult(id, .{
.content = .{ protocol.ImageContent{ .data = image }, protocol.TextContent([]const u8){ .text = result.text } },
const Content = struct { protocol.ImageContent(lp.screenshot.Prepared), protocol.TextContent([]const u8) };
return server.sendResult(id, protocol.CallToolResult(Content){
.content = .{ .{ .data = image, .mimeType = "image/png" }, .{ .text = result.text } },
.isError = result.is_error,
});
}
@@ -289,7 +290,7 @@ fn writeScript(path: []const u8, content: []const u8) !void {
fn sendToolResultText(server: *Server, id: std.json.Value, msg: []const u8, is_error: bool) !void {
const content = [_]protocol.TextContent([]const u8){.{ .text = msg }};
try server.sendResult(id, protocol.CallToolResult([]const u8){ .content = &content, .isError = is_error });
try server.sendResult(id, protocol.CallToolResult([]const protocol.TextContent([]const u8)){ .content = &content, .isError = is_error });
}
fn sendErrorContent(server: *Server, id: std.json.Value, msg: []const u8) !void {
+3 -1
View File
@@ -180,7 +180,9 @@ fn positionalOptional(s: *const Schema, p: []const u8) bool {
if (s.findField(p)) |f| {
if (f.default_true) return true;
}
if (std.mem.eql(u8, p, "selector") and s.tool.needsLocator()) return false;
for (s.tool.replayRequires()) |r| {
if (std.mem.eql(u8, r, p)) return false;
}
for (s.required) |r| {
if (std.mem.eql(u8, r, p)) return false;
}