diff --git a/src/agent/Agent.zig b/src/agent/Agent.zig index 82b2f4034..8064704e7 100644 --- a/src/agent/Agent.zig +++ b/src/agent/Agent.zig @@ -696,11 +696,12 @@ fn runRepl(self: *Agent) void { self.terminal.beginTool(tc.name(), slash_split.?.rest); const result = self.runCommand(aa, cmd); self.terminal.endTool(); - self.printCommandResult(cmd, result); + const image = renderImage(aa, result); + self.printCommandResult(cmd, result, image); if (!result.is_error) { self.recordSaveCommand(navigationGoto(aa, tc.tool, tc.args) orelse cmd); } - self.recordSlashToolCall(command_text, tc.name(), tc.args, result) catch |err| { + self.recordSlashToolCall(command_text, tc.name(), tc.args, result, image) catch |err| { self.terminal.printWarning("LLM conversation out of sync (/{s}: {s}); next prompt may not see this action", .{ tc.name(), @errorName(err) }); }; }, @@ -1488,8 +1489,8 @@ fn runCommand(self: *Agent, arena: std.mem.Allocator, cmd: Command) browser_tool .tool_call => |t| t, else => return .{ .text = "internal: command has no tool mapping", .is_error = true }, }; - // The terminal can't show an image, but the conversation can. - return browser_tools.call(arena, self.session, &self.node_registry, tc.name(), tc.args, .{ .inline_image = self.ai_client != null }) catch |err| .{ + const inline_image = self.ai_client != null or self.terminal.canShowImage(); + return browser_tools.call(arena, self.session, &self.node_registry, tc.name(), tc.args, .{ .inline_image = inline_image }) catch |err| .{ .text = switch (err) { error.OutOfMemory => "out of memory", error.FrameNotLoaded => "no page loaded — run /goto first", @@ -1504,7 +1505,7 @@ fn runCommand(self: *Agent, arena: std.mem.Allocator, cmd: Command) browser_tool /// `printToolOutcome`, which lays down the green ● / red ● dot shared with the /// LLM tool-call path. Callers only invoke this for `.tool_call` commands (the /// comment/login/acceptCookies branches take other paths). -fn printCommandResult(self: *Agent, cmd: Command, result: browser_tools.ToolResult) void { +fn printCommandResult(self: *Agent, cmd: Command, result: browser_tools.ToolResult, image: ?Terminal.Image) void { const tc = switch (cmd) { .tool_call => |t| t, else => return, @@ -1513,9 +1514,21 @@ fn printCommandResult(self: *Agent, cmd: Command, result: browser_tools.ToolResu self.printData(result.text); return; } + if (image) |img| return self.terminal.printToolImage(tc.name(), result.text, img); self.terminal.printToolOutcome(tc.name(), result.text, result.is_error); } +/// Rendered once for both the terminal and the conversation. Null when the +/// renderer failed, which it logs. +fn renderImage(arena: std.mem.Allocator, result: browser_tools.ToolResult) ?Terminal.Image { + const prepared = result.image orelse return null; + return .{ + .png_base64 = prepared.base64Alloc(arena) catch return null, + .width = prepared.opts.width, + .height = prepared.opts.height, + }; +} + /// Re-indent JSON for the terminal; MCP keeps renderJson's compact form. fn printData(self: *Agent, text: []const u8) void { var arena = std.heap.ArenaAllocator.init(self.allocator); @@ -1595,6 +1608,7 @@ fn recordSlashToolCall( tool_name: []const u8, args: ?std.json.Value, result: browser_tools.ToolResult, + image: ?Terminal.Image, ) !void { if (self.ai_client == null) return; try self.conversation.ensureSystemPrompt(); @@ -1621,7 +1635,7 @@ fn recordSlashToolCall( .id = try ma.dupe(u8, tool_calls[0].id), .name = try ma.dupe(u8, tool_calls[0].name), .content = content, - .parts = if (result.image) |image| try imageParts(ma, content, &image) else null, + .parts = if (image) |img| try imageParts(ma, content, try ma.dupe(u8, img.png_base64)) else null, .is_error = result.is_error, }; @@ -1941,13 +1955,12 @@ fn toolOutcome(self: *Agent, allocator: std.mem.Allocator, tool_name: []const u8 return .{ .content = content, .is_error = result.is_error, - .parts = if (result.image) |image| try imageParts(allocator, content, &image) else null, + .parts = if (result.image) |image| try imageParts(allocator, content, image.base64Alloc(allocator) catch return error.InternalError) else null, }; } -fn imageParts(arena: std.mem.Allocator, text: []const u8, image: *const lp.screenshot.Prepared) browser_tools.ToolError![]const zenai.provider.ContentPart { - const data = image.base64Alloc(arena) catch return error.InternalError; - return try arena.dupe(zenai.provider.ContentPart, &.{ .{ .text = text }, .{ .image = .{ .data = data, .mime_type = "image/png" } } }); +fn imageParts(arena: std.mem.Allocator, text: []const u8, png_base64: []const u8) browser_tools.ToolError![]const zenai.provider.ContentPart { + return try arena.dupe(zenai.provider.ContentPart, &.{ .{ .text = text }, .{ .image = .{ .data = png_base64, .mime_type = "image/png" } } }); } /// One-shot for `--list-models`: resolve provider+key, fetch chat-capable model diff --git a/src/agent/Spinner.zig b/src/agent/Spinner.zig index 500b55cbc..bcdc398b4 100644 --- a/src/agent/Spinner.zig +++ b/src/agent/Spinner.zig @@ -20,6 +20,7 @@ const std = @import("std"); const lp = @import("lightpanda"); const log = lp.log; const ansi = @import("ansi.zig"); +const tty = @import("tty.zig"); const truncateUtf8 = @import("../string.zig").truncateUtf8; const Spinner = @This(); @@ -310,18 +311,8 @@ fn renderLocked(self: *Spinner) void { _ = std.c.write(std.posix.STDERR_FILENO, (written).ptr, (written).len); } -/// Current terminal width in columns, queried via TIOCGWINSZ on stderr. -/// Null when stderr isn't a tty, the ioctl fails, or the kernel reports 0 -/// (some pseudo-ttys leave the field unset). Cheap enough to call per render -/// frame; picks up resizes without SIGWINCH plumbing. fn columns() ?u16 { - var ws: std.posix.winsize = undefined; - // bitcast via c_uint: on archs where `_IOR` sets the direction bit - // (MIPS/PPC/SPARC), `IOCGWINSZ` exceeds i32 range, so a plain @intCast - // panics; the bitcast preserves the bit pattern. - const req: c_int = @bitCast(@as(c_uint, std.posix.T.IOCGWINSZ)); - const rc = std.c.ioctl(std.posix.STDERR_FILENO, req, &ws); - if (rc != 0 or ws.col == 0) return null; + const ws = tty.windowSize(std.posix.STDERR_FILENO) orelse return null; return ws.col; } diff --git a/src/agent/Terminal.zig b/src/agent/Terminal.zig index 94875f910..67f482bd3 100644 --- a/src/agent/Terminal.zig +++ b/src/agent/Terminal.zig @@ -25,6 +25,8 @@ const Spinner = @import("Spinner.zig"); const md_term = @import("md_term.zig"); const prompt_assist = @import("prompt_assist.zig"); const ansi = @import("ansi.zig"); +const kitty = @import("kitty.zig"); +const tty = @import("tty.zig"); const c = @import("isocline"); const Terminal = @This(); @@ -53,6 +55,8 @@ assist: prompt_assist.State, /// resets everything so fence state can't leak into the next message. Only /// used on the styled (REPL tty) path, hence the placeholder. md_stream: md_term.Stream = .{ .show_table_placeholder = true }, +/// Probed on first use: it costs a round trip to the terminal. +shows_images: ?bool = null, pub const CompletionSource = prompt_assist.CompletionSource; pub const HistoryPaths = prompt_assist.HistoryPaths; @@ -211,6 +215,7 @@ pub fn clearPromptFrame(self: *Terminal) void { } test { + _ = kitty; _ = md_term; _ = prompt_assist; } @@ -290,6 +295,35 @@ pub fn printToolOutcome(self: *Terminal, name: []const u8, text: []const u8, is_ std.debug.print("{s}{s}[result: {s}]{s} {s}{s}\n", .{ ansi.dim, color, name, ansi.reset, truncated, ellipsis }); } +pub fn canShowImage(self: *Terminal) bool { + if (!self.styledOutput()) return false; + if (self.shows_images == null) self.shows_images = kitty.detect(); + return self.shows_images.?; +} + +pub const Image = struct { + png_base64: []const u8, + width: u32, + height: u32, +}; + +// Leaves room for the outcome line and the prompt frame under the image. +const image_max_rows_reserve = 4; + +/// An image the terminal can't show only came because a model wanted it, +/// hence the note. +pub fn printToolImage(self: *Terminal, name: []const u8, text: []const u8, image: Image) void { + self.printToolOutcome(name, text, false); + if (!self.canShowImage()) return self.printDimmed(" sent to the model only; pass `path` to save it", .{}); + + const ws: std.posix.winsize = tty.windowSize(std.posix.STDOUT_FILENO) orelse .{ .row = 24, .col = 80, .xpixel = 0, .ypixel = 0 }; + const cells = kitty.fit(image.width, image.height, ws, ws.row -| image_max_rows_reserve); + var buf: [4096]u8 = undefined; + var fw = std.Io.File.stdout().writerStreaming(lp.io, &buf); + kitty.write(&fw.interface, image.png_base64, cells) catch return; + fw.interface.flush() catch {}; +} + /// Freeze the script spinner into a green bullet for a `/load` run that /// produced no output — mirrors a `/goto` outcome line, swapping the braille /// glyph for a `●`. Only fires when the spinner was shown (REPL + TTY); diff --git a/src/agent/kitty.zig b/src/agent/kitty.zig new file mode 100644 index 000000000..900b4da8a --- /dev/null +++ b/src/agent/kitty.zig @@ -0,0 +1,159 @@ +// Copyright (C) 2023-2026 Lightpanda (Selecy SAS) +// +// Francis Bouvier +// Pierre Tachoire +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +//! Kitty graphics protocol: probe the terminal for support and transmit a +//! PNG for immediate display. Spoken by kitty, ghostty, WezTerm and Konsole. + +const std = @import("std"); +const lp = @import("lightpanda"); +const tty = @import("tty.zig"); + +const apc = "\x1b_G"; +const st = "\x1b\\"; + +/// A 1x1 RGB graphics query followed by Device Attributes. A terminal that +/// speaks the protocol answers the query first; every terminal answers DA, +/// which marks the end of the reply so nothing is left in stdin for the line +/// editor. Multiplexers that don't pass graphics through (tmux) answer DA +/// themselves, so they read as unsupported. +const probe = apc ++ "i=31,s=1,v=1,a=q,t=d,f=24;AAAA" ++ st ++ "\x1b[c"; + +/// One round trip to the terminal, up to 100ms when it never answers; +/// callers cache the result. Stdout being a tty is the caller's check. +pub fn detect() bool { + if (!(std.Io.File.stdin().isTty(lp.io) catch false)) return false; + var raw = tty.Raw.enable() catch return false; + defer raw.restore(); + + if (std.c.write(std.posix.STDOUT_FILENO, probe.ptr, probe.len) != probe.len) return false; + + var buf: [256]u8 = undefined; + var len: usize = 0; + while (len < buf.len) { + const n = std.posix.read(std.posix.STDIN_FILENO, buf[len..]) catch return false; + if (n == 0) break; + len += n; + if (endsDeviceAttributes(buf[0..len])) break; + } + return supported(buf[0..len]); +} + +fn supported(reply: []const u8) bool { + return std.mem.find(u8, reply, apc) != null; +} + +/// DA1 replies `ESC [ ? Ps ; … c`. +fn endsDeviceAttributes(reply: []const u8) bool { + const start = std.mem.findLast(u8, reply, "\x1b[?") orelse return false; + return std.mem.findScalarPos(u8, reply, start, 'c') != null; +} + +pub const Cells = struct { + cols: u16, + rows: u16, +}; + +/// Keeps the aspect ratio and never upscales. Terminals that don't report +/// their pixel size get the usual 1:2 cell. +pub fn fit(width: u32, height: u32, ws: std.posix.winsize, max_rows: u16) Cells { + const cell_w: f32 = if (ws.xpixel > 0) @as(f32, @floatFromInt(ws.xpixel)) / @as(f32, @floatFromInt(ws.col)) else 8; + const cell_h: f32 = if (ws.ypixel > 0) @as(f32, @floatFromInt(ws.ypixel)) / @as(f32, @floatFromInt(ws.row)) else 16; + const w: f32 = @floatFromInt(width); + const h: f32 = @floatFromInt(height); + const box_w = @as(f32, @floatFromInt(ws.col)) * cell_w; + const box_h = @as(f32, @floatFromInt(max_rows)) * cell_h; + const scale = @min(1.0, box_w / w, box_h / h); + return .{ + .cols = @ceil(@max(1.0, w * scale / cell_w)), + .rows = @ceil(@max(1.0, h * scale / cell_h)), + }; +} + +/// The protocol caps a chunk's payload at 4096 bytes; a multiple of 4 keeps +/// base64 groups whole. +const chunk_len = 4096; + +/// `q=2` silences the terminal's acknowledgement, which would otherwise land +/// in stdin. +pub fn write(writer: *std.Io.Writer, png_base64: []const u8, cells: Cells) std.Io.Writer.Error!void { + try writer.print(apc ++ "a=T,f=100,q=2,c={d},r={d},", .{ cells.cols, cells.rows }); + var rest = png_base64; + while (true) { + const n = @min(rest.len, chunk_len); + const more = rest.len > n; + try writer.print("m={d};", .{@intFromBool(more)}); + try writer.writeAll(rest[0..n]); + try writer.writeAll(st); + if (!more) break; + rest = rest[n..]; + try writer.writeAll(apc); + } + try writer.writeByte('\n'); +} + +const testing = std.testing; + +test "kitty: probe reply parsing" { + try testing.expect(supported("\x1b_Gi=31;OK\x1b\\\x1b[?62;c")); + try testing.expect(supported("\x1b_Gi=31;EINVAL:bad\x1b\\\x1b[?1;2c")); + try testing.expect(!supported("\x1b[?1;2c")); + try testing.expect(!supported("")); + + try testing.expect(endsDeviceAttributes("\x1b_Gi=31;OK\x1b\\\x1b[?62;c")); + try testing.expect(endsDeviceAttributes("\x1b[?1;2c")); + try testing.expect(!endsDeviceAttributes("\x1b_Gi=31;OK\x1b\\")); + try testing.expect(!endsDeviceAttributes("\x1b_Gi=31;OK\x1b\\\x1b[?62;")); +} + +test "kitty.fit: fills the width without upscaling" { + const ws: std.posix.winsize = .{ .row = 50, .col = 100, .xpixel = 1000, .ypixel = 1000 }; + // 10x20 cells; a 1280x720 image spans the 1000px width at 720*1000/1280 px tall. + try testing.expectEqual(Cells{ .cols = 100, .rows = 29 }, fit(1280, 720, ws, 48)); + // A small image keeps its size: 300x100 px → 30x5 cells. + try testing.expectEqual(Cells{ .cols = 30, .rows = 5 }, fit(300, 100, ws, 48)); +} + +test "kitty.fit: caps the rows and falls back to 8x16 cells" { + const ws: std.posix.winsize = .{ .row = 50, .col = 100, .xpixel = 1000, .ypixel = 1000 }; + // 1280x4096 into 1000x480 px → scale 480/4096, 150x480 px → 15x24 cells. + try testing.expectEqual(Cells{ .cols = 15, .rows = 24 }, fit(1280, 4096, ws, 24)); + + const unsized: std.posix.winsize = .{ .row = 24, .col = 80, .xpixel = 0, .ypixel = 0 }; + // 640x384 px box; 1280x720 → scale 0.5, 640x360 px → 80x23 cells. + try testing.expectEqual(Cells{ .cols = 80, .rows = 23 }, fit(1280, 720, unsized, 24)); +} + +test "kitty.write: chunks the payload at 4096 bytes" { + var aw: std.Io.Writer.Allocating = .init(testing.allocator); + defer aw.deinit(); + try write(&aw.writer, "AAAA", .{ .cols = 3, .rows = 2 }); + try testing.expectEqualStrings("\x1b_Ga=T,f=100,q=2,c=3,r=2,m=0;AAAA\x1b\\\n", aw.written()); + + aw.clearRetainingCapacity(); + const payload = [_]u8{'A'} ** (chunk_len * 2 + 4); + try write(&aw.writer, &payload, .{ .cols = 80, .rows = 24 }); + const out = aw.written(); + const head = "\x1b_Ga=T,f=100,q=2,c=80,r=24,m=1;"; + try testing.expectEqualStrings(head, out[0..head.len]); + var chunks = std.mem.splitSequence(u8, out, "\x1b\\"); + try testing.expectEqual(head.len + chunk_len, chunks.next().?.len); + try testing.expectEqual("\x1b_Gm=1;".len + chunk_len, chunks.next().?.len); + try testing.expectEqualStrings("\x1b_Gm=0;AAAA", chunks.next().?); + try testing.expectEqualStrings("\n", chunks.next().?); + try testing.expectEqual(null, chunks.next()); +} diff --git a/src/agent/picker.zig b/src/agent/picker.zig index 79879b32a..60449998d 100644 --- a/src/agent/picker.zig +++ b/src/agent/picker.zig @@ -23,6 +23,7 @@ const std = @import("std"); const lp = @import("lightpanda"); const ansi = @import("ansi.zig"); +const tty = @import("tty.zig"); pub fn interactiveTty() bool { const stdin_tty = std.Io.File.stdin().isTty(lp.io) catch false; @@ -100,38 +101,23 @@ const ChoiceState = struct { }; const RawTerminal = struct { - original: std.posix.termios, + raw: tty.Raw, fn enable() error{NotInteractive}!RawTerminal { if (!interactiveTty()) return error.NotInteractive; // A tty that refuses raw mode is non-interactive for our purposes. - const original = std.posix.tcgetattr(std.posix.STDIN_FILENO) catch return error.NotInteractive; - var raw = original; - raw.iflag.BRKINT = false; - raw.iflag.ICRNL = false; - raw.iflag.INPCK = false; - raw.iflag.ISTRIP = false; - raw.iflag.IXON = false; - raw.oflag.OPOST = false; - raw.cflag.CSIZE = .CS8; - raw.lflag.ECHO = false; - raw.lflag.ICANON = false; - raw.lflag.IEXTEN = false; - raw.lflag.ISIG = false; - raw.cc[@intFromEnum(std.c.V.MIN)] = 0; - raw.cc[@intFromEnum(std.c.V.TIME)] = 1; - std.posix.tcsetattr(std.posix.STDIN_FILENO, .FLUSH, raw) catch return error.NotInteractive; + const raw = tty.Raw.enable() catch return error.NotInteractive; // Under `ansi.kitty_disambiguate` (pushed by `Terminal.readLine`), // cursor keys arrive as CSI-u the byte reader can't parse; push the // legacy encoding to force plain arrows. restore() pops back to // whatever the REPL had pushed. _ = std.c.write(std.posix.STDOUT_FILENO, ansi.kitty_legacy.ptr, ansi.kitty_legacy.len); - return .{ .original = original }; + return .{ .raw = raw }; } fn restore(self: *const RawTerminal) void { _ = std.c.write(std.posix.STDOUT_FILENO, ansi.kitty_pop.ptr, ansi.kitty_pop.len); - std.posix.tcsetattr(std.posix.STDIN_FILENO, .FLUSH, self.original) catch {}; + self.raw.restore(); } }; diff --git a/src/agent/tty.zig b/src/agent/tty.zig new file mode 100644 index 000000000..ae319e5cd --- /dev/null +++ b/src/agent/tty.zig @@ -0,0 +1,65 @@ +// Copyright (C) 2023-2026 Lightpanda (Selecy SAS) +// +// Francis Bouvier +// Pierre Tachoire +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +//! Raw mode and size queries on the controlling terminal, for the +//! interactive pieces that run outside isocline's line editor. + +const std = @import("std"); + +/// A read returns as soon as a byte arrives, or empty after 100ms. +pub const Raw = struct { + original: std.posix.termios, + + pub fn enable() !Raw { + const original = try std.posix.tcgetattr(std.posix.STDIN_FILENO); + var raw = original; + raw.iflag.BRKINT = false; + raw.iflag.ICRNL = false; + raw.iflag.INPCK = false; + raw.iflag.ISTRIP = false; + raw.iflag.IXON = false; + raw.oflag.OPOST = false; + raw.cflag.CSIZE = .CS8; + raw.lflag.ECHO = false; + raw.lflag.ICANON = false; + raw.lflag.IEXTEN = false; + raw.lflag.ISIG = false; + raw.cc[@intFromEnum(std.c.V.MIN)] = 0; + raw.cc[@intFromEnum(std.c.V.TIME)] = 1; + try std.posix.tcsetattr(std.posix.STDIN_FILENO, .FLUSH, raw); + return .{ .original = original }; + } + + pub fn restore(self: *const Raw) void { + std.posix.tcsetattr(std.posix.STDIN_FILENO, .FLUSH, self.original) catch {}; + } +}; + +/// The size of the terminal behind `fd`. Null when it isn't a tty, the ioctl +/// fails, or the kernel reports 0 columns (some pseudo-ttys leave the field +/// unset). Cheap enough to call per render frame; picks up resizes without +/// SIGWINCH plumbing. +pub fn windowSize(fd: std.posix.fd_t) ?std.posix.winsize { + var ws: std.posix.winsize = undefined; + // bitcast via c_uint: on archs where `_IOR` sets the direction bit + // (MIPS/PPC/SPARC), `IOCGWINSZ` exceeds i32 range, so a plain @intCast + // panics; the bitcast preserves the bit pattern. + const req: c_int = @bitCast(@as(c_uint, std.posix.T.IOCGWINSZ)); + if (std.c.ioctl(fd, req, &ws) != 0 or ws.col == 0) return null; + return ws; +} diff --git a/src/browser/tools.zig b/src/browser/tools.zig index 55b4444fe..4b6060088 100644 --- a/src/browser/tools.zig +++ b/src/browser/tools.zig @@ -1361,7 +1361,7 @@ fn execScreenshot(arena: std.mem.Allocator, session: *lp.Session, registry: *CDP if (args.path) |path| { if (!isPathSafe(path)) return .{ .text = unsafe_path_message, .is_error = true }; } else if (!inline_image) { - return .{ .text = "pass `path`: this client cannot display an inline image", .is_error = true }; + return .{ .text = "pass `path`: this client can't display images", .is_error = true }; } const page = try ensurePage(session, registry, args.url, args.timeout); const node = try resolveScope(session, registry, page, args.selector, args.backendNodeId);