From 56241e26b84f34d47fd6482f4e96435792c686bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A0=20Arrufat?= Date: Wed, 26 Aug 2026 15:44:48 +0200 Subject: [PATCH] dump: take max_bytes in Opts Review feedback: markdown caps its own output through Opts.max_bytes while the html dump left it to each caller, so the html tool and fetch each carried a copy of the LimitedWriter wrapping. root and deep now do it, with a no-cap fast path so innerHTML/outerHTML are unaffected. --- src/browser/dump.zig | 38 ++++++++++++++++++++++++++++++++++++-- src/browser/tools.zig | 29 +++++++---------------------- src/lightpanda.zig | 20 +++++--------------- src/main.zig | 2 +- 4 files changed, 49 insertions(+), 40 deletions(-) diff --git a/src/browser/dump.zig b/src/browser/dump.zig index 89bd6fd64..b8cc01fe6 100644 --- a/src/browser/dump.zig +++ b/src/browser/dump.zig @@ -19,6 +19,7 @@ const std = @import("std"); const lp = @import("lightpanda"); const Frame = @import("Frame.zig"); +const LimitedWriter = @import("../LimitedWriter.zig"); const Node = @import("webapi/Node.zig"); const Slot = @import("webapi/element/html/Slot.zig"); const IFrame = @import("webapi/element/html/IFrame.zig"); @@ -28,6 +29,9 @@ pub const Opts = struct { with_frames: bool = false, strip: Opts.Strip = .{}, shadow: Opts.Shadow = .rendered, + /// Soft cap: output is cut at a UTF-8 boundary and a truncation marker + /// appended. + max_bytes: ?u32 = null, pub const Strip = packed struct(u4) { js: bool = false, @@ -59,6 +63,16 @@ pub const Opts = struct { }; pub fn root(doc: *Node.Document, opts: Opts, writer: *std.Io.Writer, frame: *Frame) !void { + if (opts.max_bytes == null) return rootUncapped(doc, opts, writer, frame); + + var lw: LimitedWriter = .init(writer, opts.max_bytes); + rootUncapped(doc, opts, &lw.writer, frame) catch |err| { + if (!lw.truncated) return err; + try writer.writeAll(LimitedWriter.truncation_marker); + }; +} + +fn rootUncapped(doc: *Node.Document, opts: Opts, writer: *std.Io.Writer, frame: *Frame) !void { if (doc.is(Node.Document.HTMLDocument)) |html_doc| { blk: { // Ideally we just render the doctype which is part of the document @@ -80,11 +94,17 @@ pub fn root(doc: *Node.Document, opts: Opts, writer: *std.Io.Writer, frame: *Fra } } - return deep(doc.asNode(), opts, writer, frame); + return _deep(doc.asNode(), opts, false, writer, frame); } pub fn deep(node: *Node, opts: Opts, writer: *std.Io.Writer, frame: *Frame) error{WriteFailed}!void { - return _deep(node, opts, false, writer, frame); + if (opts.max_bytes == null) return _deep(node, opts, false, writer, frame); + + var lw: LimitedWriter = .init(writer, opts.max_bytes); + _deep(node, opts, false, &lw.writer, frame) catch |err| { + if (!lw.truncated) return err; + try writer.writeAll(LimitedWriter.truncation_marker); + }; } fn _deep(node: *Node, opts: Opts, comptime force_slot: bool, writer: *std.Io.Writer, frame: *Frame) error{WriteFailed}!void { @@ -499,6 +519,20 @@ test "dump: strip.ui removes css plus visual elements" { ); } +test "dump: max_bytes truncates with a marker" { + var page = try testing.pageTest("dump.html", .{}); + defer page.close(); + const frame = page.frame().?; + + var aw: std.Io.Writer.Allocating = .init(testing.arena_allocator); + try root(frame.window._document, .{ .max_bytes = 24 }, &aw.writer, frame); + try testing.expectString("\n" ++ LimitedWriter.truncation_marker, aw.written()); +} + test "dump: strip.invisible removes author display:none elements" { try expectDump(.{ .strip = .{ .invisible = true } }, \\ diff --git a/src/browser/tools.zig b/src/browser/tools.zig index 7862e7874..09306e082 100644 --- a/src/browser/tools.zig +++ b/src/browser/tools.zig @@ -28,7 +28,6 @@ const keenable = zenai.search.keenable; const DOMNode = @import("webapi/Node.zig"); const CDPNode = @import("../cdp/Node.zig"); -const LimitedWriter = @import("../LimitedWriter.zig"); const Selector = @import("webapi/selector/Selector.zig"); /// Conventions any LLM driving Lightpanda should follow. The standalone @@ -1287,32 +1286,18 @@ fn execHtml(arena: std.mem.Allocator, session: *lp.Session, registry: *CDPNode.R const args = try parseArgsOrDefault(HtmlParams, arena, arguments); const page = try ensurePage(session, registry, args.url, args.timeout); + const opts: lp.dump.Opts = .{ .strip = args.strip, .max_bytes = args.maxBytes }; var aw: std.Io.Writer.Allocating = .init(arena); - var lw: LimitedWriter = .init(&aw.writer, args.maxBytes); - dumpHtml(session, registry, page, args, &lw.writer) catch |err| switch (err) { - error.WriteFailed => if (!lw.truncated) return ToolError.InternalError, - else => |e| return e, - }; - if (lw.truncated) { - aw.writer.writeAll(LimitedWriter.truncation_marker) catch return ToolError.InternalError; - } - return aw.written(); -} - -fn dumpHtml(session: *lp.Session, registry: *CDPNode.Registry, page: *lp.Frame, args: HtmlParams, writer: *std.Io.Writer) (ToolError || error{WriteFailed})!void { - const opts: lp.dump.Opts = .{ .strip = args.strip }; if (args.selector) |sel| { const resolved = try resolveBySelector(session, sel); - return lp.dump.deep(resolved.node, opts, writer, resolved.page); - } - if (args.backendNodeId) |nid| { + lp.dump.deep(resolved.node, opts, &aw.writer, resolved.page) catch return ToolError.InternalError; + } else if (args.backendNodeId) |nid| { const resolved = try resolveNodeAndPage(session, registry, nid); - return lp.dump.deep(resolved.node, opts, writer, resolved.page); + lp.dump.deep(resolved.node, opts, &aw.writer, resolved.page) catch return ToolError.InternalError; + } else { + lp.dump.root(page.document, opts, &aw.writer, page) catch return ToolError.InternalError; } - return lp.dump.root(page.document, opts, writer, page) catch |err| switch (err) { - error.WriteFailed => error.WriteFailed, - else => ToolError.InternalError, - }; + return aw.written(); } fn execLinks(arena: std.mem.Allocator, session: *lp.Session, registry: *CDPNode.Registry, arguments: ?std.json.Value) ToolError![]const u8 { diff --git a/src/lightpanda.zig b/src/lightpanda.zig index fbbaf9010..148bf7a38 100644 --- a/src/lightpanda.zig +++ b/src/lightpanda.zig @@ -40,7 +40,6 @@ pub const dump = @import("browser/dump.zig"); pub const markdown = @import("browser/markdown.zig"); pub const screenshot = @import("browser/screenshot.zig"); pub const Base64Writer = @import("Base64Writer.zig"); -const LimitedWriter = @import("LimitedWriter.zig"); const Selector = @import("browser/webapi/selector/Selector.zig"); const Node = @import("browser/webapi/Node.zig"); pub const SemanticTree = @import("SemanticTree.zig"); @@ -191,8 +190,6 @@ pub const FetchOpts = struct { dump_mode: ?Config.DumpFormat = null, /// Dump only the first match instead of the document. selector: ?[:0]const u8 = null, - /// html and markdown only. - max_bytes: ?u32 = null, /// Any page with an HTTP status >= 400 fails the fetch with `error.HttpError`. fail_on_http_error: bool = false, writer: ?*std.Io.Writer = null, @@ -438,18 +435,11 @@ fn dumpRoot(frame: *Frame, selector: ?[]const u8) !*Node { fn dumpContent(app: *App, mode: Config.DumpFormat, opts: FetchOpts, frame: *Frame, writer: *std.Io.Writer) !void { const root = try dumpRoot(frame, opts.selector); switch (mode) { - .html => { - var lw: LimitedWriter = .init(writer, opts.max_bytes); - const result = if (opts.selector == null) - dump.root(frame.window._document, opts.dump, &lw.writer, frame) - else - dump.deep(root, opts.dump, &lw.writer, frame); - result catch |err| { - if (!lw.truncated) return err; - try writer.writeAll(LimitedWriter.truncation_marker); - }; - }, - .markdown => try markdown.dump(root, .{ .max_bytes = opts.max_bytes, .strip = opts.dump.strip }, writer, frame), + .html => if (opts.selector == null) + try dump.root(frame.window._document, opts.dump, writer, frame) + else + try dump.deep(root, opts.dump, writer, frame), + .markdown => try markdown.dump(root, .{ .max_bytes = opts.dump.max_bytes, .strip = opts.dump.strip }, writer, frame), .png => { var arena: std.heap.ArenaAllocator = .init(app.allocator); defer arena.deinit(); diff --git a/src/main.zig b/src/main.zig index 7b68b7e6a..80e40a802 100644 --- a/src/main.zig +++ b/src/main.zig @@ -177,12 +177,12 @@ fn run(allocator: Allocator, main_arena: Allocator, proc_args: std.process.Args) .wait_selector = opts.wait_selector, .dump_mode = opts.dump, .selector = opts.dump_selector, - .max_bytes = opts.dump_max_bytes, .fail_on_http_error = opts.fail_on_http_error, .dump = .{ .strip = opts.strip_mode, .with_base = opts.with_base, .with_frames = opts.with_frames, + .max_bytes = opts.dump_max_bytes, }, .json = opts.json, };