mirror/browser
1
0 Fork 0
mirror of https://github.com/lightpanda-io/browser.git synced 2026-09-18 09:54:36 -04:00
Code Issues Packages Projects Releases Wiki Activity

dump: take max_bytes in Opts

Browse Source 
Review feedback: markdown caps its own output through Opts.max_bytes while
the html dump left it to each caller, so the html tool and fetch each
carried a copy of the LimitedWriter wrapping. root and deep now do it,
with a no-cap fast path so innerHTML/outerHTML are unaffected.
This commit is contained in:
Adrià Arrufat committed 2026-08-26 15:44:48 +02:00
1 parent 6811cc8226
commit 56241e26b8
4 files changed
+49 -40
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/browser/dump.zig
+36 -2
View File
@@ -19,6 +19,7 @@
const std = @import("std");
const lp = @import("lightpanda");
const Frame = @import("Frame.zig");
const LimitedWriter = @import("../LimitedWriter.zig");
const Node = @import("webapi/Node.zig");
const Slot = @import("webapi/element/html/Slot.zig");
const IFrame = @import("webapi/element/html/IFrame.zig");
@@ -28,6 +29,9 @@ pub const Opts = struct {
with_frames: bool = false,
strip: Opts.Strip = .{},
shadow: Opts.Shadow = .rendered,
/// Soft cap: output is cut at a UTF-8 boundary and a truncation marker
/// appended.
max_bytes: ?u32 = null,
pub const Strip = packed struct(u4) {
js: bool = false,
@@ -59,6 +63,16 @@ pub const Opts = struct {
};
pub fn root(doc: *Node.Document, opts: Opts, writer: *std.Io.Writer, frame: *Frame) !void {
if (opts.max_bytes == null) return rootUncapped(doc, opts, writer, frame);
var lw: LimitedWriter = .init(writer, opts.max_bytes);
rootUncapped(doc, opts, &lw.writer, frame) catch |err| {
if (!lw.truncated) return err;
try writer.writeAll(LimitedWriter.truncation_marker);
};
}
fn rootUncapped(doc: *Node.Document, opts: Opts, writer: *std.Io.Writer, frame: *Frame) !void {
if (doc.is(Node.Document.HTMLDocument)) |html_doc| {
blk: {
// Ideally we just render the doctype which is part of the document
@@ -80,11 +94,17 @@ pub fn root(doc: *Node.Document, opts: Opts, writer: *std.Io.Writer, frame: *Fra
}
}
return deep(doc.asNode(), opts, writer, frame);
return _deep(doc.asNode(), opts, false, writer, frame);
}
pub fn deep(node: *Node, opts: Opts, writer: *std.Io.Writer, frame: *Frame) error{WriteFailed}!void {
return _deep(node, opts, false, writer, frame);
if (opts.max_bytes == null) return _deep(node, opts, false, writer, frame);
var lw: LimitedWriter = .init(writer, opts.max_bytes);
_deep(node, opts, false, &lw.writer, frame) catch |err| {
if (!lw.truncated) return err;
try writer.writeAll(LimitedWriter.truncation_marker);
};
}
fn _deep(node: *Node, opts: Opts, comptime force_slot: bool, writer: *std.Io.Writer, frame: *Frame) error{WriteFailed}!void {
@@ -499,6 +519,20 @@ test "dump: strip.ui removes css plus visual elements" {
);
}
test "dump: max_bytes truncates with a marker" {
var page = try testing.pageTest("dump.html", .{});
defer page.close();
const frame = page.frame().?;
var aw: std.Io.Writer.Allocating = .init(testing.arena_allocator);
try root(frame.window._document, .{ .max_bytes = 24 }, &aw.writer, frame);
try testing.expectString("<!DOCTYPE html>\n<html><h" ++ LimitedWriter.truncation_marker, aw.written());
aw.clearRetainingCapacity();
try deep(frame.window._document.asNode().lastChild().?, .{ .max_bytes = 6 }, &aw.writer, frame);
try testing.expectString("<html>" ++ LimitedWriter.truncation_marker, aw.written());
}
test "dump: strip.invisible removes author display:none elements" {
try expectDump(.{ .strip = .{ .invisible = true } },
\\<!DOCTYPE html>
src/browser/tools.zig
+7 -22
View File
@@ -28,7 +28,6 @@ const keenable = zenai.search.keenable;
const DOMNode = @import("webapi/Node.zig");
const CDPNode = @import("../cdp/Node.zig");
const LimitedWriter = @import("../LimitedWriter.zig");
const Selector = @import("webapi/selector/Selector.zig");
/// Conventions any LLM driving Lightpanda should follow. The standalone
@@ -1287,32 +1286,18 @@ fn execHtml(arena: std.mem.Allocator, session: *lp.Session, registry: *CDPNode.R
const args = try parseArgsOrDefault(HtmlParams, arena, arguments);
const page = try ensurePage(session, registry, args.url, args.timeout);
const opts: lp.dump.Opts = .{ .strip = args.strip, .max_bytes = args.maxBytes };
var aw: std.Io.Writer.Allocating = .init(arena);
var lw: LimitedWriter = .init(&aw.writer, args.maxBytes);
dumpHtml(session, registry, page, args, &lw.writer) catch |err| switch (err) {
error.WriteFailed => if (!lw.truncated) return ToolError.InternalError,
else => |e| return e,
};
if (lw.truncated) {
aw.writer.writeAll(LimitedWriter.truncation_marker) catch return ToolError.InternalError;
}
return aw.written();
}
fn dumpHtml(session: *lp.Session, registry: *CDPNode.Registry, page: *lp.Frame, args: HtmlParams, writer: *std.Io.Writer) (ToolError || error{WriteFailed})!void {
const opts: lp.dump.Opts = .{ .strip = args.strip };
if (args.selector) |sel| {
const resolved = try resolveBySelector(session, sel);
return lp.dump.deep(resolved.node, opts, writer, resolved.page);
}
if (args.backendNodeId) |nid| {
lp.dump.deep(resolved.node, opts, &aw.writer, resolved.page) catch return ToolError.InternalError;
} else if (args.backendNodeId) |nid| {
const resolved = try resolveNodeAndPage(session, registry, nid);
return lp.dump.deep(resolved.node, opts, writer, resolved.page);
lp.dump.deep(resolved.node, opts, &aw.writer, resolved.page) catch return ToolError.InternalError;
} else {
lp.dump.root(page.document, opts, &aw.writer, page) catch return ToolError.InternalError;
}
return lp.dump.root(page.document, opts, writer, page) catch |err| switch (err) {
error.WriteFailed => error.WriteFailed,
else => ToolError.InternalError,
};
return aw.written();
}
fn execLinks(arena: std.mem.Allocator, session: *lp.Session, registry: *CDPNode.Registry, arguments: ?std.json.Value) ToolError![]const u8 {
src/lightpanda.zig
+5 -15
View File
@@ -40,7 +40,6 @@ pub const dump = @import("browser/dump.zig");
pub const markdown = @import("browser/markdown.zig");
pub const screenshot = @import("browser/screenshot.zig");
pub const Base64Writer = @import("Base64Writer.zig");
const LimitedWriter = @import("LimitedWriter.zig");
const Selector = @import("browser/webapi/selector/Selector.zig");
const Node = @import("browser/webapi/Node.zig");
pub const SemanticTree = @import("SemanticTree.zig");
@@ -191,8 +190,6 @@ pub const FetchOpts = struct {
dump_mode: ?Config.DumpFormat = null,
/// Dump only the first match instead of the document.
selector: ?[:0]const u8 = null,
/// html and markdown only.
max_bytes: ?u32 = null,
/// Any page with an HTTP status >= 400 fails the fetch with `error.HttpError`.
fail_on_http_error: bool = false,
writer: ?*std.Io.Writer = null,
@@ -438,18 +435,11 @@ fn dumpRoot(frame: *Frame, selector: ?[]const u8) !*Node {
fn dumpContent(app: *App, mode: Config.DumpFormat, opts: FetchOpts, frame: *Frame, writer: *std.Io.Writer) !void {
const root = try dumpRoot(frame, opts.selector);
switch (mode) {
.html => {
var lw: LimitedWriter = .init(writer, opts.max_bytes);
const result = if (opts.selector == null)
dump.root(frame.window._document, opts.dump, &lw.writer, frame)
else
dump.deep(root, opts.dump, &lw.writer, frame);
result catch |err| {
if (!lw.truncated) return err;
try writer.writeAll(LimitedWriter.truncation_marker);
};
},
.markdown => try markdown.dump(root, .{ .max_bytes = opts.max_bytes, .strip = opts.dump.strip }, writer, frame),
.html => if (opts.selector == null)
try dump.root(frame.window._document, opts.dump, writer, frame)
else
try dump.deep(root, opts.dump, writer, frame),
.markdown => try markdown.dump(root, .{ .max_bytes = opts.dump.max_bytes, .strip = opts.dump.strip }, writer, frame),
.png => {
var arena: std.heap.ArenaAllocator = .init(app.allocator);
defer arena.deinit();
src/main.zig
+1 -1
View File
@@ -177,12 +177,12 @@ fn run(allocator: Allocator, main_arena: Allocator, proc_args: std.process.Args)
.wait_selector = opts.wait_selector,
.dump_mode = opts.dump,
.selector = opts.dump_selector,
.max_bytes = opts.dump_max_bytes,
.fail_on_http_error = opts.fail_on_http_error,
.dump = .{
.strip = opts.strip_mode,
.with_base = opts.with_base,
.with_frames = opts.with_frames,
.max_bytes = opts.dump_max_bytes,
},
.json = opts.json,
};