render: add "shell" option to --strip-mode

Some additional API changes:
- Add strip-mode support to pdf/png generation.
- Add LP.dump which provides greater content gathering capability to CDP,
  exposing most `fetch` dump-related parameters (e.g. format, strip, selector,
  ...)

The new "--strip-mode shell" is designed to try to remove non-content elements
such as the header and footer. The end goal is to use readibility.js test cases
as a baseline, but this isn't a port of readibility.js.

This is just the basic implementation of this, e.g removing a few key tags, e.g.
<header>, <footer> and considering some specific roles.

Even if --strip-mode shell is used, we might decide to stick with a whole dump:
it's better to strip not enough than to strip too much. This currently works by
measuring the ratio of non-link text of the stripped vs unstripped page.
This commit is contained in:
Karl Seguin committed 2026-09-11 06:35:26 +08:00
1 parent c21462cc38
commit cdceca37c3
11 files changed
+447 -27

No files matched your search

+16 -8
View File
@@ -68,7 +68,6 @@ pub const crash_handler = @import("crash_handler.zig");
pub const core_dump = @import("core_dump.zig");
pub var metrics = @import("Metrics.zig"){};
pub const IS_TEST = @import("builtin").is_test;
pub const IS_DEBUG = @import("builtin").mode == .Debug;
@@ -458,13 +457,20 @@ const Binary = union(enum) {
fn prepareBinary(arena: std.mem.Allocator, frame: *Frame, opts: FetchOpts) !Binary {
const root = try dumpRoot(frame, opts.selector);
const strip = RenderTree.resolveStrip(root, opts.dump.strip, frame);
return switch (opts.dump_mode.?) {
.png => .{ .png = try screenshot.preparePng(arena, root, .fromViewport(frame._page.getViewport(), true), frame) },
.pdf => .{ .pdf = try pdf.prepare(arena, root, .{}, frame) },
.png => .{ .png = try screenshot.preparePng(arena, root, pngOpts(frame, strip), frame) },
.pdf => .{ .pdf = try pdf.prepare(arena, root, .{ .strip = strip }, frame) },
else => unreachable,
};
}
fn pngOpts(frame: *Frame, strip: dump.Opts.Strip) screenshot.Opts {
var opts: screenshot.Opts = .fromViewport(frame._page.getViewport(), true);
opts.strip = strip;
return opts;
}
fn dumpRoot(frame: *Frame, selector: ?[]const u8) !*Node {
const document = frame.window._document.asNode();
const sel = selector orelse return document;
@@ -474,21 +480,23 @@ fn dumpRoot(frame: *Frame, selector: ?[]const u8) !*Node {
fn dumpContent(app: *App, mode: Config.DumpFormat, opts: FetchOpts, frame: *Frame, writer: *std.Io.Writer) !void {
const root = try dumpRoot(frame, opts.selector);
var dump_opts = opts.dump;
dump_opts.strip = RenderTree.resolveStrip(root, dump_opts.strip, frame);
switch (mode) {
.html => if (opts.selector == null)
try dump.root(frame.window._document, opts.dump, writer, frame)
try dump.root(frame.window._document, dump_opts, writer, frame)
else
try dump.deep(root, opts.dump, writer, frame),
.markdown => try markdown.dump(root, .{ .max_bytes = opts.dump.max_bytes, .strip = opts.dump.strip }, writer, frame),
try dump.deep(root, dump_opts, writer, frame),
.markdown => try markdown.dump(root, .{ .max_bytes = dump_opts.max_bytes, .strip = dump_opts.strip }, writer, frame),
.png => {
var arena: std.heap.ArenaAllocator = .init(app.allocator);
defer arena.deinit();
_ = try screenshot.png(arena.allocator(), root, .fromViewport(frame._page.getViewport(), true), writer, frame);
_ = try screenshot.png(arena.allocator(), root, pngOpts(frame, dump_opts.strip), writer, frame);
},
.pdf => {
var arena: std.heap.ArenaAllocator = .init(app.allocator);
defer arena.deinit();
try pdf.print(arena.allocator(), root, .{}, writer, frame);
try pdf.print(arena.allocator(), root, .{ .strip = dump_opts.strip }, writer, frame);
},
.semantic_tree, .semantic_tree_text => {
var registry = NodeRegistry.init(app.allocator);