mirror/browser
1
0 Fork 0
mirror of https://github.com/lightpanda-io/browser.git synced 2026-10-09 04:42:30 -04:00
Code Issues Packages Projects Releases Wiki Activity

render: add "clutter" option to --strip-mode

Browse Source 
Adds a `clutter` option to --strip-mode. This is based on readability.js. It
isn't a direct port (e.g. it doesn't strop bylines). It fallsback to `shell` if
it strips too much (and shell itself can fallback to not stripping anything).
But clutter rarely fallback to shell, only when a page is very small or when
it strips out _a lot_.

Also expanded shell to look at class names and ids.
This commit is contained in:
Karl Seguin committed 2026-09-11 06:36:48 +08:00
1 parent cdceca37c3
commit 5bc4f7482c
13 files changed
+1165 -150
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/lightpanda.zig
+43 -45
View File
@@ -19,53 +19,55 @@
const std = @import("std");
pub const log = @import("log.zig");
pub const datetime = @import("datetime.zig");
pub const mcp = @import("mcp.zig");
pub const App = @import("App.zig");
pub const Arena = @import("Arena.zig");
pub const ArenaPool = @import("ArenaPool.zig");
pub const Network = @import("network/Network.zig");
pub const Config = @import("Config.zig");
pub const cookies = @import("cookies.zig");
pub const datetime = @import("datetime.zig");
pub const core_dump = @import("core_dump.zig");
pub const ArenaPool = @import("ArenaPool.zig");
pub const build_config = @import("build_config");
pub const String = @import("string.zig").String;
pub const Notification = @import("Notification.zig");
pub const ToolSession = @import("ToolSession.zig");
pub const Notification = @import("Notification.zig");
pub const Server = @import("server/Server.zig");
pub const Base64Writer = @import("Base64Writer.zig");
pub const SemanticTree = @import("SemanticTree.zig");
pub const NodeRegistry = @import("NodeRegistry.zig");
pub const crash_handler = @import("crash_handler.zig");
pub const Network = @import("network/Network.zig");
pub const HttpClient = @import("network/HttpClient.zig");
pub const pdf = @import("browser/pdf.zig");
pub const URL = @import("browser/URL.zig");
pub const dump = @import("browser/dump.zig");
pub const Page = @import("browser/Page.zig");
pub const Frame = @import("browser/Frame.zig");
pub const forms = @import("browser/forms.zig");
pub const links = @import("browser/links.zig");
pub const tools = @import("browser/tools.zig");
pub const actions = @import("browser/actions.zig");
pub const Browser = @import("browser/Browser.zig");
pub const Session = @import("browser/Session.zig");
pub const markdown = @import("browser/markdown.zig");
pub const screenshot = @import("browser/screenshot.zig");
pub const RenderTree = @import("browser/RenderTree.zig");
pub const interactive = @import("browser/interactive.zig");
pub const structured_data = @import("browser/structured_data.zig");
pub const GlobalScope = @import("browser/global_scope.zig").GlobalScope;
pub const js = @import("browser/js/js.zig");
pub const dump = @import("browser/dump.zig");
pub const markdown = @import("browser/markdown.zig");
pub const screenshot = @import("browser/screenshot.zig");
pub const pdf = @import("browser/pdf.zig");
pub const Base64Writer = @import("Base64Writer.zig");
const Selector = @import("browser/webapi/selector/Selector.zig");
const Node = @import("browser/webapi/Node.zig");
pub const SemanticTree = @import("SemanticTree.zig");
pub const NodeRegistry = @import("NodeRegistry.zig");
pub const interactive = @import("browser/interactive.zig");
pub const links = @import("browser/links.zig");
pub const forms = @import("browser/forms.zig");
pub const actions = @import("browser/actions.zig");
pub const structured_data = @import("browser/structured_data.zig");
pub const tools = @import("browser/tools.zig");
pub const HttpClient = @import("network/HttpClient.zig");
const Selector = @import("browser/webapi/selector/Selector.zig");
pub const mcp = @import("mcp.zig");
pub const Agent = @import("agent/Agent.zig");
pub const Command = @import("script/command.zig").Command;
pub const skill = @import("script/skill.zig");
pub const Recorder = @import("script/Recorder.zig");
pub const Runtime = @import("script/Runtime.zig");
pub const Schema = @import("script/Schema.zig");
pub const skill = @import("script/skill.zig");
pub const cookies = @import("cookies.zig");
pub const build_config = @import("build_config");
pub const crash_handler = @import("crash_handler.zig");
pub const core_dump = @import("core_dump.zig");
pub const Command = @import("script/command.zig").Command;
pub var metrics = @import("Metrics.zig"){};
pub const IS_TEST = @import("builtin").is_test;
@@ -456,19 +458,16 @@ const Binary = union(enum) {
};
fn prepareBinary(arena: std.mem.Allocator, frame: *Frame, opts: FetchOpts) !Binary {
const root = try dumpRoot(frame, opts.selector);
const strip = RenderTree.resolveStrip(root, opts.dump.strip, frame);
const state = try RenderTree.resolve(arena, try dumpRoot(frame, opts.selector), opts.dump.strip, frame);
return switch (opts.dump_mode.?) {
.png => .{ .png = try screenshot.preparePng(arena, root, pngOpts(frame, strip), frame) },
.pdf => .{ .pdf = try pdf.prepare(arena, root, .{ .strip = strip }, frame) },
.png => .{ .png = try screenshot.preparePng(arena, state, pngOpts(frame), frame) },
.pdf => .{ .pdf = try pdf.prepare(arena, state, .{}, frame) },
else => unreachable,
};
}
fn pngOpts(frame: *Frame, strip: dump.Opts.Strip) screenshot.Opts {
var opts: screenshot.Opts = .fromViewport(frame._page.getViewport(), true);
opts.strip = strip;
return opts;
fn pngOpts(frame: *Frame) screenshot.Opts {
return .fromViewport(frame._page.getViewport(), true);
}
fn dumpRoot(frame: *Frame, selector: ?[]const u8) !*Node {
@@ -479,31 +478,30 @@ fn dumpRoot(frame: *Frame, selector: ?[]const u8) !*Node {
}
fn dumpContent(app: *App, mode: Config.DumpFormat, opts: FetchOpts, frame: *Frame, writer: *std.Io.Writer) !void {
const root = try dumpRoot(frame, opts.selector);
var arena: std.heap.ArenaAllocator = .init(app.allocator);
defer arena.deinit();
const state = try RenderTree.resolve(arena.allocator(), try dumpRoot(frame, opts.selector), opts.dump.strip, frame);
var dump_opts = opts.dump;
dump_opts.strip = RenderTree.resolveStrip(root, dump_opts.strip, frame);
dump_opts.strip = state.strip;
dump_opts.pruned = state.pruned;
switch (mode) {
.html => if (opts.selector == null)
try dump.root(frame.window._document, dump_opts, writer, frame)
else
try dump.deep(root, dump_opts, writer, frame),
.markdown => try markdown.dump(root, .{ .max_bytes = dump_opts.max_bytes, .strip = dump_opts.strip }, writer, frame),
try dump.deep(state.root, dump_opts, writer, frame),
.markdown => try markdown.dump(state, .{ .max_bytes = dump_opts.max_bytes }, writer, frame),
.png => {
var arena: std.heap.ArenaAllocator = .init(app.allocator);
defer arena.deinit();
_ = try screenshot.png(arena.allocator(), root, pngOpts(frame, dump_opts.strip), writer, frame);
_ = try screenshot.png(arena.allocator(), state, pngOpts(frame), writer, frame);
},
.pdf => {
var arena: std.heap.ArenaAllocator = .init(app.allocator);
defer arena.deinit();
try pdf.print(arena.allocator(), root, .{ .strip = dump_opts.strip }, writer, frame);
try pdf.print(arena.allocator(), state, .{}, writer, frame);
},
.semantic_tree, .semantic_tree_text => {
var registry = NodeRegistry.init(app.allocator);
defer registry.deinit();
const st: SemanticTree = .{
.dom_node = root,
.dom_node = state.root,
.registry = &registry,
.frame = frame,
.arena = frame.call_arena,