mirror of
https://github.com/lightpanda-io/browser.git
synced 2026-09-14 15:03:27 -04:00
render: add "clutter" option to --strip-mode
Adds a `clutter` option to --strip-mode. This is based on readability.js. It isn't a direct port (e.g. it doesn't strop bylines). It fallsback to `shell` if it strips too much (and shell itself can fallback to not stripping anything). But clutter rarely fallback to shell, only when a page is very small or when it strips out _a lot_. Also expanded shell to look at class names and ids.
This commit is contained in:
13 files changed
+1165
-150
No files matched your search
+51
-13
@@ -28,16 +28,24 @@ const TreeWalker = @import("webapi/TreeWalker.zig");
|
||||
const Slot = @import("webapi/element/html/Slot.zig");
|
||||
|
||||
const dump_html = @import("dump.zig");
|
||||
const clutter = @import("clutter.zig");
|
||||
const isAllWhitespace = @import("../string.zig").isAllWhitespace;
|
||||
|
||||
const log = lp.log;
|
||||
pub const Strip = dump_html.Opts.Strip;
|
||||
pub const PruneSet = std.AutoHashMapUnmanaged(*Node, void);
|
||||
|
||||
const RenderTree = @This();
|
||||
|
||||
// What / how we're going to render.
|
||||
pub const State = struct {
|
||||
root: *Node, // the not to start from
|
||||
strip: Strip = .{}, // the strip flag we'll use
|
||||
pruned: ?*const PruneSet = null, // the nodes that'll get pruned
|
||||
};
|
||||
|
||||
state: State,
|
||||
frame: *Frame,
|
||||
root: *Node,
|
||||
strip: Strip = .{},
|
||||
|
||||
pub const Child = struct {
|
||||
node: *Node,
|
||||
@@ -121,6 +129,11 @@ pub fn classify(self: *const RenderTree, node: *Node, opts: ClassifyOpts) ?Child
|
||||
return .{ .node = node, .what = .{ .element = d }, .separated = false };
|
||||
}
|
||||
const text_node = node.is(Node.CData.Text) orelse return null;
|
||||
if (self.state.pruned) |set| {
|
||||
if (set.contains(node)) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
var text = text_node.ownData();
|
||||
if (opts.boxed) {
|
||||
text = std.mem.trim(u8, text, &std.ascii.whitespace);
|
||||
@@ -140,10 +153,14 @@ pub fn classify(self: *const RenderTree, node: *Node, opts: ClassifyOpts) ?Child
|
||||
|
||||
fn display(self: *const RenderTree, el: *Element, is_slotted: bool) ?StyleManager.Display {
|
||||
const d = visibleDisplay(el, self.frame) orelse {
|
||||
if (el.asNode() != self.root) return null;
|
||||
if (el.asNode() != self.state.root) {
|
||||
return null;
|
||||
}
|
||||
return .other;
|
||||
};
|
||||
if (dump_html.shouldStripElement(el, self.strip, self.frame)) return null;
|
||||
if (dump_html.shouldStripElement(el, self.state.strip, self.state.pruned, self.frame)) {
|
||||
return null;
|
||||
}
|
||||
if (!is_slotted and el.getSlot() != null) return null;
|
||||
return d;
|
||||
}
|
||||
@@ -219,11 +236,32 @@ pub fn isStandaloneAnchor(el: *Element, frame: *Frame) bool {
|
||||
return true;
|
||||
}
|
||||
|
||||
/// Decides once, before rendering, what a dump of `root` renders: the strip
|
||||
/// bits that survive their safeguards and, for clutter, the prune set.
|
||||
/// Every dump entry point calls this and renders the result. The prune set
|
||||
/// lives in `allocator` for as long as the dump.
|
||||
pub fn resolve(allocator: std.mem.Allocator, root: *Node, requested_strip: Strip, frame: *Frame) !State {
|
||||
var strip = requested_strip;
|
||||
if (strip.clutter) {
|
||||
// The shell strip is the floor the selection stands on.
|
||||
strip.shell = true;
|
||||
strip.invisible = true;
|
||||
}
|
||||
strip = resolveShell(root, strip, frame);
|
||||
if (strip.clutter) {
|
||||
if (try clutter.select(allocator, root, strip, frame)) |pruned| {
|
||||
return .{ .root = root, .strip = strip, .pruned = pruned };
|
||||
}
|
||||
strip.clutter = false;
|
||||
}
|
||||
return .{ .root = root, .strip = strip };
|
||||
}
|
||||
|
||||
/// Shell stripping is undone when it would remove most of the content. Better
|
||||
/// to leave too much in than to strip too muchout. Non-link text is
|
||||
/// the measure (nav and footer text is mostly links); a page with none is
|
||||
/// judged on all of its text.
|
||||
pub fn resolveStrip(root: *Node, strip: Strip, frame: *Frame) Strip {
|
||||
fn resolveShell(root: *Node, strip: Strip, frame: *Frame) Strip {
|
||||
if (strip.shell == false) {
|
||||
return strip;
|
||||
}
|
||||
@@ -232,7 +270,7 @@ pub fn resolveStrip(root: *Node, strip: Strip, frame: *Frame) Strip {
|
||||
render_with_shell.shell = false;
|
||||
|
||||
var m: Measure = .{};
|
||||
const tree: RenderTree = .{ .frame = frame, .root = root, .strip = render_with_shell };
|
||||
const tree: RenderTree = .{ .frame = frame, .state = .{ .root = root, .strip = render_with_shell } };
|
||||
tree.measure(root, .{}, &m);
|
||||
|
||||
const total, const shell = if (m.prose > 0) .{ m.prose, m.shell_prose } else .{ m.all, m.shell_all };
|
||||
@@ -357,22 +395,22 @@ pub fn analyzeContent(root: *Node, frame: *Frame) ContentInfo {
|
||||
const testing = @import("../testing.zig");
|
||||
|
||||
test "RenderTree: resolveStrip keeps shell when the content holds the text" {
|
||||
try testing.expectEqual(true, try resolveShell(
|
||||
try testing.expectEqual(true, try shellSurvives(
|
||||
\\<nav><a href="/">Home</a><a href="/about">About us</a><a href="/blog">Blog</a></nav><main><p>Some article text.</p></main><footer>Copyright</footer>
|
||||
));
|
||||
}
|
||||
|
||||
test "RenderTree: resolveStrip undoes shell when the shell holds the text" {
|
||||
try testing.expectEqual(false, try resolveShell(
|
||||
try testing.expectEqual(false, try shellSurvives(
|
||||
\\<nav><p>All of the text on this page lives inside a nav element.</p></nav><main>hi</main>
|
||||
));
|
||||
}
|
||||
|
||||
test "RenderTree: resolveStrip judges an all-link page on its links" {
|
||||
try testing.expectEqual(false, try resolveShell(
|
||||
try testing.expectEqual(false, try shellSurvives(
|
||||
\\<nav><a href="/1">one</a><a href="/2">two</a><a href="/3">three</a></nav><p><a href="/x">x</a></p>
|
||||
));
|
||||
try testing.expectEqual(true, try resolveShell(
|
||||
try testing.expectEqual(true, try shellSurvives(
|
||||
\\<nav><a href="/1">one</a></nav><p><a href="/x">a longer list of links</a><a href="/y">and another</a></p>
|
||||
));
|
||||
}
|
||||
@@ -380,12 +418,12 @@ test "RenderTree: resolveStrip judges an all-link page on its links" {
|
||||
test "RenderTree: resolveStrip ignores what other strip bits already drop" {
|
||||
// The script text is not content; without strip.js it would tip the
|
||||
// balance toward keeping the shell.
|
||||
try testing.expectEqual(false, try resolveShell(
|
||||
try testing.expectEqual(false, try shellSurvives(
|
||||
\\<nav><p>All of the text on this page lives inside a nav element.</p></nav><main>hi<script>var a_very_long_script_body_that_is_not_content = 1;</script></main>
|
||||
));
|
||||
}
|
||||
|
||||
fn resolveShell(html: []const u8) !bool {
|
||||
fn shellSurvives(html: []const u8) !bool {
|
||||
const frame = try testing.createFrame();
|
||||
defer testing.test_session.closeAllPages();
|
||||
|
||||
@@ -393,7 +431,7 @@ fn resolveShell(html: []const u8) !bool {
|
||||
const div = try doc.createElement("div", null, frame);
|
||||
try Frame.parse.htmlAsChildren(frame, div.asNode(), html);
|
||||
|
||||
const strip = resolveStrip(div.asNode(), .{ .js = true, .shell = true }, frame);
|
||||
const strip = resolveShell(div.asNode(), .{ .js = true, .shell = true }, frame);
|
||||
// Only the shell bit is ever undone.
|
||||
try testing.expectEqual(true, strip.js);
|
||||
return strip.shell;
|
||||
|
||||
@@ -0,0 +1,934 @@
|
||||
// Copyright (C) 2023-2026 Lightpanda (Selecy SAS)
|
||||
//
|
||||
// Francis Bouvier <francis@lightpanda.io>
|
||||
// Pierre Tachoire <pierre@lightpanda.io>
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as
|
||||
// published by the Free Software Foundation, either version 3 of the
|
||||
// License, or (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
//! Main-content selection, after readability.js's grabArticle: paragraphs
|
||||
//! score their ancestors, the best-scoring ancestor (plus qualifying
|
||||
//! siblings) becomes the render root, and clutter inside it is pruned by
|
||||
//! link density, class names and element mix. Nothing is mutated and the
|
||||
//! render root never moves: the result is the set of nodes to skip, which
|
||||
//! holds everything beside the selection's path from the root as well as
|
||||
//! the clutter pruned inside it. RenderTree.State carries it to the
|
||||
//! renderers.
|
||||
//!
|
||||
//! Where readability returns its longest attempt no matter how poor, this
|
||||
//! returns null and the caller falls back to the shell strip.
|
||||
|
||||
const std = @import("std");
|
||||
const lp = @import("lightpanda");
|
||||
|
||||
const Frame = @import("Frame.zig");
|
||||
const RenderTree = @import("RenderTree.zig");
|
||||
|
||||
const Node = @import("webapi/Node.zig");
|
||||
const Element = @import("webapi/Element.zig");
|
||||
const Slot = @import("webapi/element/html/Slot.zig");
|
||||
|
||||
const log = lp.log;
|
||||
const Allocator = std.mem.Allocator;
|
||||
|
||||
/// Below this many characters of selected text an attempt is rejected.
|
||||
const char_threshold = 500;
|
||||
|
||||
/// readability's retry ladder: each step drops one heuristic.
|
||||
const Flags = struct {
|
||||
unlikelys: bool,
|
||||
weight_classes: bool,
|
||||
clean: bool,
|
||||
};
|
||||
|
||||
const attempts = [_]Flags{
|
||||
.{ .unlikelys = true, .weight_classes = true, .clean = true },
|
||||
.{ .unlikelys = false, .weight_classes = true, .clean = true },
|
||||
.{ .unlikelys = false, .weight_classes = false, .clean = true },
|
||||
.{ .unlikelys = false, .weight_classes = false, .clean = false },
|
||||
};
|
||||
|
||||
/// The nodes under `root` to skip so that only the main content renders,
|
||||
/// or null when no attempt found enough content. The set is allocated in
|
||||
/// `allocator`; the scratch is not.
|
||||
pub fn select(allocator: Allocator, root: *Node, strip_: RenderTree.Strip, frame: *Frame) !?*const RenderTree.PruneSet {
|
||||
// The walk measures what the flags alone render.
|
||||
var strip = strip_;
|
||||
strip.clutter = false;
|
||||
|
||||
var arena_state: std.heap.ArenaAllocator = .init(allocator);
|
||||
defer arena_state.deinit();
|
||||
const arena = arena_state.allocator();
|
||||
|
||||
for (attempts, 0..) |flags, i| {
|
||||
_ = arena_state.reset(.retain_capacity);
|
||||
var pass: Pass = .{
|
||||
.arena = arena,
|
||||
.frame = frame,
|
||||
.root = root,
|
||||
.flags = flags,
|
||||
.tree = .{ .frame = frame, .state = .{ .root = root, .strip = strip } },
|
||||
};
|
||||
const selected = try pass.run() orelse continue;
|
||||
const chars = pass.selectedText(selected);
|
||||
log.debug(.browser, "strip clutter", .{ .attempt = i, .chars = chars, .total = pass.total, .root = describe(selected) });
|
||||
if (chars < char_threshold) {
|
||||
continue;
|
||||
}
|
||||
try pass.pruneBeside(selected);
|
||||
const pruned = try allocator.create(RenderTree.PruneSet);
|
||||
pruned.* = try pass.pruned.clone(allocator);
|
||||
return pruned;
|
||||
}
|
||||
log.info(.browser, "strip clutter fallback", .{});
|
||||
return null;
|
||||
}
|
||||
|
||||
const Rule = enum { unlikely, role, sibling, tag, header, share, weight, words, mix };
|
||||
|
||||
// For the debug log: "tag.class#id", truncated.
|
||||
fn describe(node: *Node) []const u8 {
|
||||
const el = node.is(Element) orelse return @tagName(node._type);
|
||||
const S = struct {
|
||||
threadlocal var buf: [96]u8 = undefined;
|
||||
};
|
||||
return std.fmt.bufPrint(&S.buf, "{s}.{s}#{s}", .{ @tagName(el.getTag()), el.getClassName() orelse "", el.getId() orelse "" }) catch S.buf[0..];
|
||||
}
|
||||
|
||||
const Stats = struct {
|
||||
// Rendered text, whitespace collapsed, and the part of it inside links,
|
||||
// headings and lists.
|
||||
text: usize = 0,
|
||||
link: usize = 0,
|
||||
heading: usize = 0,
|
||||
list: usize = 0,
|
||||
// Text inside readability's "textish" tags (span, li, td, p, div...).
|
||||
textish: usize = 0,
|
||||
commas: usize = 0,
|
||||
|
||||
// Descendant counts.
|
||||
p: u32 = 0,
|
||||
img: u32 = 0,
|
||||
li: u32 = 0,
|
||||
input: u32 = 0,
|
||||
embeds: u32 = 0,
|
||||
rows: u32 = 0,
|
||||
cells: u32 = 0,
|
||||
th: u32 = 0,
|
||||
tables: u32 = 0,
|
||||
table_head: bool = false,
|
||||
has_data_table: bool = false,
|
||||
|
||||
// A direct child that keeps a <div> from reading as a paragraph.
|
||||
block_child: bool = false,
|
||||
data_table: bool = false,
|
||||
|
||||
candidate: bool = false,
|
||||
score: f32 = 0,
|
||||
|
||||
fn add(self: *Stats, other: *const Stats) void {
|
||||
self.text += other.text;
|
||||
self.link += other.link;
|
||||
self.heading += other.heading;
|
||||
self.list += other.list;
|
||||
self.textish += other.textish;
|
||||
self.commas += other.commas;
|
||||
self.p += other.p;
|
||||
self.img += other.img;
|
||||
self.li += other.li;
|
||||
self.input += other.input;
|
||||
self.embeds += other.embeds;
|
||||
self.rows += other.rows;
|
||||
self.cells += other.cells;
|
||||
self.th += other.th;
|
||||
self.tables += other.tables;
|
||||
self.table_head = self.table_head or other.table_head;
|
||||
self.has_data_table = self.has_data_table or other.has_data_table;
|
||||
}
|
||||
|
||||
fn sub(self: *Stats, other: *const Stats) void {
|
||||
self.text -= other.text;
|
||||
self.link -= other.link;
|
||||
self.heading -= other.heading;
|
||||
self.list -= other.list;
|
||||
self.textish -= other.textish;
|
||||
self.commas -= other.commas;
|
||||
self.p -= other.p;
|
||||
self.img -= other.img;
|
||||
self.li -= other.li;
|
||||
self.input -= other.input;
|
||||
self.embeds -= other.embeds;
|
||||
self.rows -= other.rows;
|
||||
self.cells -= other.cells;
|
||||
self.th -= other.th;
|
||||
self.tables -= other.tables;
|
||||
}
|
||||
|
||||
// What the element itself contributes to its ancestors' counts.
|
||||
fn countSelf(self: *Stats, tag: Element.Tag, delta: i2) void {
|
||||
const field = switch (tag) {
|
||||
.p => &self.p,
|
||||
.img => &self.img,
|
||||
.li => &self.li,
|
||||
.input => &self.input,
|
||||
.iframe, .object, .embed, .video, .audio => &self.embeds,
|
||||
.tr => &self.rows,
|
||||
.td => &self.cells,
|
||||
.th => &self.th,
|
||||
.table => &self.tables,
|
||||
else => return,
|
||||
};
|
||||
if (delta > 0) field.* += 1 else field.* -= 1;
|
||||
}
|
||||
|
||||
fn linkDensity(self: *const Stats) f32 {
|
||||
if (self.text == 0) return 0;
|
||||
return @as(f32, @floatFromInt(self.link)) / @as(f32, @floatFromInt(self.text));
|
||||
}
|
||||
|
||||
fn ratio(part: usize, whole: usize) f32 {
|
||||
if (whole == 0) return 0;
|
||||
return @as(f32, @floatFromInt(part)) / @as(f32, @floatFromInt(whole));
|
||||
}
|
||||
};
|
||||
|
||||
// Ancestor state that readability reads through hasAncestorTag.
|
||||
const Ctx = struct {
|
||||
in_link: bool = false,
|
||||
hash_link: bool = false,
|
||||
in_heading: bool = false,
|
||||
in_list: bool = false,
|
||||
in_code: bool = false,
|
||||
in_table: bool = false,
|
||||
in_figure: bool = false,
|
||||
in_data_table: bool = false,
|
||||
};
|
||||
|
||||
const Pass = struct {
|
||||
arena: Allocator,
|
||||
frame: *Frame,
|
||||
root: *Node,
|
||||
flags: Flags,
|
||||
tree: RenderTree,
|
||||
total: usize = 0,
|
||||
stats: std.AutoHashMapUnmanaged(*Element, *Stats) = .{},
|
||||
pruned: RenderTree.PruneSet = .{},
|
||||
candidates: std.ArrayListUnmanaged(*Element) = .empty,
|
||||
|
||||
fn run(self: *Pass) !?*Node {
|
||||
var root_stats: Stats = .{};
|
||||
switch (self.root._type) {
|
||||
.document, .document_fragment => {
|
||||
var it = self.tree.children(self.root, false);
|
||||
while (it.next()) |child| {
|
||||
try self.walkChild(child, .{}, &root_stats);
|
||||
}
|
||||
},
|
||||
else => if (self.tree.classify(self.root, .{})) |child| {
|
||||
try self.walkChild(child, .{}, &root_stats);
|
||||
},
|
||||
}
|
||||
self.total = root_stats.text;
|
||||
|
||||
const top = self.topCandidate() orelse return null;
|
||||
const candidate = self.refine(top);
|
||||
log.debug(.browser, "clutter candidate", .{ .top = describe(top.asNode()), .score = self.score(top), .refined = describe(candidate.asNode()) });
|
||||
const tag = candidate.getTag();
|
||||
if (tag == .body or tag == .html) {
|
||||
// readability wraps the whole body here; that is no selection.
|
||||
return null;
|
||||
}
|
||||
const selected = self.withSiblings(candidate);
|
||||
try self.clean(selected, .{});
|
||||
return selected;
|
||||
}
|
||||
|
||||
// --- stats -------------------------------------------------------------
|
||||
|
||||
fn walkChild(self: *Pass, child: RenderTree.Child, ctx: Ctx, parent: *Stats) !void {
|
||||
switch (child.what) {
|
||||
.text => |text| {
|
||||
const m = measureText(text);
|
||||
parent.text += m.len;
|
||||
parent.commas += m.commas;
|
||||
if (ctx.in_link) {
|
||||
// In-page links count for less, as in readability.
|
||||
parent.link += if (ctx.hash_link) m.len * 3 / 10 else m.len;
|
||||
}
|
||||
if (ctx.in_heading) parent.heading += m.len;
|
||||
if (ctx.in_list) parent.list += m.len;
|
||||
},
|
||||
.element => |d| {
|
||||
const el = child.node.subtype(Element);
|
||||
const tag = el.getTag();
|
||||
if (self.isUnlikely(el, tag, ctx)) |rule| {
|
||||
try self.pruned.put(self.arena, el.asNode(), {});
|
||||
log.debug(.browser, "clutter prune", .{ .rule = rule, .el = describe(el.asNode()) });
|
||||
return;
|
||||
}
|
||||
|
||||
const st = try self.arena.create(Stats);
|
||||
st.* = .{};
|
||||
try self.stats.put(self.arena, el, st);
|
||||
|
||||
var inner = ctx;
|
||||
switch (tag) {
|
||||
.anchor => {
|
||||
inner.in_link = true;
|
||||
const href = el.getAttributeSafe(comptime .wrap("href")) orelse "";
|
||||
inner.hash_link = href.len > 0 and href[0] == '#';
|
||||
},
|
||||
.h1, .h2, .h3, .h4, .h5, .h6 => inner.in_heading = true,
|
||||
.ul, .ol => inner.in_list = true,
|
||||
.code, .pre => inner.in_code = true,
|
||||
.table => inner.in_table = true,
|
||||
.figure => inner.in_figure = true,
|
||||
else => {},
|
||||
}
|
||||
|
||||
if (el.is(Slot)) |slot| {
|
||||
var it = self.tree.slotted(slot);
|
||||
while (it.next()) |c| {
|
||||
try self.walkChild(c, inner, st);
|
||||
}
|
||||
} else {
|
||||
const boxed = d == .flex or d == .grid;
|
||||
var it = self.tree.content(el, boxed);
|
||||
while (it.next()) |c| {
|
||||
try self.walkChild(c, inner, st);
|
||||
}
|
||||
}
|
||||
|
||||
if (tag == .table) {
|
||||
st.data_table = isDataTable(el, st);
|
||||
st.has_data_table = st.has_data_table or st.data_table;
|
||||
}
|
||||
if (isParagraphLike(tag, st)) {
|
||||
try self.scoreParagraph(el, st);
|
||||
}
|
||||
|
||||
parent.add(st);
|
||||
parent.countSelf(countedAs(tag, st), 1);
|
||||
parent.textish += if (isTextish(tag)) st.text else st.textish;
|
||||
if (blocksParagraph(tag)) {
|
||||
parent.block_child = true;
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
fn isUnlikely(self: *const Pass, el: *Element, tag: Element.Tag, ctx: Ctx) ?Rule {
|
||||
if (self.flags.unlikelys == false) return null;
|
||||
if (tag != .body and tag != .anchor and !ctx.in_table and !ctx.in_code) {
|
||||
const names = self.classAndId(el);
|
||||
if (containsAny(names, &unlikely) and !containsAny(names, &maybe_candidate)) {
|
||||
return .unlikely;
|
||||
}
|
||||
}
|
||||
if (hasRole(el, &.{ "menu", "menubar", "complementary", "navigation", "alert", "alertdialog", "dialog" })) {
|
||||
return .role;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
fn scoreParagraph(self: *Pass, el: *Element, st: *Stats) !void {
|
||||
if (st.text < 25) return;
|
||||
const points: f32 = 1 + @as(f32, @floatFromInt(st.commas)) + @min(@as(f32, @floatFromInt(st.text / 100)), 3);
|
||||
|
||||
var level: usize = 0;
|
||||
var ancestor = self.parentElement(el);
|
||||
while (ancestor) |a| : (ancestor = self.parentElement(a)) {
|
||||
if (level == 5) break;
|
||||
const ast = self.stats.get(a) orelse break;
|
||||
if (ast.candidate == false) {
|
||||
ast.candidate = true;
|
||||
ast.score = tagScore(a.getTag()) + self.classWeight(a);
|
||||
try self.candidates.append(self.arena, a);
|
||||
}
|
||||
const divider: f32 = switch (level) {
|
||||
0 => 1,
|
||||
1 => 2,
|
||||
else => @floatFromInt(level * 3),
|
||||
};
|
||||
ast.score += points / divider;
|
||||
level += 1;
|
||||
}
|
||||
}
|
||||
|
||||
// --- candidate ---------------------------------------------------------
|
||||
|
||||
fn topCandidate(self: *Pass) ?*Element {
|
||||
var top: ?*Element = null;
|
||||
var top_score: f32 = 0;
|
||||
for (self.candidates.items) |el| {
|
||||
const st = self.stats.get(el).?;
|
||||
st.score *= 1 - st.linkDensity();
|
||||
if (top == null or st.score > top_score) {
|
||||
top = el;
|
||||
top_score = st.score;
|
||||
}
|
||||
}
|
||||
if (log.enabled(.browser, .debug)) {
|
||||
for (self.candidates.items) |el| {
|
||||
const st = self.stats.get(el).?;
|
||||
if (st.score >= top_score * 0.2) {
|
||||
log.debug(.browser, "clutter score", .{ .el = describe(el.asNode()), .score = st.score });
|
||||
}
|
||||
}
|
||||
}
|
||||
return top;
|
||||
}
|
||||
|
||||
fn score(self: *const Pass, el: *Element) f32 {
|
||||
const st = self.stats.get(el) orelse return 0;
|
||||
return if (st.candidate) st.score else 0;
|
||||
}
|
||||
|
||||
/// readability's walk up from the top candidate: an ancestor shared by
|
||||
/// three strong alternatives, then better-scoring parents, then
|
||||
/// single-child wrappers.
|
||||
fn refine(self: *const Pass, top: *Element) *Element {
|
||||
var candidate = top;
|
||||
const top_score = self.score(top);
|
||||
|
||||
var parent = self.parentElement(candidate);
|
||||
while (parent) |p| : (parent = self.parentElement(p)) {
|
||||
if (p.getTag() == .body) break;
|
||||
var shared: usize = 0;
|
||||
for (self.candidates.items) |alt| {
|
||||
if (alt == top or self.score(alt) < top_score * 0.75) continue;
|
||||
if (self.isAncestor(p, alt)) shared += 1;
|
||||
}
|
||||
if (shared >= 3) {
|
||||
candidate = p;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
var last = self.score(candidate);
|
||||
const threshold = last / 3;
|
||||
parent = self.parentElement(candidate);
|
||||
while (parent) |p| : (parent = self.parentElement(p)) {
|
||||
if (p.getTag() == .body) break;
|
||||
const ps = self.score(p);
|
||||
if (ps == 0) continue;
|
||||
if (ps < threshold) break;
|
||||
if (ps > last) {
|
||||
candidate = p;
|
||||
break;
|
||||
}
|
||||
last = ps;
|
||||
}
|
||||
|
||||
parent = self.parentElement(candidate);
|
||||
while (parent) |p| : (parent = self.parentElement(p)) {
|
||||
if (p.getTag() == .body) break;
|
||||
if (self.renderedElementCount(p) != 1) break;
|
||||
candidate = p;
|
||||
}
|
||||
return candidate;
|
||||
}
|
||||
|
||||
/// The candidate's parent when siblings qualify (the rest pruned), else
|
||||
/// the candidate itself.
|
||||
fn withSiblings(self: *Pass, candidate: *Element) *Node {
|
||||
const parent = self.parentElement(candidate) orelse return candidate.asNode();
|
||||
const top_score = self.score(candidate);
|
||||
const threshold = @max(10, top_score * 0.2);
|
||||
const class = candidate.getClassName() orelse "";
|
||||
|
||||
var kept: usize = 0;
|
||||
var it = self.tree.children(parent.asNode(), false);
|
||||
while (it.next()) |child| {
|
||||
if (child.what != .element) {
|
||||
// Loose text between siblings is not part of any of them.
|
||||
self.pruned.put(self.arena, child.node, {}) catch {};
|
||||
continue;
|
||||
}
|
||||
const sibling = child.node.subtype(Element);
|
||||
if (sibling == candidate) continue;
|
||||
const st = self.stats.get(sibling) orelse continue;
|
||||
|
||||
var keep = false;
|
||||
var bonus: f32 = 0;
|
||||
if (class.len > 0 and std.mem.eql(u8, class, sibling.getClassName() orelse "")) {
|
||||
bonus = top_score * 0.2;
|
||||
}
|
||||
if (self.score(sibling) + bonus >= threshold) {
|
||||
keep = true;
|
||||
} else if (countedAs(sibling.getTag(), st) == .p) {
|
||||
const density = st.linkDensity();
|
||||
if (st.text > 80 and density < 0.25) {
|
||||
keep = true;
|
||||
} else if (st.text > 0 and st.text < 80 and density == 0 and self.endsSentence(sibling)) {
|
||||
keep = true;
|
||||
}
|
||||
}
|
||||
if (keep) {
|
||||
kept += 1;
|
||||
} else {
|
||||
self.pruned.put(self.arena, sibling.asNode(), {}) catch {};
|
||||
log.debug(.browser, "clutter prune", .{ .rule = Rule.sibling, .el = describe(sibling.asNode()) });
|
||||
}
|
||||
}
|
||||
return if (kept > 0) parent.asNode() else candidate.asNode();
|
||||
}
|
||||
|
||||
/// Everything beside the path from the dump root down to `selected`:
|
||||
/// each ancestor keeps only the child on the path. Shadow trees render
|
||||
/// in place of their host, so a shadow root's parent is its host.
|
||||
fn pruneBeside(self: *Pass, selected: *Node) !void {
|
||||
var current = selected;
|
||||
while (current != self.root) {
|
||||
const parent = current.parentNode() orelse blk: {
|
||||
const shadow = current.is(Node.ShadowRoot) orelse return;
|
||||
break :blk shadow.getHost().asNode();
|
||||
};
|
||||
var it = self.tree.children(parent, false);
|
||||
while (it.next()) |child| {
|
||||
if (child.node == current) continue;
|
||||
try self.pruned.put(self.arena, child.node, {});
|
||||
}
|
||||
current = parent;
|
||||
}
|
||||
}
|
||||
|
||||
// --- cleaning ----------------------------------------------------------
|
||||
|
||||
/// Post-order so a pruned inner block no longer counts against its
|
||||
/// container, as readability's reverse-order removal achieves.
|
||||
fn clean(self: *Pass, node: *Node, ctx: Ctx) !void {
|
||||
var it = self.tree.children(node, false);
|
||||
while (it.next()) |child| {
|
||||
if (child.what != .element) continue;
|
||||
const el = child.node.subtype(Element);
|
||||
if (self.pruned.contains(child.node)) continue;
|
||||
const st = self.stats.get(el) orelse continue;
|
||||
const tag = el.getTag();
|
||||
|
||||
var inner = ctx;
|
||||
switch (tag) {
|
||||
.code, .pre => inner.in_code = true,
|
||||
.figure => inner.in_figure = true,
|
||||
.table => inner.in_data_table = ctx.in_data_table or st.data_table,
|
||||
else => {},
|
||||
}
|
||||
const content = if (el.hostedShadowRoot(self.frame)) |shadow| shadow.asNode() else el.asNode();
|
||||
try self.clean(content, inner);
|
||||
|
||||
if (self.shouldPrune(el, tag, st, inner)) |rule| {
|
||||
try self.prune(el, tag, st);
|
||||
log.debug(.browser, "clutter prune", .{ .rule = rule, .el = describe(el.asNode()) });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn prune(self: *Pass, el: *Element, tag: Element.Tag, st: *const Stats) !void {
|
||||
try self.pruned.put(self.arena, el.asNode(), {});
|
||||
const textish: usize = if (isTextish(tag)) st.text else st.textish;
|
||||
var ancestor = self.parentElement(el);
|
||||
while (ancestor) |a| : (ancestor = self.parentElement(a)) {
|
||||
const ast = self.stats.get(a) orelse break;
|
||||
ast.sub(st);
|
||||
ast.countSelf(countedAs(tag, st), -1);
|
||||
ast.textish -= textish;
|
||||
}
|
||||
}
|
||||
|
||||
fn shouldPrune(self: *const Pass, el: *Element, tag: Element.Tag, st: *const Stats, ctx: Ctx) ?Rule {
|
||||
switch (tag) {
|
||||
.aside, .footer, .object, .embed, .iframe, .input, .textarea, .select, .button => return .tag,
|
||||
.h1, .h2 => return if (self.classWeight(el) < 0) .header else null,
|
||||
else => {},
|
||||
}
|
||||
|
||||
if (st.text < 500 and isShareElement(self.classAndId(el))) {
|
||||
return .share;
|
||||
}
|
||||
|
||||
if (self.flags.clean == false) return null;
|
||||
switch (tag) {
|
||||
.form, .fieldset, .table, .ul, .ol => {},
|
||||
// A <div> without block children is a paragraph to readability.
|
||||
.div => if (st.block_child == false) return null,
|
||||
else => return null,
|
||||
}
|
||||
if (ctx.in_data_table or ctx.in_code or st.has_data_table) return null;
|
||||
|
||||
const weight = self.classWeight(el);
|
||||
if (weight < 0) {
|
||||
return .weight;
|
||||
}
|
||||
if (st.commas >= 10) return null;
|
||||
if (st.text <= 24 and self.isAdOrLoading(el)) {
|
||||
return .words;
|
||||
}
|
||||
|
||||
const is_list = tag == .ul or tag == .ol or Stats.ratio(st.list, st.text) > 0.9;
|
||||
const density = st.linkDensity();
|
||||
const heading_density = Stats.ratio(st.heading, st.text);
|
||||
const p: f32 = @floatFromInt(st.p);
|
||||
const img: f32 = @floatFromInt(st.img);
|
||||
|
||||
const remove =
|
||||
(!ctx.in_figure and st.img > 1 and p / img < 0.5) or
|
||||
// readability subtracts 100 from the li count first.
|
||||
(!is_list and st.li > st.p + 100) or
|
||||
(st.input > st.p / 3) or
|
||||
(!is_list and !ctx.in_figure and heading_density < 0.9 and st.text < 25 and (st.img == 0 or st.img > 2) and density > 0) or
|
||||
(!is_list and weight < 25 and density > 0.2) or
|
||||
(weight >= 25 and density > 0.5) or
|
||||
(st.embeds == 1 and st.text < 75) or st.embeds > 1 or
|
||||
(st.img == 0 and st.textish == 0);
|
||||
|
||||
if (remove and is_list and self.isImageList(el, st)) {
|
||||
return null;
|
||||
}
|
||||
return if (remove) .mix else null;
|
||||
}
|
||||
|
||||
// A list whose every item is one image stays.
|
||||
fn isImageList(self: *const Pass, el: *Element, st: *const Stats) bool {
|
||||
var it = self.tree.content(el, false);
|
||||
while (it.next()) |child| {
|
||||
if (child.what != .element) continue;
|
||||
if (self.renderedElementCount(child.node.subtype(Element)) > 1) return false;
|
||||
}
|
||||
return st.img > 0 and st.img == st.li;
|
||||
}
|
||||
|
||||
fn isAdOrLoading(self: *const Pass, el: *Element) bool {
|
||||
var buf: [32]u8 = undefined;
|
||||
var text = std.mem.trim(u8, self.gatherText(el, &buf), &std.ascii.whitespace);
|
||||
for ([_][]const u8{ "...", "\xE2\x80\xA6" }) |ellipsis| {
|
||||
if (std.mem.endsWith(u8, text, ellipsis)) text = text[0 .. text.len - ellipsis.len];
|
||||
}
|
||||
for (ad_words) |w| {
|
||||
if (std.ascii.eqlIgnoreCase(text, w)) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
fn gatherText(self: *const Pass, el: *Element, buf: []u8) []const u8 {
|
||||
var n: usize = 0;
|
||||
var it = self.tree.content(el, false);
|
||||
while (it.next()) |child| {
|
||||
switch (child.what) {
|
||||
.text => |text| {
|
||||
for (text) |c| {
|
||||
if (n == buf.len) return buf;
|
||||
buf[n] = c;
|
||||
n += 1;
|
||||
}
|
||||
},
|
||||
.element => {
|
||||
const inner = self.gatherText(child.node.subtype(Element), buf[n..]);
|
||||
n += inner.len;
|
||||
if (n == buf.len) return buf;
|
||||
},
|
||||
}
|
||||
}
|
||||
return buf[0..n];
|
||||
}
|
||||
|
||||
// --- helpers -----------------------------------------------------------
|
||||
|
||||
fn parentElement(self: *const Pass, el: *Element) ?*Element {
|
||||
const node = el.asNode();
|
||||
if (node == self.root) return null;
|
||||
const parent = node.parentNode() orelse return null;
|
||||
return parent.is(Element);
|
||||
}
|
||||
|
||||
fn isAncestor(self: *const Pass, ancestor: *Element, el: *Element) bool {
|
||||
var parent = self.parentElement(el);
|
||||
while (parent) |p| : (parent = self.parentElement(p)) {
|
||||
if (p == ancestor) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
fn renderedElementCount(self: *const Pass, el: *Element) usize {
|
||||
var n: usize = 0;
|
||||
var it = self.tree.content(el, false);
|
||||
while (it.next()) |child| {
|
||||
if (child.what != .element) continue;
|
||||
if (self.pruned.contains(child.node)) continue;
|
||||
n += 1;
|
||||
}
|
||||
return n;
|
||||
}
|
||||
|
||||
fn endsSentence(self: *const Pass, el: *Element) bool {
|
||||
var last: u8 = 0;
|
||||
var it = self.tree.content(el, false);
|
||||
while (it.next()) |child| {
|
||||
switch (child.what) {
|
||||
.text => |text| {
|
||||
const trimmed = std.mem.trimEnd(u8, text, &std.ascii.whitespace);
|
||||
if (trimmed.len > 0) last = trimmed[trimmed.len - 1];
|
||||
},
|
||||
.element => last = 0,
|
||||
}
|
||||
}
|
||||
return last == '.';
|
||||
}
|
||||
|
||||
fn classWeight(self: *const Pass, el: *Element) f32 {
|
||||
if (self.flags.weight_classes == false) return 0;
|
||||
var weight: f32 = 0;
|
||||
inline for (.{ el.getClassName(), el.getId() }) |attr| {
|
||||
if (attr) |raw| {
|
||||
const name = std.ascii.allocLowerString(self.arena, raw) catch return weight;
|
||||
if (isNegative(name)) weight -= 25;
|
||||
if (containsAny(name, &positive)) weight += 25;
|
||||
}
|
||||
}
|
||||
return weight;
|
||||
}
|
||||
|
||||
// Lowercased "class id", readability's matchString. Never truncated: a
|
||||
// framework's <html> class list runs to kilobytes and the escape word
|
||||
// can sit anywhere in it.
|
||||
fn classAndId(self: *const Pass, el: *Element) []const u8 {
|
||||
const class = el.getClassName() orelse "";
|
||||
const id = el.getId() orelse "";
|
||||
const buf = self.arena.alloc(u8, class.len + 1 + id.len) catch return "";
|
||||
_ = std.ascii.lowerString(buf[0..class.len], class);
|
||||
buf[class.len] = ' ';
|
||||
_ = std.ascii.lowerString(buf[class.len + 1 ..], id);
|
||||
return buf;
|
||||
}
|
||||
|
||||
/// Text under `node` once the prune set is applied.
|
||||
fn selectedText(self: *const Pass, node: *Node) usize {
|
||||
var n: usize = 0;
|
||||
var it = self.tree.children(node, false);
|
||||
while (it.next()) |child| {
|
||||
switch (child.what) {
|
||||
.text => |text| n += measureText(text).len,
|
||||
.element => {
|
||||
const el = child.node.subtype(Element);
|
||||
if (self.pruned.contains(child.node)) continue;
|
||||
const content = if (el.hostedShadowRoot(self.frame)) |shadow| shadow.asNode() else el.asNode();
|
||||
n += self.selectedText(content);
|
||||
},
|
||||
}
|
||||
}
|
||||
return n;
|
||||
}
|
||||
};
|
||||
|
||||
// readability's DIV_TO_P_ELEMS: a <div> with none of these children is a
|
||||
// paragraph for scoring.
|
||||
fn blocksParagraph(tag: Element.Tag) bool {
|
||||
return switch (tag) {
|
||||
.blockquote, .dl, .div, .img, .ol, .p, .pre, .table, .ul => true,
|
||||
else => false,
|
||||
};
|
||||
}
|
||||
|
||||
// readability's DEFAULT_TAGS_TO_SCORE, plus the <div>s it turns into <p>.
|
||||
fn isParagraphLike(tag: Element.Tag, st: *const Stats) bool {
|
||||
return switch (tag) {
|
||||
.section, .h2, .h3, .h4, .h5, .h6, .p, .td, .pre => true,
|
||||
.div => st.block_child == false,
|
||||
else => false,
|
||||
};
|
||||
}
|
||||
|
||||
fn countedAs(tag: Element.Tag, st: *const Stats) Element.Tag {
|
||||
return if (tag == .div and st.block_child == false) .p else tag;
|
||||
}
|
||||
|
||||
// readability's textish tags: SPAN, LI, TD and DIV_TO_P_ELEMS.
|
||||
fn isTextish(tag: Element.Tag) bool {
|
||||
return switch (tag) {
|
||||
.span, .li, .td, .blockquote, .dl, .div, .img, .ol, .p, .pre, .table, .ul => true,
|
||||
else => false,
|
||||
};
|
||||
}
|
||||
|
||||
// readability's adWords and loadingWords, matched whole.
|
||||
const ad_words = [_][]const u8{ "ad", "advertising", "advertisement", "pub", "publicit\xC3\xA9", "werb", "werbung", "\xE5\xB9\xBF\xE5\x91\x8A", "\xD0\xA0\xD0\xB5\xD0\xBA\xD0\xBB\xD0\xB0\xD0\xBC\xD0\xB0", "anuncio", "loading", "\xE6\xAD\xA3\xE5\x9C\xA8\xE5\x8A\xA0\xE8\xBD\xBD", "\xD0\x97\xD0\xB0\xD0\xB3\xD1\x80\xD1\x83\xD0\xB7\xD0\xBA\xD0\xB0", "chargement", "cargando" };
|
||||
|
||||
fn tagScore(tag: Element.Tag) f32 {
|
||||
return switch (tag) {
|
||||
.div => 5,
|
||||
.pre, .td, .blockquote => 3,
|
||||
.address, .ol, .ul, .dl, .dd, .dt, .li, .form => -3,
|
||||
.h1, .h2, .h3, .h4, .h5, .h6, .th => -5,
|
||||
else => 0,
|
||||
};
|
||||
}
|
||||
|
||||
fn isDataTable(el: *Element, st: *const Stats) bool {
|
||||
if (hasRole(el, &.{"presentation"})) return false;
|
||||
if (el.getAttributeSafe(comptime .wrap("datatable"))) |v| {
|
||||
if (std.mem.eql(u8, v, "0")) return false;
|
||||
}
|
||||
if (el.getAttributeSafe(comptime .wrap("summary")) != null) return true;
|
||||
if (st.th > 0 or st.table_head) return true;
|
||||
if (st.tables > 0) return false;
|
||||
const cols = if (st.rows == 0) st.cells else st.cells / st.rows;
|
||||
if (st.rows >= 10 or cols > 4) return true;
|
||||
return st.rows * cols > 10;
|
||||
}
|
||||
|
||||
const TextMeasure = struct { len: usize, commas: usize };
|
||||
|
||||
/// Length as innerText would report it: whitespace runs collapse to one,
|
||||
/// leading and trailing runs vanish.
|
||||
fn measureText(text: []const u8) TextMeasure {
|
||||
var len: usize = 0;
|
||||
var commas: usize = 0;
|
||||
var pending_space = false;
|
||||
for (text, 0..) |c, i| {
|
||||
if (std.ascii.isWhitespace(c)) {
|
||||
pending_space = len > 0;
|
||||
continue;
|
||||
}
|
||||
if (pending_space) {
|
||||
len += 1;
|
||||
pending_space = false;
|
||||
}
|
||||
len += 1;
|
||||
if (c == ',') {
|
||||
commas += 1;
|
||||
} else if (c == 0xEF and i + 2 < text.len and text[i + 1] == 0xBC and text[i + 2] == 0x8C) {
|
||||
commas += 1; // U+FF0C fullwidth comma
|
||||
} else if (c == 0xE3 and i + 2 < text.len and text[i + 1] == 0x80 and text[i + 2] == 0x81) {
|
||||
commas += 1; // U+3001 ideographic comma
|
||||
}
|
||||
}
|
||||
return .{ .len = len, .commas = commas };
|
||||
}
|
||||
|
||||
// readability's REGEXPS as substring lists, matched on the lowercased
|
||||
// class and id.
|
||||
const unlikely = [_][]const u8{ "-ad-", "ai2html", "banner", "breadcrumbs", "combx", "comment", "community", "cover-wrap", "disqus", "extra", "footer", "gdpr", "header", "legends", "menu", "related", "remark", "replies", "rss", "shoutbox", "sidebar", "skyscraper", "social", "sponsor", "supplemental", "ad-break", "agegate", "pagination", "pager", "popup", "yom-remote" };
|
||||
const maybe_candidate = [_][]const u8{ "and", "article", "body", "column", "content", "main", "shadow" };
|
||||
const positive = [_][]const u8{ "article", "body", "content", "entry", "hentry", "h-entry", "main", "page", "pagination", "post", "text", "blog", "story" };
|
||||
const negative = [_][]const u8{ "-ad-", "hidden", "banner", "combx", "comment", "com-", "contact", "footer", "gdpr", "masthead", "media", "meta", "outbrain", "promo", "related", "scroll", "share", "shoutbox", "sidebar", "skyscraper", "sponsor", "shopping", "tags", "widget" };
|
||||
|
||||
fn containsAny(haystack: []const u8, needles: []const []const u8) bool {
|
||||
for (needles) |needle| {
|
||||
if (std.mem.indexOf(u8, haystack, needle) != null) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
fn isNegative(name: []const u8) bool {
|
||||
return containsAny(name, &negative) or hasWord(name, "hid");
|
||||
}
|
||||
|
||||
// readability: /(\b|_)(share|sharedaddy)(\b|_)/
|
||||
fn isShareElement(names: []const u8) bool {
|
||||
return hasWord(names, "share") or hasWord(names, "sharedaddy");
|
||||
}
|
||||
|
||||
fn hasWord(haystack: []const u8, word: []const u8) bool {
|
||||
var start: usize = 0;
|
||||
while (std.mem.indexOfPos(u8, haystack, start, word)) |i| : (start = i + 1) {
|
||||
const before = i == 0 or !std.ascii.isAlphanumeric(haystack[i - 1]);
|
||||
const end = i + word.len;
|
||||
const after = end == haystack.len or !std.ascii.isAlphanumeric(haystack[end]);
|
||||
if (before and after) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
// ARIA `role` is a space-separated fallback list; the first token wins.
|
||||
fn hasRole(el: *Element, roles: []const []const u8) bool {
|
||||
const attr = el.getAttributeSafe(comptime .wrap("role")) orelse return false;
|
||||
var it = std.mem.tokenizeAny(u8, attr, " \t\n\r");
|
||||
const role = it.next() orelse return false;
|
||||
for (roles) |candidate| {
|
||||
if (std.ascii.eqlIgnoreCase(role, candidate)) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
const testing = @import("../testing.zig");
|
||||
const markdown = @import("markdown.zig");
|
||||
|
||||
const prose = "Sourdough is a bread made by the fermentation of dough using wild lactobacillaceae and yeast, which give it a mildly sour taste. The lactic acid produced by the bacteria gives it a longer shelf life than breads made with baker's yeast. ";
|
||||
|
||||
test "clutter: keeps the article, drops the teaser list and share bar" {
|
||||
const html =
|
||||
"<div class=\"teasers\"><div><a href=\"/1\">Ten things to know</a></div><div><a href=\"/2\">What happened next</a></div><div><a href=\"/3\">More for you</a></div><div><a href=\"/4\">Trending now</a></div></div>" ++
|
||||
"<div class=\"article\"><h1>Sourdough</h1><div class=\"share\"><a href=\"/s\">Share on X</a> <a href=\"/f\">Share on Facebook</a></div>" ++
|
||||
"<p>" ++ prose ++ "</p><p>" ++ prose ++ "</p><p>" ++ prose ++ "</p></div>";
|
||||
const out = try extract(html);
|
||||
try testing.expectEqual(true, std.mem.indexOf(u8, out, "wild lactobacillaceae") != null);
|
||||
try testing.expectEqual(true, std.mem.indexOf(u8, out, "# Sourdough") != null);
|
||||
try testing.expectEqual(null, std.mem.indexOf(u8, out, "Trending now"));
|
||||
try testing.expectEqual(null, std.mem.indexOf(u8, out, "Share on"));
|
||||
}
|
||||
|
||||
test "clutter: too little text falls back" {
|
||||
const out = try extract("<div><p>A short note.</p></div><div><a href=\"/1\">one</a> <a href=\"/2\">two</a></div>");
|
||||
try testing.expectEqual(true, std.mem.indexOf(u8, out, "A short note.") != null);
|
||||
try testing.expectEqual(true, std.mem.indexOf(u8, out, "two") != null);
|
||||
}
|
||||
|
||||
test "clutter: retry ladder rescues content in an unlikely class" {
|
||||
const out = try extract("<div class=\"sidebar\"><p>" ++ prose ++ "</p><p>" ++ prose ++ "</p><p>" ++ prose ++ "</p></div><div><a href=\"/x\">elsewhere</a></div>");
|
||||
try testing.expectEqual(true, std.mem.indexOf(u8, out, "wild lactobacillaceae") != null);
|
||||
try testing.expectEqual(null, std.mem.indexOf(u8, out, "elsewhere"));
|
||||
}
|
||||
|
||||
test "clutter: qualifying sibling paragraphs come along" {
|
||||
const out = try extract("<div id=\"wrap\"><div class=\"body\"><p>" ++ prose ++ "</p><p>" ++ prose ++ "</p></div><p>" ++ prose ++ "</p><div class=\"nav\"><a href=\"/a\">a</a> <a href=\"/b\">b</a> <a href=\"/c\">c</a></div></div>");
|
||||
try testing.expectEqual(3, std.mem.count(u8, out, "wild lactobacillaceae"));
|
||||
try testing.expectEqual(null, std.mem.indexOf(u8, out, "[a]"));
|
||||
}
|
||||
|
||||
test "clutter: the prune set lives in the caller's allocator" {
|
||||
const frame = try testing.createFrame();
|
||||
defer testing.test_session.closeAllPages();
|
||||
const doc = frame.window._document;
|
||||
const div = try doc.createElement("div", null, frame);
|
||||
try Frame.parse.htmlAsChildren(frame, div.asNode(), "<div class=\"article\"><p>" ++ prose ++ "</p><p>" ++ prose ++ "</p><p>" ++ prose ++ "</p></div><div class=\"share\"><a href=\"/s\">Share</a></div>");
|
||||
|
||||
var arena: std.heap.ArenaAllocator = .init(testing.allocator);
|
||||
defer arena.deinit();
|
||||
const state = try RenderTree.resolve(arena.allocator(), div.asNode(), .{ .clutter = true }, frame);
|
||||
try testing.expectEqual(true, state.strip.clutter);
|
||||
try testing.expectEqual(true, state.pruned.?.contains(div.lastElementChild().?.asNode()));
|
||||
|
||||
// A plain dump of the same tree is unaffected.
|
||||
var aw: std.Io.Writer.Allocating = .init(testing.allocator);
|
||||
defer aw.deinit();
|
||||
try markdown.dump(.{ .root = div.asNode() }, .{}, &aw.writer, frame);
|
||||
try testing.expectEqual(true, std.mem.indexOf(u8, aw.written(), "Share") != null);
|
||||
|
||||
// Too little text: no set, no clutter.
|
||||
const second = try RenderTree.resolve(arena.allocator(), div.lastElementChild().?.asNode(), .{ .clutter = true }, frame);
|
||||
try testing.expectEqual(false, second.strip.clutter);
|
||||
try testing.expectEqual(null, second.pruned);
|
||||
}
|
||||
|
||||
fn extract(html: []const u8) ![]const u8 {
|
||||
const frame = try testing.createFrame();
|
||||
defer testing.test_session.closeAllPages();
|
||||
|
||||
const doc = frame.window._document;
|
||||
const div = try doc.createElement("div", null, frame);
|
||||
try Frame.parse.htmlAsChildren(frame, div.asNode(), html);
|
||||
|
||||
const state = try RenderTree.resolve(testing.arena_allocator, div.asNode(), .{ .clutter = true }, frame);
|
||||
var aw: std.Io.Writer.Allocating = .init(testing.arena_allocator);
|
||||
try markdown.dump(state, .{}, &aw.writer, frame);
|
||||
return aw.written();
|
||||
}
|
||||
+48
-6
@@ -19,6 +19,7 @@
|
||||
const std = @import("std");
|
||||
const lp = @import("lightpanda");
|
||||
const Frame = @import("Frame.zig");
|
||||
const RenderTree = @import("RenderTree.zig");
|
||||
const LimitedWriter = @import("../LimitedWriter.zig");
|
||||
const Node = @import("webapi/Node.zig");
|
||||
const Slot = @import("webapi/element/html/Slot.zig");
|
||||
@@ -29,16 +30,21 @@ pub const Opts = struct {
|
||||
with_frames: bool = false,
|
||||
strip: Opts.Strip = .{},
|
||||
shadow: Opts.Shadow = .rendered,
|
||||
|
||||
/// Soft cap: output is cut at a UTF-8 boundary and a truncation marker
|
||||
/// appended.
|
||||
max_bytes: ?u32 = null,
|
||||
|
||||
pub const Strip = packed struct(u5) {
|
||||
// Nodes to remove (clutter remove, from RenderTree.resolve)
|
||||
pruned: ?*const RenderTree.PruneSet = null,
|
||||
|
||||
pub const Strip = packed struct(u6) {
|
||||
js: bool = false,
|
||||
ui: bool = false,
|
||||
css: bool = false,
|
||||
invisible: bool = false,
|
||||
shell: bool = false,
|
||||
clutter: bool = false,
|
||||
};
|
||||
|
||||
pub const Shadow = union(enum) {
|
||||
@@ -83,7 +89,7 @@ fn rootUncapped(doc: *Node.Document, opts: Opts, writer: *std.Io.Writer, frame:
|
||||
}
|
||||
}
|
||||
// But if the doc has no child, or the first child isn't a doctype
|
||||
// well force it.
|
||||
// we'll force it.
|
||||
try writer.writeAll("<!DOCTYPE html>");
|
||||
}
|
||||
|
||||
@@ -111,6 +117,9 @@ pub fn deep(node: *Node, opts: Opts, writer: *std.Io.Writer, frame: *Frame) erro
|
||||
fn _deep(node: *Node, opts: Opts, comptime force_slot: bool, writer: *std.Io.Writer, frame: *Frame) error{WriteFailed}!void {
|
||||
switch (node._type) {
|
||||
.cdata => {
|
||||
if (opts.pruned) |set| {
|
||||
if (set.contains(node)) return;
|
||||
}
|
||||
const cd = node.subtype(Node.CData);
|
||||
if (node.is(Node.CData.Comment)) |_| {
|
||||
try writer.writeAll("<!--");
|
||||
@@ -132,7 +141,7 @@ fn _deep(node: *Node, opts: Opts, comptime force_slot: bool, writer: *std.Io.Wri
|
||||
},
|
||||
.element => {
|
||||
const el = node.subtype(Node.Element);
|
||||
if (shouldStripElement(el, opts.strip, frame)) {
|
||||
if (shouldStripElement(el, opts.strip, opts.pruned, frame)) {
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -363,9 +372,9 @@ fn isVoidElement(el: *Node.Element) bool {
|
||||
};
|
||||
}
|
||||
|
||||
pub fn shouldStripElement(el: *Node.Element, strip: Opts.Strip, frame: *Frame) bool {
|
||||
pub fn shouldStripElement(el: *Node.Element, strip: Opts.Strip, pruned: ?*const RenderTree.PruneSet, frame: *Frame) bool {
|
||||
// Fast path: with no strip flags set (every innerHTML/outerHTML call)
|
||||
if (@as(u5, @bitCast(strip)) == 0) {
|
||||
if (@as(u6, @bitCast(strip)) == 0) {
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -417,6 +426,12 @@ pub fn shouldStripElement(el: *Node.Element, strip: Opts.Strip, frame: *Frame) b
|
||||
return true;
|
||||
}
|
||||
|
||||
if (pruned) |set| {
|
||||
if (set.contains(el.asNode())) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -429,9 +444,28 @@ pub fn isShellElement(el: *Node.Element) bool {
|
||||
.header, .footer => return !hasSectioningAncestor(el),
|
||||
else => {},
|
||||
}
|
||||
return hasRole(el, &.{ "banner", "complementary", "contentinfo", "navigation", "search", "dialog", "alertdialog", "menu", "menubar" });
|
||||
if (hasRole(el, &.{ "banner", "complementary", "contentinfo", "navigation", "search", "dialog", "alertdialog", "menu", "menubar" })) {
|
||||
return true;
|
||||
}
|
||||
if (hasShellToken(el.getClassName()) or hasShellToken(el.getId())) {
|
||||
return !hasSectioningAncestor(el);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
// Words that name page chrome and nothing else. "menu" is left out: it also
|
||||
// names content.
|
||||
const shell_tokens = [_][]const u8{ "header", "footer", "nav", "navbar", "navigation", "sidebar", "masthead" };
|
||||
|
||||
fn hasShellToken(value: ?[]const u8) bool {
|
||||
var it = std.mem.tokenizeAny(u8, value orelse return false, " \t\n\r");
|
||||
while (it.next()) |token| {
|
||||
for (shell_tokens) |shell_token| {
|
||||
if (std.ascii.eqlIgnoreCase(token, shell_token)) return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
fn hasSectioningAncestor(el: *Node.Element) bool {
|
||||
var node = el.asNode().parentNode();
|
||||
while (node) |n| : (node = n.parentNode()) {
|
||||
@@ -639,6 +673,14 @@ test "dump: strip.shell removes page chrome but keeps sectioned header/footer" {
|
||||
);
|
||||
}
|
||||
|
||||
test "dump: strip.shell honours chrome class and id tokens" {
|
||||
try expectShellDump(
|
||||
\\<div class="header">H</div><div id="Footer">F</div><div class="site navbar">N</div><div class="subheader">S</div><div class="post-footer">P</div><main><div class="header">MH</div><p>x</p></main><div class="menu">M</div>
|
||||
,
|
||||
\\<div><div class="subheader">S</div><div class="post-footer">P</div><main><div class="header">MH</div><p>x</p></main><div class="menu">M</div></div>
|
||||
);
|
||||
}
|
||||
|
||||
test "dump: strip.shell honours landmark roles" {
|
||||
try expectShellDump(
|
||||
\\<div role="navigation">N</div><div role="BANNER search">B</div><section><div role="contentinfo">C</div></section><p>x</p><div role="region"><header>RH</header></div><div role="main"><footer>MF</footer></div>
|
||||
|
||||
+17
-18
@@ -18,22 +18,20 @@
|
||||
|
||||
const std = @import("std");
|
||||
|
||||
const URL = @import("URL.zig");
|
||||
const Frame = @import("Frame.zig");
|
||||
const RenderTree = @import("RenderTree.zig");
|
||||
const StyleManager = @import("StyleManager.zig");
|
||||
const URL = @import("URL.zig");
|
||||
|
||||
const Node = @import("webapi/Node.zig");
|
||||
const Element = @import("webapi/Element.zig");
|
||||
const Slot = @import("webapi/element/html/Slot.zig");
|
||||
|
||||
const isAllWhitespace = @import("../string.zig").isAllWhitespace;
|
||||
const LimitedWriter = @import("../LimitedWriter.zig");
|
||||
const Strip = RenderTree.Strip;
|
||||
const isAllWhitespace = @import("../string.zig").isAllWhitespace;
|
||||
|
||||
pub const Opts = struct {
|
||||
max_bytes: ?u32 = null,
|
||||
strip: Strip = .{},
|
||||
};
|
||||
|
||||
const truncation_marker = LimitedWriter.truncation_marker;
|
||||
@@ -417,14 +415,15 @@ const Context = struct {
|
||||
}
|
||||
};
|
||||
|
||||
pub fn dump(node: *Node, opts: Opts, writer: *std.Io.Writer, frame: *Frame) !void {
|
||||
pub fn dump(state: RenderTree.State, opts: Opts, writer: *std.Io.Writer, frame: *Frame) !void {
|
||||
const node = state.root;
|
||||
if (opts.max_bytes) |limit| {
|
||||
var lw = LimitedWriter.init(writer, limit);
|
||||
var ctx: Context = .{
|
||||
.state = .{},
|
||||
.writer = &lw.writer,
|
||||
.frame = frame,
|
||||
.tree = .{ .frame = frame, .root = node, .strip = opts.strip },
|
||||
.tree = .{ .frame = frame, .state = state },
|
||||
};
|
||||
ctx.render(node) catch |err| switch (err) {
|
||||
error.WriteFailed => {
|
||||
@@ -443,7 +442,7 @@ pub fn dump(node: *Node, opts: Opts, writer: *std.Io.Writer, frame: *Frame) !voi
|
||||
.state = .{},
|
||||
.writer = writer,
|
||||
.frame = frame,
|
||||
.tree = .{ .frame = frame, .root = node, .strip = opts.strip },
|
||||
.tree = .{ .frame = frame, .state = state },
|
||||
};
|
||||
try ctx.render(node);
|
||||
if (!ctx.state.last_char_was_newline) {
|
||||
@@ -465,7 +464,7 @@ fn testMarkdownHTML(html: []const u8, expected: []const u8) !void {
|
||||
|
||||
var aw: std.Io.Writer.Allocating = .init(testing.allocator);
|
||||
defer aw.deinit();
|
||||
try dump(div.asNode(), .{}, &aw.writer, frame);
|
||||
try dump(.{ .root = div.asNode() }, .{}, &aw.writer, frame);
|
||||
|
||||
try testing.expectString(expected, aw.written());
|
||||
}
|
||||
@@ -532,7 +531,7 @@ test "browser.markdown: flex from a stylesheet" {
|
||||
const frame = page.frame().?;
|
||||
|
||||
var aw: std.Io.Writer.Allocating = .init(testing.arena_allocator);
|
||||
try dump(frame.window._document.asNode(), .{}, &aw.writer, frame);
|
||||
try dump(.{ .root = frame.window._document.asNode() }, .{}, &aw.writer, frame);
|
||||
try testing.expectString(
|
||||
\\[Title **Aug 04 2026**](http://127.0.0.1:9582/p)
|
||||
\\
|
||||
@@ -700,7 +699,7 @@ test "browser.markdown: resolve links" {
|
||||
|
||||
var aw: std.Io.Writer.Allocating = .init(testing.allocator);
|
||||
defer aw.deinit();
|
||||
try dump(div.asNode(), .{}, &aw.writer, frame);
|
||||
try dump(.{ .root = div.asNode() }, .{}, &aw.writer, frame);
|
||||
|
||||
try testing.expectString(
|
||||
\\[Link](https://example.com/a/b)
|
||||
@@ -751,7 +750,7 @@ test "browser.markdown: stylesheet display:none is skipped" {
|
||||
const frame = page.frame().?;
|
||||
|
||||
var aw: std.Io.Writer.Allocating = .init(testing.arena_allocator);
|
||||
try dump(frame.window._document.asNode(), .{}, &aw.writer, frame);
|
||||
try dump(.{ .root = frame.window._document.asNode() }, .{}, &aw.writer, frame);
|
||||
|
||||
try testing.expectString(
|
||||
\\
|
||||
@@ -783,7 +782,7 @@ test "browser.markdown: scoped dump of a hidden subtree still renders it" {
|
||||
|
||||
var aw: std.Io.Writer.Allocating = .init(testing.allocator);
|
||||
defer aw.deinit();
|
||||
try dump(modal, .{}, &aw.writer, frame);
|
||||
try dump(.{ .root = modal }, .{}, &aw.writer, frame);
|
||||
|
||||
try testing.expectString("\ndialog text\n", aw.written());
|
||||
}
|
||||
@@ -799,7 +798,7 @@ test "browser.markdown: strip.ui drops images and other visual elements" {
|
||||
|
||||
var aw: std.Io.Writer.Allocating = .init(testing.allocator);
|
||||
defer aw.deinit();
|
||||
try dump(div.asNode(), .{ .strip = .{ .ui = true } }, &aw.writer, frame);
|
||||
try dump(.{ .root = div.asNode(), .strip = .{ .ui = true } }, .{}, &aw.writer, frame);
|
||||
|
||||
try testing.expectString("\nText more\n", aw.written());
|
||||
}
|
||||
@@ -815,7 +814,7 @@ test "browser.markdown: strip.shell drops page chrome" {
|
||||
|
||||
var aw: std.Io.Writer.Allocating = .init(testing.allocator);
|
||||
defer aw.deinit();
|
||||
try dump(div.asNode(), .{ .strip = .{ .shell = true } }, &aw.writer, frame);
|
||||
try dump(.{ .root = div.asNode(), .strip = .{ .shell = true } }, .{}, &aw.writer, frame);
|
||||
|
||||
try testing.expectString("\nBody\n", aw.written());
|
||||
}
|
||||
@@ -831,7 +830,7 @@ test "browser.markdown: max_bytes leaves output untouched when under cap" {
|
||||
|
||||
var aw: std.Io.Writer.Allocating = .init(testing.allocator);
|
||||
defer aw.deinit();
|
||||
try dump(div.asNode(), .{ .max_bytes = 1024 }, &aw.writer, frame);
|
||||
try dump(.{ .root = div.asNode() }, .{ .max_bytes = 1024 }, &aw.writer, frame);
|
||||
|
||||
try testing.expectString("\nShort\n", aw.written());
|
||||
}
|
||||
@@ -847,7 +846,7 @@ test "browser.markdown: max_bytes truncates with marker" {
|
||||
|
||||
var aw: std.Io.Writer.Allocating = .init(testing.allocator);
|
||||
defer aw.deinit();
|
||||
try dump(div.asNode(), .{ .max_bytes = 50 }, &aw.writer, frame);
|
||||
try dump(.{ .root = div.asNode() }, .{ .max_bytes = 50 }, &aw.writer, frame);
|
||||
|
||||
const out = aw.written();
|
||||
try testing.expect(std.mem.endsWith(u8, out, "[truncated]\n"));
|
||||
@@ -874,7 +873,7 @@ fn testMarkdownShadow(light: []const u8, shadow: []const u8, expected: []const u
|
||||
|
||||
var aw: std.Io.Writer.Allocating = .init(testing.allocator);
|
||||
defer aw.deinit();
|
||||
try dump(host.asNode(), .{}, &aw.writer, frame);
|
||||
try dump(.{ .root = host.asNode() }, .{}, &aw.writer, frame);
|
||||
|
||||
try testing.expectString(expected, aw.written());
|
||||
}
|
||||
@@ -924,7 +923,7 @@ test "browser.markdown: declarative shadow DOM renders through piercing" {
|
||||
|
||||
var aw: std.Io.Writer.Allocating = .init(testing.allocator);
|
||||
defer aw.deinit();
|
||||
try dump(host.asNode(), .{}, &aw.writer, frame);
|
||||
try dump(.{ .root = host.asNode() }, .{}, &aw.writer, frame);
|
||||
|
||||
try testing.expectString("\nshadow content\n", aw.written());
|
||||
}
|
||||
+16
-17
@@ -56,7 +56,6 @@ pub const Opts = struct {
|
||||
/// that ends up empty is an error. `parsePageRanges` reads CDP's text
|
||||
/// form.
|
||||
page_ranges: []const PageRange = &.{},
|
||||
strip: RenderTree.Strip = .{},
|
||||
};
|
||||
|
||||
/// 1-based, inclusive. `to = maxInt(u32)` for an open end ("5-").
|
||||
@@ -97,8 +96,8 @@ fn parsePageNumber(s: []const u8) ?u32 {
|
||||
}
|
||||
|
||||
/// Prints `node` as a paginated, text PDF.
|
||||
pub fn print(arena: Allocator, node: *Node, opts: Opts, writer: *std.Io.Writer, frame: *Frame) !void {
|
||||
const prepared = try prepare(arena, node, opts, frame);
|
||||
pub fn print(arena: Allocator, state: RenderTree.State, opts: Opts, writer: *std.Io.Writer, frame: *Frame) !void {
|
||||
const prepared = try prepare(arena, state, opts, frame);
|
||||
return prepared.write(writer);
|
||||
}
|
||||
|
||||
@@ -107,7 +106,7 @@ pub fn print(arena: Allocator, node: *Node, opts: Opts, writer: *std.Io.Writer,
|
||||
/// that can fail does so here, before any output: options, and a page
|
||||
/// selection that hits no page (that one costs a layout pass, only paid
|
||||
/// when ranges are given).
|
||||
pub fn prepare(arena: Allocator, node: *Node, opts: Opts, frame: *Frame) !Prepared {
|
||||
pub fn prepare(arena: Allocator, state: RenderTree.State, opts: Opts, frame: *Frame) !Prepared {
|
||||
const content_w = opts.paper_width - opts.margin_left - opts.margin_right;
|
||||
const content_h = opts.paper_height - opts.margin_top - opts.margin_bottom;
|
||||
if (!(opts.paper_width > 0 and opts.paper_height > 0) or
|
||||
@@ -123,7 +122,7 @@ pub fn prepare(arena: Allocator, node: *Node, opts: Opts, frame: *Frame) !Prepar
|
||||
const prepared: Prepared = .{
|
||||
.arena = arena,
|
||||
.opts = opts,
|
||||
.blocks = try screenshot.collect(arena, node, opts.strip, frame),
|
||||
.blocks = try screenshot.collect(arena, state, frame),
|
||||
.renderer = try screenshot.rendererFor(frame),
|
||||
};
|
||||
if (opts.page_ranges.len > 0) {
|
||||
@@ -1039,7 +1038,7 @@ test "browser.pdf: structure, pagination and links" {
|
||||
try Frame.parse.htmlAsChildren(frame, div.asNode(), "<h1>Title</h1><p>Hello <b>world</b> <a href='/x'>link</a></p><pre>code</pre>");
|
||||
|
||||
var aw: std.Io.Writer.Allocating = .init(testing.arena_allocator);
|
||||
try print(testing.arena_allocator, div.asNode(), .{}, &aw.writer, frame);
|
||||
try print(testing.arena_allocator, .{ .root = div.asNode() }, .{}, &aw.writer, frame);
|
||||
const out = aw.written();
|
||||
try testing.expectEqual("%PDF-1.4\n", out[0..9]);
|
||||
try testing.expectEqual("%%EOF\n", out[out.len - 6 ..]);
|
||||
@@ -1064,28 +1063,28 @@ test "browser.pdf: structure, pagination and links" {
|
||||
try Frame.parse.htmlAsChildren(frame, long.asNode(), html.items);
|
||||
|
||||
aw.clearRetainingCapacity();
|
||||
try print(testing.arena_allocator, long.asNode(), .{}, &aw.writer, frame);
|
||||
try print(testing.arena_allocator, .{ .root = long.asNode() }, .{}, &aw.writer, frame);
|
||||
const pages = std.mem.count(u8, aw.written(), "/Type /Page ");
|
||||
try testing.expectEqual(true, pages >= 5 and pages <= 8);
|
||||
|
||||
aw.clearRetainingCapacity();
|
||||
try print(testing.arena_allocator, long.asNode(), .{ .page_ranges = try parsePageRanges(testing.arena_allocator, "2-3, 5") }, &aw.writer, frame);
|
||||
try print(testing.arena_allocator, .{ .root = long.asNode() }, .{ .page_ranges = try parsePageRanges(testing.arena_allocator, "2-3, 5") }, &aw.writer, frame);
|
||||
try testing.expectEqual(3, std.mem.count(u8, aw.written(), "/Type /Page "));
|
||||
|
||||
// Document order, once each, the tail past the end ignored.
|
||||
aw.clearRetainingCapacity();
|
||||
try print(testing.arena_allocator, long.asNode(), .{ .page_ranges = try parsePageRanges(testing.arena_allocator, "5, 2-3, 3, 4000-") }, &aw.writer, frame);
|
||||
try print(testing.arena_allocator, .{ .root = long.asNode() }, .{ .page_ranges = try parsePageRanges(testing.arena_allocator, "5, 2-3, 3, 4000-") }, &aw.writer, frame);
|
||||
try testing.expectEqual(3, std.mem.count(u8, aw.written(), "/Type /Page "));
|
||||
|
||||
// Halving the scale roughly halves the pages; landscape is the caller's
|
||||
// swap, and margins shrink the content box.
|
||||
aw.clearRetainingCapacity();
|
||||
try print(testing.arena_allocator, long.asNode(), .{ .scale = 0.5 }, &aw.writer, frame);
|
||||
try print(testing.arena_allocator, .{ .root = long.asNode() }, .{ .scale = 0.5 }, &aw.writer, frame);
|
||||
const half = std.mem.count(u8, aw.written(), "/Type /Page ");
|
||||
try testing.expectEqual(true, half < pages and half >= pages / 3);
|
||||
|
||||
aw.clearRetainingCapacity();
|
||||
try print(testing.arena_allocator, long.asNode(), .{ .paper_width = 1056, .paper_height = 816, .margin_top = 300, .margin_bottom = 300 }, &aw.writer, frame);
|
||||
try print(testing.arena_allocator, .{ .root = long.asNode() }, .{ .paper_width = 1056, .paper_height = 816, .margin_top = 300, .margin_bottom = 300 }, &aw.writer, frame);
|
||||
try testing.expectEqual(true, std.mem.indexOf(u8, aw.written(), "/MediaBox [0 0 792.000 612.000]") != null);
|
||||
try testing.expectEqual(true, std.mem.count(u8, aw.written(), "/Type /Page ") > pages);
|
||||
}
|
||||
@@ -1102,17 +1101,17 @@ test "browser.pdf: rejects bad options" {
|
||||
var discard: std.Io.Writer.Discarding = .init(&.{});
|
||||
const w = &discard.writer;
|
||||
const a = testing.arena_allocator;
|
||||
try testing.expectError(error.InvalidPdfOptions, print(a, div.asNode(), .{ .scale = 3 }, w, frame));
|
||||
try testing.expectError(error.InvalidPdfOptions, print(a, div.asNode(), .{ .paper_width = 0 }, w, frame));
|
||||
try testing.expectError(error.InvalidPdfOptions, print(a, div.asNode(), .{ .margin_left = 500, .margin_right = 500 }, w, frame));
|
||||
try testing.expectError(error.InvalidPdfOptions, print(a, div.asNode(), .{ .page_ranges = &.{.{ .from = 3, .to = 1 }} }, w, frame));
|
||||
try testing.expectError(error.InvalidPdfOptions, print(a, .{ .root = div.asNode() }, .{ .scale = 3 }, w, frame));
|
||||
try testing.expectError(error.InvalidPdfOptions, print(a, .{ .root = div.asNode() }, .{ .paper_width = 0 }, w, frame));
|
||||
try testing.expectError(error.InvalidPdfOptions, print(a, .{ .root = div.asNode() }, .{ .margin_left = 500, .margin_right = 500 }, w, frame));
|
||||
try testing.expectError(error.InvalidPdfOptions, print(a, .{ .root = div.asNode() }, .{ .page_ranges = &.{.{ .from = 3, .to = 1 }} }, w, frame));
|
||||
// Well-formed, but this is a one-page document.
|
||||
try testing.expectError(error.PageRangeExceedsPageCount, print(a, div.asNode(), .{ .page_ranges = &.{.{ .from = 7, .to = 9 }} }, w, frame));
|
||||
try testing.expectError(error.PageRangeExceedsPageCount, print(a, .{ .root = div.asNode() }, .{ .page_ranges = &.{.{ .from = 7, .to = 9 }} }, w, frame));
|
||||
|
||||
// Far too small for the file, so the sink refuses partway through.
|
||||
var buf: [64]u8 = undefined;
|
||||
var fixed = std.Io.Writer.fixed(&buf);
|
||||
try testing.expectError(error.WriteFailed, print(a, div.asNode(), .{}, &fixed, frame));
|
||||
try testing.expectError(error.WriteFailed, print(a, .{ .root = div.asNode() }, .{}, &fixed, frame));
|
||||
}
|
||||
|
||||
test "browser.pdf: parsePageRanges follows the CDP grammar" {
|
||||
|
||||
+24
-25
@@ -40,7 +40,6 @@ pub const Opts = struct {
|
||||
height: u32 = 0,
|
||||
clip: ?Clip = null,
|
||||
scale: f32 = 1.0,
|
||||
strip: RenderTree.Strip = .{},
|
||||
|
||||
const Clip = struct {
|
||||
x: f32,
|
||||
@@ -117,38 +116,38 @@ pub fn rendererFor(frame: *Frame) !*Renderer {
|
||||
return r;
|
||||
}
|
||||
|
||||
pub fn png(arena: Allocator, node: *Node, opts: Opts, writer: *std.Io.Writer, frame: *Frame) !u32 {
|
||||
const prepared = try preparePng(arena, node, opts, frame);
|
||||
pub fn png(arena: Allocator, state: RenderTree.State, opts: Opts, writer: *std.Io.Writer, frame: *Frame) !u32 {
|
||||
const prepared = try preparePng(arena, state, opts, frame);
|
||||
return prepared.write(writer);
|
||||
}
|
||||
|
||||
/// The height a render at `width` would have.
|
||||
pub fn contentHeight(arena: Allocator, node: *Node, width: u32, frame: *Frame) !u32 {
|
||||
const prepared = try preparePng(arena, node, .{ .width = width }, frame);
|
||||
const prepared = try preparePng(arena, .{ .root = node }, .{ .width = width }, frame);
|
||||
return prepared.measure();
|
||||
}
|
||||
|
||||
// The DOM walk, done up front so it can fail (allocation) before any output
|
||||
// starts. Rasterizing is then a pure write: `Prepared` can be embedded in a
|
||||
// std.json value and streams itself as a base64 string.
|
||||
pub fn preparePng(arena: Allocator, node: *Node, opts: Opts, frame: *Frame) !Prepared {
|
||||
pub fn preparePng(arena: Allocator, state: RenderTree.State, opts: Opts, frame: *Frame) !Prepared {
|
||||
if (opts.width == 0 or !(opts.scale > 0 and opts.scale <= 8)) {
|
||||
return error.InvalidScreenshotOptions;
|
||||
}
|
||||
return .{
|
||||
.opts = opts,
|
||||
.blocks = try collect(arena, node, opts.strip, frame),
|
||||
.blocks = try collect(arena, state, frame),
|
||||
.renderer = try rendererFor(frame),
|
||||
};
|
||||
}
|
||||
|
||||
pub fn collect(arena: Allocator, node: *Node, strip: RenderTree.Strip, frame: *Frame) ![]const LpBlock {
|
||||
pub fn collect(arena: Allocator, state: RenderTree.State, frame: *Frame) ![]const LpBlock {
|
||||
var builder: Builder = .{
|
||||
.frame = frame,
|
||||
.arena = arena,
|
||||
.tree = .{ .frame = frame, .root = node, .strip = strip },
|
||||
.tree = .{ .frame = frame, .state = state },
|
||||
};
|
||||
try builder.render(node);
|
||||
try builder.render(state.root);
|
||||
try builder.closeBlock();
|
||||
return builder.blocks.items;
|
||||
}
|
||||
@@ -1170,7 +1169,7 @@ test "browser.screenshot: fixed height, clip and scale" {
|
||||
try Frame.parse.htmlAsChildren(frame, div.asNode(), "<p>one</p><p>two</p><p>three</p>");
|
||||
|
||||
var aw: std.Io.Writer.Allocating = .init(testing.arena_allocator);
|
||||
const content_height = try png(testing.arena_allocator, div.asNode(), .{ .width = 300, .height = 50, .scale = 2.0 }, &aw.writer, frame);
|
||||
const content_height = try png(testing.arena_allocator, .{ .root = div.asNode() }, .{ .width = 300, .height = 50, .scale = 2.0 }, &aw.writer, frame);
|
||||
try testing.expectEqual(true, content_height > 50);
|
||||
try testing.expectEqual(600, std.mem.readInt(u32, aw.written()[16..20], .big));
|
||||
try testing.expectEqual(100, std.mem.readInt(u32, aw.written()[20..24], .big));
|
||||
@@ -1178,7 +1177,7 @@ test "browser.screenshot: fixed height, clip and scale" {
|
||||
try testing.expectEqual(content_height, try contentHeight(frame.call_arena, div.asNode(), 300, frame));
|
||||
|
||||
aw.clearRetainingCapacity();
|
||||
_ = try png(frame.call_arena, div.asNode(), .{
|
||||
_ = try png(frame.call_arena, .{ .root = div.asNode() }, .{
|
||||
.width = 300,
|
||||
.clip = .{ .x = 10, .y = 10, .width = 100, .height = 40 },
|
||||
}, &aw.writer, frame);
|
||||
@@ -1198,7 +1197,7 @@ test "browser.screenshot: a clip past the viewport extends the strip" {
|
||||
try testing.expectEqual(true, full > 100);
|
||||
|
||||
var aw: std.Io.Writer.Allocating = .init(testing.arena_allocator);
|
||||
_ = try png(frame.call_arena, div.asNode(), .{
|
||||
_ = try png(frame.call_arena, .{ .root = div.asNode() }, .{
|
||||
.width = 300,
|
||||
.height = 100,
|
||||
.clip = .{ .x = 0, .y = 0, .width = 300, .height = @floatFromInt(full) },
|
||||
@@ -1209,7 +1208,7 @@ test "browser.screenshot: a clip past the viewport extends the strip" {
|
||||
// Never past the content, so an absurd probe resolves to the full page
|
||||
// instead of a 1e8-tall raster.
|
||||
aw.clearRetainingCapacity();
|
||||
_ = try png(testing.arena_allocator, div.asNode(), .{
|
||||
_ = try png(testing.arena_allocator, .{ .root = div.asNode() }, .{
|
||||
.width = 300,
|
||||
.height = 100,
|
||||
.clip = .{ .x = 0, .y = 0, .width = 300, .height = 1e8 },
|
||||
@@ -1227,12 +1226,12 @@ test "browser.screenshot: raster is bounded" {
|
||||
|
||||
var aw: std.Io.Writer.Allocating = .init(testing.arena_allocator);
|
||||
|
||||
_ = try png(testing.arena_allocator, div.asNode(), .{ .width = 300000, .height = 8 }, &aw.writer, frame);
|
||||
_ = try png(testing.arena_allocator, .{ .root = div.asNode() }, .{ .width = 300000, .height = 8 }, &aw.writer, frame);
|
||||
try testing.expectEqual(16384, std.mem.readInt(u32, aw.written()[16..20], .big));
|
||||
try testing.expectEqual(8, std.mem.readInt(u32, aw.written()[20..24], .big));
|
||||
|
||||
aw.clearRetainingCapacity();
|
||||
_ = try png(testing.arena_allocator, div.asNode(), .{ .width = 8, .height = 100000 }, &aw.writer, frame);
|
||||
_ = try png(testing.arena_allocator, .{ .root = div.asNode() }, .{ .width = 8, .height = 100000 }, &aw.writer, frame);
|
||||
try testing.expectEqual(8, std.mem.readInt(u32, aw.written()[16..20], .big));
|
||||
try testing.expectEqual(16384, std.mem.readInt(u32, aw.written()[20..24], .big));
|
||||
}
|
||||
@@ -1250,7 +1249,7 @@ test "browser.screenshot: a refused write fails the capture" {
|
||||
// has to surface as an error and not as a truncated image.
|
||||
var buf: [64]u8 = undefined;
|
||||
var w = std.Io.Writer.fixed(&buf);
|
||||
try testing.expectError(error.WriteFailed, png(testing.arena_allocator, div.asNode(), .{ .width = 300 }, &w, frame));
|
||||
try testing.expectError(error.WriteFailed, png(testing.arena_allocator, .{ .root = div.asNode() }, .{ .width = 300 }, &w, frame));
|
||||
}
|
||||
|
||||
test "browser.screenshot: json streams base64" {
|
||||
@@ -1261,7 +1260,7 @@ test "browser.screenshot: json streams base64" {
|
||||
const div = try doc.createElement("div", null, frame);
|
||||
try Frame.parse.htmlAsChildren(frame, div.asNode(), "<p>hello</p>");
|
||||
|
||||
const prepared = try preparePng(testing.arena_allocator, div.asNode(), .{ .width = 200 }, frame);
|
||||
const prepared = try preparePng(testing.arena_allocator, .{ .root = div.asNode() }, .{ .width = 200 }, frame);
|
||||
|
||||
var raw: std.Io.Writer.Allocating = .init(testing.allocator);
|
||||
defer raw.deinit();
|
||||
@@ -1300,7 +1299,7 @@ test "browser.screenshot: block extraction" {
|
||||
\\<div> </div>
|
||||
);
|
||||
|
||||
var builder: Builder = .{ .arena = testing.arena_allocator, .frame = frame, .tree = .{ .frame = frame, .root = div.asNode() } };
|
||||
var builder: Builder = .{ .arena = testing.arena_allocator, .frame = frame, .tree = .{ .frame = frame, .state = .{ .root = div.asNode() } } };
|
||||
try builder.render(div.asNode());
|
||||
try builder.closeBlock();
|
||||
|
||||
@@ -1373,7 +1372,7 @@ test "browser.screenshot: adjacent anchors" {
|
||||
\\<p><a href="/a">Log In</a><a href="/b">Sign Up</a><b>!</b> see <a href="/c">this</a>.</p>
|
||||
);
|
||||
|
||||
var builder: Builder = .{ .arena = testing.arena_allocator, .frame = frame, .tree = .{ .frame = frame, .root = div.asNode() } };
|
||||
var builder: Builder = .{ .arena = testing.arena_allocator, .frame = frame, .tree = .{ .frame = frame, .state = .{ .root = div.asNode() } } };
|
||||
try builder.render(div.asNode());
|
||||
try builder.closeBlock();
|
||||
const blocks = builder.blocks.items;
|
||||
@@ -1395,7 +1394,7 @@ test "browser.screenshot: standalone anchors get their own block" {
|
||||
\\<p>inline <a href="/x">link</a> here</p>
|
||||
);
|
||||
|
||||
var builder: Builder = .{ .arena = testing.arena_allocator, .frame = frame, .tree = .{ .frame = frame, .root = div.asNode() } };
|
||||
var builder: Builder = .{ .arena = testing.arena_allocator, .frame = frame, .tree = .{ .frame = frame, .state = .{ .root = div.asNode() } } };
|
||||
try builder.render(div.asNode());
|
||||
try builder.closeBlock();
|
||||
const blocks = builder.blocks.items;
|
||||
@@ -1421,7 +1420,7 @@ test "browser.screenshot: shadow dom and slots" {
|
||||
\\<x-host><template shadowrootmode="open"><p>shadow <slot></slot></p></template>light</x-host>
|
||||
, frame);
|
||||
|
||||
var builder: Builder = .{ .arena = testing.arena_allocator, .frame = frame, .tree = .{ .frame = frame, .root = div.asNode() } };
|
||||
var builder: Builder = .{ .arena = testing.arena_allocator, .frame = frame, .tree = .{ .frame = frame, .state = .{ .root = div.asNode() } } };
|
||||
try builder.render(div.asNode());
|
||||
try builder.closeBlock();
|
||||
const blocks = builder.blocks.items;
|
||||
@@ -1488,7 +1487,7 @@ test "browser.screenshot: flex from a stylesheet" {
|
||||
// The text of each block on its own line: what renders, not how.
|
||||
fn collectLines(node: *Node, frame: *Frame) ![]const u8 {
|
||||
const arena = testing.arena_allocator;
|
||||
const blocks = try collect(arena, node, .{}, frame);
|
||||
const blocks = try collect(arena, .{ .root = node }, frame);
|
||||
var out: std.ArrayList(u8) = .empty;
|
||||
for (blocks, 0..) |b, i| {
|
||||
if (i > 0) try out.append(arena, '\n');
|
||||
@@ -1506,7 +1505,7 @@ fn testPng(html: []const u8, width: u32) ![]const u8 {
|
||||
try Frame.parse.htmlAsChildren(frame, div.asNode(), html);
|
||||
|
||||
var aw: std.Io.Writer.Allocating = .init(testing.arena_allocator);
|
||||
_ = try png(testing.arena_allocator, div.asNode(), .{ .width = width }, &aw.writer, frame);
|
||||
_ = try png(testing.arena_allocator, .{ .root = div.asNode() }, .{ .width = width }, &aw.writer, frame);
|
||||
return aw.written();
|
||||
}
|
||||
|
||||
@@ -1529,10 +1528,10 @@ test "browser.screenshot: collect honours strip flags" {
|
||||
}
|
||||
};
|
||||
|
||||
const all = try collect(arena, div.asNode(), .{}, frame);
|
||||
const all = try collect(arena, .{ .root = div.asNode() }, frame);
|
||||
try testing.expectEqual(4, all.len);
|
||||
|
||||
const stripped = try collect(arena, div.asNode(), .{ .shell = true, .ui = true }, frame);
|
||||
const stripped = try collect(arena, .{ .root = div.asNode(), .strip = .{ .shell = true, .ui = true } }, frame);
|
||||
try testing.expectEqual(1, stripped.len);
|
||||
try testing.expectEqual("Body text", try S.text(stripped[0], arena));
|
||||
}
|
||||
+13
-11
@@ -1296,7 +1296,7 @@ fn writeSingleLine(w: *std.Io.Writer, text: []const u8) !void {
|
||||
|
||||
fn renderFrameMarkdown(arena: std.mem.Allocator, frame: *lp.Frame) ToolError![]const u8 {
|
||||
var aw: std.Io.Writer.Allocating = .init(arena);
|
||||
lp.markdown.dump(frame.document.asNode(), .{}, &aw.writer, frame) catch
|
||||
lp.markdown.dump(.{ .root = frame.document.asNode() }, .{}, &aw.writer, frame) catch
|
||||
return ToolError.InternalError;
|
||||
return aw.written();
|
||||
}
|
||||
@@ -1315,7 +1315,7 @@ fn execMarkdown(arena: std.mem.Allocator, session: *lp.Session, registry: *NodeR
|
||||
const node = try resolveScope(session, registry, page, args.selector, args.backendNodeId);
|
||||
|
||||
var aw: std.Io.Writer.Allocating = .init(arena);
|
||||
lp.markdown.dump(node, .{ .max_bytes = args.maxBytes }, &aw.writer, page) catch return ToolError.InternalError;
|
||||
lp.markdown.dump(.{ .root = node }, .{ .max_bytes = args.maxBytes }, &aw.writer, page) catch return ToolError.InternalError;
|
||||
return aw.written();
|
||||
}
|
||||
|
||||
@@ -1340,16 +1340,18 @@ fn execHtml(arena: std.mem.Allocator, session: *lp.Session, registry: *NodeRegis
|
||||
const page = try ensurePage(session, registry, args.url, args.timeout);
|
||||
|
||||
const whole = args.selector == null and args.backendNodeId == null;
|
||||
const node = if (whole) page.document.asNode() else (try resolveTarget(session, registry, args.selector, args.backendNodeId)).node;
|
||||
const target = if (whole) page.document.asNode() else (try resolveTarget(session, registry, args.selector, args.backendNodeId)).node;
|
||||
const state = lp.RenderTree.resolve(arena, target, args.strip, page) catch return ToolError.OutOfMemory;
|
||||
const opts: lp.dump.Opts = .{
|
||||
.strip = lp.RenderTree.resolveStrip(node, args.strip, page),
|
||||
.strip = state.strip,
|
||||
.pruned = state.pruned,
|
||||
.max_bytes = args.maxBytes,
|
||||
};
|
||||
var aw: std.Io.Writer.Allocating = .init(arena);
|
||||
if (whole) {
|
||||
lp.dump.root(page.document, opts, &aw.writer, page) catch return ToolError.InternalError;
|
||||
if (state.root.is(DOMNode.Document)) |document| {
|
||||
lp.dump.root(document, opts, &aw.writer, page) catch return ToolError.InternalError;
|
||||
} else {
|
||||
lp.dump.deep(node, opts, &aw.writer, page) catch return ToolError.InternalError;
|
||||
lp.dump.deep(state.root, opts, &aw.writer, page) catch return ToolError.InternalError;
|
||||
}
|
||||
return aw.written();
|
||||
}
|
||||
@@ -1371,10 +1373,10 @@ fn execScreenshot(arena: std.mem.Allocator, session: *lp.Session, registry: *Nod
|
||||
return .{ .text = "pass `path`: this client cannot display an inline image", .is_error = true };
|
||||
}
|
||||
const page = try ensurePage(session, registry, args.url, args.timeout);
|
||||
const node = try resolveScope(session, registry, page, args.selector, args.backendNodeId);
|
||||
var opts: lp.screenshot.Opts = .fromViewport(page._page.getViewport(), args.fullPage);
|
||||
opts.strip = lp.RenderTree.resolveStrip(node, args.strip, page);
|
||||
var prepared = lp.screenshot.preparePng(arena, node, opts, page) catch
|
||||
const scope = try resolveScope(session, registry, page, args.selector, args.backendNodeId);
|
||||
const state = lp.RenderTree.resolve(arena, scope, args.strip, page) catch return ToolError.OutOfMemory;
|
||||
const opts: lp.screenshot.Opts = .fromViewport(page._page.getViewport(), args.fullPage);
|
||||
var prepared = lp.screenshot.preparePng(arena, state, opts, page) catch
|
||||
return ToolError.InternalError;
|
||||
|
||||
if (args.path) |path| {
|
||||
|
||||
File diff suppressed because one or more lines are too long.
+6
-2
@@ -120,11 +120,15 @@
|
||||
\\ Tag group to remove from dump. Can be passed multiple times.
|
||||
\\ Defaults to stripping nothing.
|
||||
\\ Allowed values:
|
||||
\\ clutter Heuristic to keep only the main content, in the
|
||||
\\ manner of reader modes. May fallback to `shell`
|
||||
\\ (which my itself fallback to a full dump).
|
||||
\\ css Includes style and link[rel=stylesheet].
|
||||
\\ js Script and link[as=script, rel=preload].
|
||||
\\ invisible Best-effort (e.g. display:none) hidden elements
|
||||
\\ shell Heuristic to attempt to remove headers, footer,
|
||||
\\ and other non-content decoration.
|
||||
\\ shell Page chrome by markup: nav, aside, dialog,
|
||||
\\ page-level header and footer. Full dump when
|
||||
\\ `shell` would remove most of the text.
|
||||
\\ ui Includes img, picture, video, CSS and SVG.
|
||||
\\ --terminate-ms <INT>
|
||||
\\ Hard deadline in milliseconds. After this time elapses, JavaScript
|
||||
|
||||
+43
-45
@@ -19,53 +19,55 @@
|
||||
const std = @import("std");
|
||||
|
||||
pub const log = @import("log.zig");
|
||||
pub const datetime = @import("datetime.zig");
|
||||
pub const mcp = @import("mcp.zig");
|
||||
pub const App = @import("App.zig");
|
||||
pub const Arena = @import("Arena.zig");
|
||||
pub const ArenaPool = @import("ArenaPool.zig");
|
||||
pub const Network = @import("network/Network.zig");
|
||||
pub const Config = @import("Config.zig");
|
||||
pub const cookies = @import("cookies.zig");
|
||||
pub const datetime = @import("datetime.zig");
|
||||
pub const core_dump = @import("core_dump.zig");
|
||||
pub const ArenaPool = @import("ArenaPool.zig");
|
||||
pub const build_config = @import("build_config");
|
||||
pub const String = @import("string.zig").String;
|
||||
pub const Notification = @import("Notification.zig");
|
||||
pub const ToolSession = @import("ToolSession.zig");
|
||||
pub const Notification = @import("Notification.zig");
|
||||
pub const Server = @import("server/Server.zig");
|
||||
pub const Base64Writer = @import("Base64Writer.zig");
|
||||
pub const SemanticTree = @import("SemanticTree.zig");
|
||||
pub const NodeRegistry = @import("NodeRegistry.zig");
|
||||
pub const crash_handler = @import("crash_handler.zig");
|
||||
|
||||
pub const Network = @import("network/Network.zig");
|
||||
pub const HttpClient = @import("network/HttpClient.zig");
|
||||
|
||||
pub const pdf = @import("browser/pdf.zig");
|
||||
pub const URL = @import("browser/URL.zig");
|
||||
pub const dump = @import("browser/dump.zig");
|
||||
pub const Page = @import("browser/Page.zig");
|
||||
pub const Frame = @import("browser/Frame.zig");
|
||||
pub const forms = @import("browser/forms.zig");
|
||||
pub const links = @import("browser/links.zig");
|
||||
pub const tools = @import("browser/tools.zig");
|
||||
pub const actions = @import("browser/actions.zig");
|
||||
pub const Browser = @import("browser/Browser.zig");
|
||||
pub const Session = @import("browser/Session.zig");
|
||||
pub const markdown = @import("browser/markdown.zig");
|
||||
pub const screenshot = @import("browser/screenshot.zig");
|
||||
pub const RenderTree = @import("browser/RenderTree.zig");
|
||||
pub const interactive = @import("browser/interactive.zig");
|
||||
pub const structured_data = @import("browser/structured_data.zig");
|
||||
pub const GlobalScope = @import("browser/global_scope.zig").GlobalScope;
|
||||
|
||||
pub const js = @import("browser/js/js.zig");
|
||||
pub const dump = @import("browser/dump.zig");
|
||||
pub const markdown = @import("browser/markdown.zig");
|
||||
pub const screenshot = @import("browser/screenshot.zig");
|
||||
pub const pdf = @import("browser/pdf.zig");
|
||||
pub const Base64Writer = @import("Base64Writer.zig");
|
||||
const Selector = @import("browser/webapi/selector/Selector.zig");
|
||||
const Node = @import("browser/webapi/Node.zig");
|
||||
pub const SemanticTree = @import("SemanticTree.zig");
|
||||
pub const NodeRegistry = @import("NodeRegistry.zig");
|
||||
pub const interactive = @import("browser/interactive.zig");
|
||||
pub const links = @import("browser/links.zig");
|
||||
pub const forms = @import("browser/forms.zig");
|
||||
pub const actions = @import("browser/actions.zig");
|
||||
pub const structured_data = @import("browser/structured_data.zig");
|
||||
pub const tools = @import("browser/tools.zig");
|
||||
pub const HttpClient = @import("network/HttpClient.zig");
|
||||
const Selector = @import("browser/webapi/selector/Selector.zig");
|
||||
|
||||
pub const mcp = @import("mcp.zig");
|
||||
pub const Agent = @import("agent/Agent.zig");
|
||||
pub const Command = @import("script/command.zig").Command;
|
||||
pub const skill = @import("script/skill.zig");
|
||||
pub const Recorder = @import("script/Recorder.zig");
|
||||
pub const Runtime = @import("script/Runtime.zig");
|
||||
pub const Schema = @import("script/Schema.zig");
|
||||
pub const skill = @import("script/skill.zig");
|
||||
pub const cookies = @import("cookies.zig");
|
||||
pub const build_config = @import("build_config");
|
||||
pub const crash_handler = @import("crash_handler.zig");
|
||||
pub const core_dump = @import("core_dump.zig");
|
||||
pub const Command = @import("script/command.zig").Command;
|
||||
|
||||
pub var metrics = @import("Metrics.zig"){};
|
||||
pub const IS_TEST = @import("builtin").is_test;
|
||||
@@ -456,19 +458,16 @@ const Binary = union(enum) {
|
||||
};
|
||||
|
||||
fn prepareBinary(arena: std.mem.Allocator, frame: *Frame, opts: FetchOpts) !Binary {
|
||||
const root = try dumpRoot(frame, opts.selector);
|
||||
const strip = RenderTree.resolveStrip(root, opts.dump.strip, frame);
|
||||
const state = try RenderTree.resolve(arena, try dumpRoot(frame, opts.selector), opts.dump.strip, frame);
|
||||
return switch (opts.dump_mode.?) {
|
||||
.png => .{ .png = try screenshot.preparePng(arena, root, pngOpts(frame, strip), frame) },
|
||||
.pdf => .{ .pdf = try pdf.prepare(arena, root, .{ .strip = strip }, frame) },
|
||||
.png => .{ .png = try screenshot.preparePng(arena, state, pngOpts(frame), frame) },
|
||||
.pdf => .{ .pdf = try pdf.prepare(arena, state, .{}, frame) },
|
||||
else => unreachable,
|
||||
};
|
||||
}
|
||||
|
||||
fn pngOpts(frame: *Frame, strip: dump.Opts.Strip) screenshot.Opts {
|
||||
var opts: screenshot.Opts = .fromViewport(frame._page.getViewport(), true);
|
||||
opts.strip = strip;
|
||||
return opts;
|
||||
fn pngOpts(frame: *Frame) screenshot.Opts {
|
||||
return .fromViewport(frame._page.getViewport(), true);
|
||||
}
|
||||
|
||||
fn dumpRoot(frame: *Frame, selector: ?[]const u8) !*Node {
|
||||
@@ -479,31 +478,30 @@ fn dumpRoot(frame: *Frame, selector: ?[]const u8) !*Node {
|
||||
}
|
||||
|
||||
fn dumpContent(app: *App, mode: Config.DumpFormat, opts: FetchOpts, frame: *Frame, writer: *std.Io.Writer) !void {
|
||||
const root = try dumpRoot(frame, opts.selector);
|
||||
var arena: std.heap.ArenaAllocator = .init(app.allocator);
|
||||
defer arena.deinit();
|
||||
const state = try RenderTree.resolve(arena.allocator(), try dumpRoot(frame, opts.selector), opts.dump.strip, frame);
|
||||
var dump_opts = opts.dump;
|
||||
dump_opts.strip = RenderTree.resolveStrip(root, dump_opts.strip, frame);
|
||||
dump_opts.strip = state.strip;
|
||||
dump_opts.pruned = state.pruned;
|
||||
switch (mode) {
|
||||
.html => if (opts.selector == null)
|
||||
try dump.root(frame.window._document, dump_opts, writer, frame)
|
||||
else
|
||||
try dump.deep(root, dump_opts, writer, frame),
|
||||
.markdown => try markdown.dump(root, .{ .max_bytes = dump_opts.max_bytes, .strip = dump_opts.strip }, writer, frame),
|
||||
try dump.deep(state.root, dump_opts, writer, frame),
|
||||
.markdown => try markdown.dump(state, .{ .max_bytes = dump_opts.max_bytes }, writer, frame),
|
||||
.png => {
|
||||
var arena: std.heap.ArenaAllocator = .init(app.allocator);
|
||||
defer arena.deinit();
|
||||
_ = try screenshot.png(arena.allocator(), root, pngOpts(frame, dump_opts.strip), writer, frame);
|
||||
_ = try screenshot.png(arena.allocator(), state, pngOpts(frame), writer, frame);
|
||||
},
|
||||
.pdf => {
|
||||
var arena: std.heap.ArenaAllocator = .init(app.allocator);
|
||||
defer arena.deinit();
|
||||
try pdf.print(arena.allocator(), root, .{ .strip = dump_opts.strip }, writer, frame);
|
||||
try pdf.print(arena.allocator(), state, .{}, writer, frame);
|
||||
},
|
||||
.semantic_tree, .semantic_tree_text => {
|
||||
var registry = NodeRegistry.init(app.allocator);
|
||||
defer registry.deinit();
|
||||
|
||||
const st: SemanticTree = .{
|
||||
.dom_node = root,
|
||||
.dom_node = state.root,
|
||||
.registry = ®istry,
|
||||
.frame = frame,
|
||||
.arena = frame.call_arena,
|
||||
|
||||
@@ -57,7 +57,7 @@ const ResourceStreamingResult = struct {
|
||||
log.err(.mcp, "html dump failed", .{ .err = err });
|
||||
return error.WriteFailed;
|
||||
},
|
||||
.markdown => lp.markdown.dump(self.frame.document.asNode(), .{}, &escaped.writer, self.frame) catch |err| {
|
||||
.markdown => lp.markdown.dump(.{ .root = self.frame.document.asNode() }, .{}, &escaped.writer, self.frame) catch |err| {
|
||||
log.err(.mcp, "markdown dump failed", .{ .err = err });
|
||||
return error.WriteFailed;
|
||||
},
|
||||
|
||||
@@ -174,7 +174,7 @@ fn dump(cmd: anytype) !void {
|
||||
const bc = cmd.browser_context orelse return error.NoBrowserContext;
|
||||
const frame = bc.mainFrame() orelse return error.FrameNotLoaded;
|
||||
|
||||
const root = blk: {
|
||||
const target = blk: {
|
||||
if (params.backendNodeId) |id| {
|
||||
break :blk (bc.node_registry.lookup_by_id.get(id) orelse return error.InvalidNodeId).dom;
|
||||
}
|
||||
@@ -185,15 +185,16 @@ fn dump(cmd: anytype) !void {
|
||||
break :blk frame.document.asNode();
|
||||
};
|
||||
|
||||
const strip = lp.RenderTree.resolveStrip(root, params.strip, frame);
|
||||
const state = try lp.RenderTree.resolve(cmd.arena, target, params.strip, frame);
|
||||
const root = state.root;
|
||||
|
||||
switch (params.format) {
|
||||
.html, .markdown => {
|
||||
var aw: std.Io.Writer.Allocating = .init(cmd.arena);
|
||||
defer aw.deinit();
|
||||
const opts: lp.dump.Opts = .{ .strip = strip, .max_bytes = params.maxBytes };
|
||||
const opts: lp.dump.Opts = .{ .strip = state.strip, .pruned = state.pruned, .max_bytes = params.maxBytes };
|
||||
if (params.format == .markdown) {
|
||||
try markdown.dump(root, .{ .strip = strip, .max_bytes = params.maxBytes }, &aw.writer, frame);
|
||||
try markdown.dump(state, .{ .max_bytes = params.maxBytes }, &aw.writer, frame);
|
||||
} else if (root.is(DOMNode.Document)) |doc| {
|
||||
try lp.dump.root(doc, opts, &aw.writer, frame);
|
||||
} else {
|
||||
@@ -205,16 +206,15 @@ fn dump(cmd: anytype) !void {
|
||||
if (params.maxBytes != null) {
|
||||
return error.InvalidParams;
|
||||
}
|
||||
var opts: lp.screenshot.Opts = .fromViewport(cmd.cdp.browser.getViewport(), true);
|
||||
opts.strip = strip;
|
||||
const prepared = try lp.screenshot.preparePng(cmd.arena, root, opts, frame);
|
||||
const opts: lp.screenshot.Opts = .fromViewport(cmd.cdp.browser.getViewport(), true);
|
||||
const prepared = try lp.screenshot.preparePng(cmd.arena, state, opts, frame);
|
||||
return cmd.sendResult(.{ .format = params.format, .content = prepared }, .{});
|
||||
},
|
||||
.pdf => {
|
||||
if (params.maxBytes != null) {
|
||||
return error.InvalidParams;
|
||||
}
|
||||
const prepared = try lp.pdf.prepare(cmd.arena, root, .{ .strip = strip }, frame);
|
||||
const prepared = try lp.pdf.prepare(cmd.arena, state, .{}, frame);
|
||||
return cmd.sendResult(.{ .format = params.format, .content = prepared }, .{});
|
||||
},
|
||||
}
|
||||
@@ -237,7 +237,7 @@ fn getMarkdown(cmd: anytype) !void {
|
||||
|
||||
var aw: std.Io.Writer.Allocating = .init(cmd.arena);
|
||||
defer aw.deinit();
|
||||
try markdown.dump(dom_node, .{}, &aw.writer, frame);
|
||||
try markdown.dump(.{ .root = dom_node }, .{}, &aw.writer, frame);
|
||||
|
||||
return cmd.sendResult(.{
|
||||
.markdown = aw.written(),
|
||||
|
||||
@@ -1067,7 +1067,7 @@ fn captureScreenshot(cmd: *CDP.Command) !void {
|
||||
}
|
||||
|
||||
// Prepared streams itself as base64 straight into the outgoing message.
|
||||
const shot = try lp.screenshot.preparePng(cmd.arena, frame.window._document.asNode(), opts, frame);
|
||||
const shot = try lp.screenshot.preparePng(cmd.arena, .{ .root = frame.window._document.asNode() }, opts, frame);
|
||||
return cmd.sendResult(.{ .data = shot }, .{});
|
||||
}
|
||||
|
||||
@@ -1112,7 +1112,7 @@ fn printToPDF(cmd: *CDP.Command) !void {
|
||||
error.OutOfMemory => return error.OutOfMemory,
|
||||
},
|
||||
};
|
||||
const prepared = lp.pdf.prepare(cmd.arena, frame.window._document.asNode(), opts, frame) catch |err| switch (err) {
|
||||
const prepared = lp.pdf.prepare(cmd.arena, .{ .root = frame.window._document.asNode() }, opts, frame) catch |err| switch (err) {
|
||||
error.InvalidPdfOptions => return cmd.sendError(-32602, "invalid print parameters", .{}),
|
||||
error.PageRangeExceedsPageCount => return cmd.sendError(-32000, "Page range exceeds page count", .{}),
|
||||
else => return err,
|
||||
|
||||
Reference in new issue
Block a user