Some article text.
diff --git a/src/browser/RenderTree.zig b/src/browser/RenderTree.zig
index b25d51465..62f9f0eaf 100644
--- a/src/browser/RenderTree.zig
+++ b/src/browser/RenderTree.zig
@@ -28,16 +28,24 @@ const TreeWalker = @import("webapi/TreeWalker.zig");
const Slot = @import("webapi/element/html/Slot.zig");
const dump_html = @import("dump.zig");
+const clutter = @import("clutter.zig");
const isAllWhitespace = @import("../string.zig").isAllWhitespace;
const log = lp.log;
pub const Strip = dump_html.Opts.Strip;
+pub const PruneSet = std.AutoHashMapUnmanaged(*Node, void);
const RenderTree = @This();
+// What / how we're going to render.
+pub const State = struct {
+ root: *Node, // the not to start from
+ strip: Strip = .{}, // the strip flag we'll use
+ pruned: ?*const PruneSet = null, // the nodes that'll get pruned
+};
+
+state: State,
frame: *Frame,
-root: *Node,
-strip: Strip = .{},
pub const Child = struct {
node: *Node,
@@ -121,6 +129,11 @@ pub fn classify(self: *const RenderTree, node: *Node, opts: ClassifyOpts) ?Child
return .{ .node = node, .what = .{ .element = d }, .separated = false };
}
const text_node = node.is(Node.CData.Text) orelse return null;
+ if (self.state.pruned) |set| {
+ if (set.contains(node)) {
+ return null;
+ }
+ }
var text = text_node.ownData();
if (opts.boxed) {
text = std.mem.trim(u8, text, &std.ascii.whitespace);
@@ -140,10 +153,14 @@ pub fn classify(self: *const RenderTree, node: *Node, opts: ClassifyOpts) ?Child
fn display(self: *const RenderTree, el: *Element, is_slotted: bool) ?StyleManager.Display {
const d = visibleDisplay(el, self.frame) orelse {
- if (el.asNode() != self.root) return null;
+ if (el.asNode() != self.state.root) {
+ return null;
+ }
return .other;
};
- if (dump_html.shouldStripElement(el, self.strip, self.frame)) return null;
+ if (dump_html.shouldStripElement(el, self.state.strip, self.state.pruned, self.frame)) {
+ return null;
+ }
if (!is_slotted and el.getSlot() != null) return null;
return d;
}
@@ -219,11 +236,32 @@ pub fn isStandaloneAnchor(el: *Element, frame: *Frame) bool {
return true;
}
+/// Decides once, before rendering, what a dump of `root` renders: the strip
+/// bits that survive their safeguards and, for clutter, the prune set.
+/// Every dump entry point calls this and renders the result. The prune set
+/// lives in `allocator` for as long as the dump.
+pub fn resolve(allocator: std.mem.Allocator, root: *Node, requested_strip: Strip, frame: *Frame) !State {
+ var strip = requested_strip;
+ if (strip.clutter) {
+ // The shell strip is the floor the selection stands on.
+ strip.shell = true;
+ strip.invisible = true;
+ }
+ strip = resolveShell(root, strip, frame);
+ if (strip.clutter) {
+ if (try clutter.select(allocator, root, strip, frame)) |pruned| {
+ return .{ .root = root, .strip = strip, .pruned = pruned };
+ }
+ strip.clutter = false;
+ }
+ return .{ .root = root, .strip = strip };
+}
+
/// Shell stripping is undone when it would remove most of the content. Better
/// to leave too much in than to strip too muchout. Non-link text is
/// the measure (nav and footer text is mostly links); a page with none is
/// judged on all of its text.
-pub fn resolveStrip(root: *Node, strip: Strip, frame: *Frame) Strip {
+fn resolveShell(root: *Node, strip: Strip, frame: *Frame) Strip {
if (strip.shell == false) {
return strip;
}
@@ -232,7 +270,7 @@ pub fn resolveStrip(root: *Node, strip: Strip, frame: *Frame) Strip {
render_with_shell.shell = false;
var m: Measure = .{};
- const tree: RenderTree = .{ .frame = frame, .root = root, .strip = render_with_shell };
+ const tree: RenderTree = .{ .frame = frame, .state = .{ .root = root, .strip = render_with_shell } };
tree.measure(root, .{}, &m);
const total, const shell = if (m.prose > 0) .{ m.prose, m.shell_prose } else .{ m.all, m.shell_all };
@@ -357,22 +395,22 @@ pub fn analyzeContent(root: *Node, frame: *Frame) ContentInfo {
const testing = @import("../testing.zig");
test "RenderTree: resolveStrip keeps shell when the content holds the text" {
- try testing.expectEqual(true, try resolveShell(
+ try testing.expectEqual(true, try shellSurvives(
\\ Some article text.
a longer list of linksand another
)); } @@ -380,12 +418,12 @@ test "RenderTree: resolveStrip judges an all-link page on its links" { test "RenderTree: resolveStrip ignores what other strip bits already drop" { // The script text is not content; without strip.js it would tip the // balance toward keeping the shell. - try testing.expectEqual(false, try resolveShell( + try testing.expectEqual(false, try shellSurvives( \\. +fn isParagraphLike(tag: Element.Tag, st: *const Stats) bool { + return switch (tag) { + .section, .h2, .h3, .h4, .h5, .h6, .p, .td, .pre => true, + .div => st.block_child == false, + else => false, + }; +} + +fn countedAs(tag: Element.Tag, st: *const Stats) Element.Tag { + return if (tag == .div and st.block_child == false) .p else tag; +} + +// readability's textish tags: SPAN, LI, TD and DIV_TO_P_ELEMS. +fn isTextish(tag: Element.Tag) bool { + return switch (tag) { + .span, .li, .td, .blockquote, .dl, .div, .img, .ol, .p, .pre, .table, .ul => true, + else => false, + }; +} + +// readability's adWords and loadingWords, matched whole. +const ad_words = [_][]const u8{ "ad", "advertising", "advertisement", "pub", "publicit\xC3\xA9", "werb", "werbung", "\xE5\xB9\xBF\xE5\x91\x8A", "\xD0\xA0\xD0\xB5\xD0\xBA\xD0\xBB\xD0\xB0\xD0\xBC\xD0\xB0", "anuncio", "loading", "\xE6\xAD\xA3\xE5\x9C\xA8\xE5\x8A\xA0\xE8\xBD\xBD", "\xD0\x97\xD0\xB0\xD0\xB3\xD1\x80\xD1\x83\xD0\xB7\xD0\xBA\xD0\xB0", "chargement", "cargando" }; + +fn tagScore(tag: Element.Tag) f32 { + return switch (tag) { + .div => 5, + .pre, .td, .blockquote => 3, + .address, .ol, .ul, .dl, .dd, .dt, .li, .form => -3, + .h1, .h2, .h3, .h4, .h5, .h6, .th => -5, + else => 0, + }; +} + +fn isDataTable(el: *Element, st: *const Stats) bool { + if (hasRole(el, &.{"presentation"})) return false; + if (el.getAttributeSafe(comptime .wrap("datatable"))) |v| { + if (std.mem.eql(u8, v, "0")) return false; + } + if (el.getAttributeSafe(comptime .wrap("summary")) != null) return true; + if (st.th > 0 or st.table_head) return true; + if (st.tables > 0) return false; + const cols = if (st.rows == 0) st.cells else st.cells / st.rows; + if (st.rows >= 10 or cols > 4) return true; + return st.rows * cols > 10; +} + +const TextMeasure = struct { len: usize, commas: usize }; + +/// Length as innerText would report it: whitespace runs collapse to one, +/// leading and trailing runs vanish. +fn measureText(text: []const u8) TextMeasure { + var len: usize = 0; + var commas: usize = 0; + var pending_space = false; + for (text, 0..) |c, i| { + if (std.ascii.isWhitespace(c)) { + pending_space = len > 0; + continue; + } + if (pending_space) { + len += 1; + pending_space = false; + } + len += 1; + if (c == ',') { + commas += 1; + } else if (c == 0xEF and i + 2 < text.len and text[i + 1] == 0xBC and text[i + 2] == 0x8C) { + commas += 1; // U+FF0C fullwidth comma + } else if (c == 0xE3 and i + 2 < text.len and text[i + 1] == 0x80 and text[i + 2] == 0x81) { + commas += 1; // U+3001 ideographic comma + } + } + return .{ .len = len, .commas = commas }; +} + +// readability's REGEXPS as substring lists, matched on the lowercased +// class and id. +const unlikely = [_][]const u8{ "-ad-", "ai2html", "banner", "breadcrumbs", "combx", "comment", "community", "cover-wrap", "disqus", "extra", "footer", "gdpr", "header", "legends", "menu", "related", "remark", "replies", "rss", "shoutbox", "sidebar", "skyscraper", "social", "sponsor", "supplemental", "ad-break", "agegate", "pagination", "pager", "popup", "yom-remote" }; +const maybe_candidate = [_][]const u8{ "and", "article", "body", "column", "content", "main", "shadow" }; +const positive = [_][]const u8{ "article", "body", "content", "entry", "hentry", "h-entry", "main", "page", "pagination", "post", "text", "blog", "story" }; +const negative = [_][]const u8{ "-ad-", "hidden", "banner", "combx", "comment", "com-", "contact", "footer", "gdpr", "masthead", "media", "meta", "outbrain", "promo", "related", "scroll", "share", "shoutbox", "sidebar", "skyscraper", "sponsor", "shopping", "tags", "widget" }; + +fn containsAny(haystack: []const u8, needles: []const []const u8) bool { + for (needles) |needle| { + if (std.mem.indexOf(u8, haystack, needle) != null) return true; + } + return false; +} + +fn isNegative(name: []const u8) bool { + return containsAny(name, &negative) or hasWord(name, "hid"); +} + +// readability: /(\b|_)(share|sharedaddy)(\b|_)/ +fn isShareElement(names: []const u8) bool { + return hasWord(names, "share") or hasWord(names, "sharedaddy"); +} + +fn hasWord(haystack: []const u8, word: []const u8) bool { + var start: usize = 0; + while (std.mem.indexOfPos(u8, haystack, start, word)) |i| : (start = i + 1) { + const before = i == 0 or !std.ascii.isAlphanumeric(haystack[i - 1]); + const end = i + word.len; + const after = end == haystack.len or !std.ascii.isAlphanumeric(haystack[end]); + if (before and after) return true; + } + return false; +} + +// ARIA `role` is a space-separated fallback list; the first token wins. +fn hasRole(el: *Element, roles: []const []const u8) bool { + const attr = el.getAttributeSafe(comptime .wrap("role")) orelse return false; + var it = std.mem.tokenizeAny(u8, attr, " \t\n\r"); + const role = it.next() orelse return false; + for (roles) |candidate| { + if (std.ascii.eqlIgnoreCase(role, candidate)) return true; + } + return false; +} + +const testing = @import("../testing.zig"); +const markdown = @import("markdown.zig"); + +const prose = "Sourdough is a bread made by the fermentation of dough using wild lactobacillaceae and yeast, which give it a mildly sour taste. The lactic acid produced by the bacteria gives it a longer shelf life than breads made with baker's yeast. "; + +test "clutter: keeps the article, drops the teaser list and share bar" { + const html = + "
" ++ + ""; + const out = try extract(html); + try testing.expectEqual(true, std.mem.indexOf(u8, out, "wild lactobacillaceae") != null); + try testing.expectEqual(true, std.mem.indexOf(u8, out, "# Sourdough") != null); + try testing.expectEqual(null, std.mem.indexOf(u8, out, "Trending now")); + try testing.expectEqual(null, std.mem.indexOf(u8, out, "Share on")); +} + +test "clutter: too little text falls back" { + const out = try extract("A short note.
" ++ prose ++ "
" ++ prose ++ "
" ++ prose ++ "