Merge pull request #3463 from lightpanda-io/adblock-regex-pcre2

`AdBlocker`: run /regex/ filters with PCRE2
This commit is contained in:
Adrià Arrufat authored and GitHub committed 2026-09-10 14:04:14 +02:00
commit d693f49872
8 files changed
+778 -65

No files matched your search

+22
View File
@@ -125,6 +125,7 @@ pub fn build(b: *Build) !void {
linkZenai(b, lightpanda_module);
linkIsocline(b, lightpanda_module);
linkSqlite(b, lightpanda_module, enable_csan, enable_tsan, orderfile != null);
linkPcre2(b, lightpanda_module, enable_csan, enable_tsan, orderfile != null);
// Check compilation
const check = b.step("check", "Check if lightpanda compiles");
@@ -486,6 +487,27 @@ fn linkSqlite(b: *Build, mod: *Build.Module, enable_csan: ?std.zig.SanitizeC, is
mod.addImport("sqlite3", translate_c.createModule());
}
fn linkPcre2(b: *Build, mod: *Build.Module, enable_csan: ?std.zig.SanitizeC, is_tsan: bool, section: bool) void {
const dep = b.dependency("pcre2", .{
.target = mod.resolved_target.?,
.optimize = mod.optimize.?,
.linkage = .static,
});
const lib = sectionize(dep.artifact("pcre2-8"), section);
lib.root_module.sanitize_c = enable_csan;
lib.root_module.sanitize_thread = is_tsan;
mod.linkLibrary(lib);
const translate_c = b.addTranslateC(.{
.root_source_file = lib.getEmittedIncludeTree().path(b, "pcre2.h"),
.target = mod.resolved_target.?,
.optimize = mod.optimize.?,
});
translate_c.defineCMacro("PCRE2_CODE_UNIT_WIDTH", "8");
mod.addImport("pcre2", translate_c.createModule());
}
fn linkCurl(b: *Build, mod: *Build.Module, is_tsan: bool, section: bool) void {
const target = mod.resolved_target.?;
+4
View File
@@ -43,6 +43,10 @@
.url = "git+https://github.com/arrufat/isocline#ec538faf435c616a6b38716f53980b5815c30f8a",
.hash = "N-V-__8AAHhtEwBIqx5nOoiGo_FLAG8gpiVC6XzZn1teMKd0",
},
.pcre2 = .{
.url = "https://github.com/PCRE2Project/pcre2/releases/download/pcre2-10.48/pcre2-10.48.tar.gz",
.hash = "pcre2-10.48.0-IZ6r69wnegD6fc08FgRTKVcmyp4Ux2aVefaVvsRndgEl",
},
},
.paths = .{""},
}
+15
View File
@@ -4390,12 +4390,27 @@ test "HttpClient: adblock verdicts apply per request" {
\\||typed.example.com^$script
\\||partied.example.com^$third-party
\\||framed.example.com^$subdocument
\\/\/[a-z]{4}\.js$/$match-case,script
);
try blocker.parse(&list);
try blocker.build();
client.network.adblocker = blocker;
defer client.network.adblocker = null;
// A regex filter reads the URL as requested: case kept, fragment gone.
try testing.expect(testIsUrlBlocked(&client, .{
.url = "https://cdn.example.com/abcd.js",
.resource_type = .script,
}));
try testing.expect(testIsUrlBlocked(&client, .{
.url = "https://cdn.example.com/abcd.js#v2",
.resource_type = .script,
}));
try testing.expect(!testIsUrlBlocked(&client, .{
.url = "https://cdn.example.com/ABCD.js",
.resource_type = .script,
}));
try testing.expect(testIsUrlBlocked(&client, .{ .url = "https://ads.example.com/pixel.gif" }));
// Hostnames are matched case-insensitively and without the port.
try testing.expect(testIsUrlBlocked(&client, .{ .url = "https://SUB.ADS.EXAMPLE.COM:8443/x" }));
+94 -19
View File
@@ -29,6 +29,7 @@ const Parser = @import("Parser.zig");
const Engine = @import("Engine.zig");
const HostnameTrie = @import("HostnameTrie.zig");
const NetworkFilter = @import("NetworkFilter.zig");
const Regex = @import("Regex.zig");
const log = lp.log;
@@ -47,6 +48,11 @@ arena: std.heap.ArenaAllocator,
filters: std.ArrayList(NetworkFilter),
/// Filled while parsing, consumed by `build`.
badfilters: std.AutoHashMapUnmanaged(u64, void),
/// Every regex compiled for a filter, whether or not `build` kept the
/// filter; they are freed here, not through `filters`.
regexes: std.ArrayList(*const Regex),
/// What the regexes are compiled and run with; PCRE2 allocates through it.
regex_context: *Regex.Context,
built: bool,
trie: HostnameTrie,
blocked: u32,
@@ -64,7 +70,7 @@ exceptions: Engine,
/// Rules that reached a trie or an index, across every list parsed so far.
rules_loaded: usize,
/// Rules the lists carry and we do not apply, across those same lists: the
/// ones the parser could not read, plus the ones no matcher can express.
/// ones the parser could not read, regex literals PCRE2 rejects included.
rules_skipped: usize,
/// Domain-scoped element-hiding rules and cosmetic-realm exceptions, across
/// those same lists. Not ours to apply, but not rules we failed at either, so
@@ -77,6 +83,8 @@ rules_cosmetic: usize,
const LINE_MAX = 8 * 1024;
pub fn init(allocator: Allocator) Allocator.Error!AdBlocker {
const regex_context: *Regex.Context = try .init(allocator);
errdefer regex_context.deinit();
var trie: HostnameTrie = try .init(allocator);
errdefer trie.deinit(allocator);
const blocked = try trie.createTrie(allocator);
@@ -89,6 +97,8 @@ pub fn init(allocator: Allocator) Allocator.Error!AdBlocker {
.arena = std.heap.ArenaAllocator.init(allocator),
.filters = .empty,
.badfilters = .empty,
.regexes = .empty,
.regex_context = regex_context,
.built = false,
.trie = trie,
.blocked = blocked,
@@ -109,6 +119,9 @@ pub fn deinit(self: *AdBlocker) void {
self.blocking_important.deinit(self.allocator);
self.exceptions.deinit(self.allocator);
self.badfilters.deinit(self.allocator);
for (self.regexes.items) |regex| regex.deinit();
self.regexes.deinit(self.allocator);
self.regex_context.deinit();
self.filters.deinit(self.allocator);
self.trie.deinit(self.allocator);
self.arena.deinit();
@@ -180,7 +193,7 @@ pub fn parse(self: *AdBlocker, reader: *Io.Reader) !void {
// both of which the next call reuses, so what we keep is copied out.
defer _ = scratch_instance.reset(.retain_capacity);
const filter = switch (item) {
var filter = switch (item) {
// The parser already counted it; all we want is whether it was an
// exception, in which case its hostname stops being blockable.
.dropped => |dropped| {
@@ -201,11 +214,6 @@ pub fn parse(self: *AdBlocker, reader: *Io.Reader) !void {
self.rules_cosmetic += 1;
continue;
}
if (!isSupported(&filter)) {
self.rules_skipped += 1;
continue;
}
if (filter.badfilter) {
// It cancels a rule that may not have been read yet, so it can
// only be resolved once every list is in. It is not a rule we
@@ -215,6 +223,22 @@ pub fn parse(self: *AdBlocker, reader: *Io.Reader) !void {
continue;
}
if (filter.kind == .regex) {
const body = filter.pattern[1 .. filter.pattern.len - 1];
const compiled = Regex.compile(self.regex_context, body, !filter.match_case) catch |err| switch (err) {
error.OutOfMemory => return error.OutOfMemory,
error.InvalidRegex => {
self.rules_skipped += 1;
continue;
},
};
errdefer compiled.deinit();
const regex = try arena.create(Regex);
regex.* = compiled;
try self.regexes.append(self.allocator, regex);
filter.regex = regex;
}
try self.filters.append(self.allocator, try dupe(arena, &filter));
self.rules_loaded += 1;
}
@@ -361,12 +385,6 @@ pub fn match(self: *const AdBlocker, request: *const Request) Verdict {
return .blocked;
}
/// Whether we can evaluate this filter at all.
fn isSupported(filter: *const NetworkFilter) bool {
// No regex engine to run them with.
return filter.kind != .regex;
}
/// Whether the filter's whole effect is "every request to this hostname and
/// its subdomains", which is all a trie can express.
fn isWholeHostname(filter: *const NetworkFilter) bool {
@@ -451,11 +469,8 @@ fn expectVerdict(
source: []const u8,
kind: ResourceTypes,
) !void {
// `match` takes a lowercased URL, as the real caller hands it; patterns
// are lowercased too, so a rule spelled "/embed/C-iDzdvIg1Y" still lands.
var buf: [512]u8 = undefined;
const lowered = std.ascii.lowerString(buf[0..url.len], url);
const request: Request = .init(lowered, source, kind);
const request: Request = Request.init(url, &buf, source, kind).?;
try testing.expectEqual(expected, blocker.match(&request));
}
@@ -591,7 +606,8 @@ test "adblock.AdBlocker: tokens past the request buffer still match" {
// 140 tokens of query noise push the filter's token ("utm", its rarest)
// past what the request holds; the engine walks the rest of the URL.
const noise = "https://example.com/?" ++ "a=1&" ** 70;
const overflowing: Request = .init(noise ++ "utm_tracker=1", "a.com", script);
var buf: [512]u8 = undefined;
const overflowing: Request = Request.init(noise ++ "utm_tracker=1", &buf, "a.com", script).?;
try testing.expect(overflowing.tail.len != 0);
try testing.expectEqual(.blocked, blocker.match(&overflowing));
// A token past the buffer finds its bucket, but the pattern does not fit.
@@ -608,7 +624,7 @@ test "adblock.AdBlocker: cosmetic-realm rules are not skipped rules" {
\\@@||example.com^$generichide
\\@@||example.com^$elemhide
\\example.com##.ad-banner
\\/regex-we-cannot-run/
\\/regex-we-cannot-run(/
);
try testing.expectEqual(1, blocker.rules_loaded);
@@ -616,6 +632,65 @@ test "adblock.AdBlocker: cosmetic-realm rules are not skipped rules" {
try testing.expectEqual(3, blocker.rules_cosmetic);
}
test "adblock.AdBlocker: the regex rules from EasyList" {
var blocker: AdBlocker = try .init(testing.allocator);
defer blocker.deinit();
try testLoad(&blocker,
\\/\/[0-9a-f]{32}\/invoke\.js/$script,third-party
\\/^https?:\/\/[0-9a-z]{5,}\.com\/.*/$script,third-party,xmlhttprequest,domain=dood.to
\\@@/\/invoke\.js/$domain=trusted.com
\\/\/[a-z]{4}\.js$/$match-case,script
);
try testing.expectEqual(4, blocker.rules_loaded);
try testing.expectEqual(0, blocker.rules_skipped);
// Each carries a literal every match has, so none rides the fallback.
try testing.expectEqual(0, blocker.blocking.fallback.len);
try testing.expectEqual(0, blocker.exceptions.fallback.len);
const invoke = "https://host.com/0123456789abcdef0123456789abcdef/invoke.js";
try expectVerdict(&blocker, .blocked, invoke, "site.com", script);
try expectVerdict(&blocker, .blocked, "https://HOST.com/0123456789ABCDEF0123456789abcdef/invoke.js", "site.com", script);
try expectVerdict(&blocker, .none, invoke, "host.com", script);
try expectVerdict(&blocker, .none, invoke, "site.com", image);
try expectVerdict(&blocker, .none, "https://host.com/0123456789abcdef0123456789abcde/invoke.js", "site.com", script);
try expectVerdict(&blocker, .allowed, invoke, "trusted.com", script);
try expectVerdict(&blocker, .blocked, "https://abcde.com/x", "dood.to", xhr);
try expectVerdict(&blocker, .none, "https://abcde.com/x", "other.to", xhr);
try expectVerdict(&blocker, .none, "https://abcd.com/x", "dood.to", xhr);
// `$match-case` reads the URL as requested, not the lowercased copy.
try expectVerdict(&blocker, .blocked, "https://x.com/abcd.js", "x.com", script);
try expectVerdict(&blocker, .none, "https://x.com/ABCD.js", "x.com", script);
}
test "adblock.AdBlocker: a regex is found under every token it may match" {
var blocker: AdBlocker = try .init(testing.allocator);
defer blocker.deinit();
try testLoad(&blocker,
\\/\/ads(\/?x|\/y)/$script
);
try expectVerdict(&blocker, .blocked, "https://example.com/adsx", "example.com", script);
try expectVerdict(&blocker, .blocked, "https://example.com/ads/x", "example.com", script);
try expectVerdict(&blocker, .blocked, "https://example.com/ads/y", "example.com", script);
try expectVerdict(&blocker, .none, "https://example.com/ads/z", "example.com", script);
}
test "adblock.AdBlocker: $badfilter removes a regex rule" {
var blocker: AdBlocker = try .init(testing.allocator);
defer blocker.deinit();
try testLoad(&blocker,
\\/\/invoke\.js/$script
\\/\/invoke\.js/$script,badfilter
);
try testing.expectEqual(0, blocker.rules_loaded);
try testing.expectEqual(2, blocker.rules_skipped);
try expectVerdict(&blocker, .none, "https://host.com/invoke.js", "site.com", script);
}
test "adblock.AdBlocker: verdict precedence" {
var blocker: AdBlocker = try .init(testing.allocator);
defer blocker.deinit();
+400 -35
View File
@@ -95,14 +95,18 @@ pub const Request = struct {
source: [SOURCE_MAX]u8,
};
/// `url` must already be lowercased and fragment-free; `source_hostname`
/// may be empty when there is no document context.
/// `buf` backs the lowercased URL; null when `url` is longer than it.
/// `source_hostname` may be empty when there is no document context.
pub fn init(
url: []const u8,
buf: []u8,
source_hostname: []const u8,
kind: NetworkFilter.ResourceTypes,
) Request {
const parsed: pattern.Url = .init(url);
) ?Request {
// No request URL carries a fragment onto the wire.
const raw = URL.stripFragment(url);
const lowered = lowercase(raw, buf) orelse return null;
const parsed: pattern.Url = .init(lowered, raw);
const source = if (source_hostname.len == 0) parsed.hostname() else source_hostname;
var request: Request = .{
.url = parsed,
@@ -113,19 +117,18 @@ pub const Request = struct {
.tokens_len = 0,
.tail = "",
};
var it: Tokens = .{ .text = url };
var it: Tokens = .{ .text = lowered };
while (request.tokens_len < request.tokens_buf.len) {
const token = it.next() orelse break;
request.tokens_buf[request.tokens_len] = token;
request.tokens_len += 1;
}
request.tail = url[it.i..];
request.tail = lowered[it.i..];
return request;
}
pub fn fromHttp(transfer: *const HttpClient.Transfer, buffers: *Buffers) ?Request {
const req = &transfer.req;
const url = normalizeUrl(req.url, &buffers.url) orelse return null;
var owner: ?*const HttpClient.Owner = transfer.owner;
var subframe = false;
@@ -148,27 +151,23 @@ pub const Request = struct {
.worker => .{ .script = true },
};
return .init(url, source, resource_type);
return .init(req.url, &buffers.url, source, resource_type);
}
inline fn tokens(self: *const Request) []const u32 {
return self.tokens_buf[0..self.tokens_len];
}
/// Lowercases `url` into `buf` with its fragment stripped; patterns are
/// stored lowercased, and no request URL carries a fragment onto the wire.
fn normalizeUrl(url: []const u8, buf: []u8) ?[]const u8 {
const end = std.mem.indexOfScalar(u8, url, '#') orelse url.len;
const trimmed = url[0..end];
const upper = for (trimmed, 0..) |c, i| {
/// Lowercases `url` into `buf`, as patterns are stored lowercased.
fn lowercase(url: []const u8, buf: []u8) ?[]const u8 {
const upper = for (url, 0..) |c, i| {
if (std.ascii.isUpper(c)) break i;
} else return trimmed;
} else return url;
if (trimmed.len > buf.len) return null;
const out = buf[0..trimmed.len];
@memcpy(out[0..upper], trimmed[0..upper]);
_ = std.ascii.lowerString(out[upper..], trimmed[upper..]);
if (url.len > buf.len) return null;
const out = buf[0..url.len];
@memcpy(out[0..upper], url[0..upper]);
_ = std.ascii.lowerString(out[upper..], url[upper..]);
return out;
}
};
@@ -326,11 +325,20 @@ fn collectTokens(filter: *const NetworkFilter, buf: []u32) []u32 {
}
}
// A /regex/ literal is never indexed (nothing can run it), and `.any`
// has no pattern at all.
if (filter.kind == .regex or filter.pattern.len == 0) return buf[0..n];
// A /regex/ literal has no hostname, so `n` is still 0 here.
if (filter.kind == .regex) return regexTokens(filter.pattern[1 .. filter.pattern.len - 1], buf);
// `.any` has no pattern at all.
if (filter.pattern.len == 0) return buf[0..n];
const text = filter.pattern;
const left_anchored = filter.left_anchor or filter.hostname_anchor;
return buf[0..boundedTokens(filter.pattern, left_anchored, filter.right_anchor, buf, n)];
}
/// Appends the tokens of `text` that any URL matching it carries whole: the
/// alphanumeric runs no `*` may extend, and that the open ends of the text
/// may not extend either unless anchored. Returns the new count.
fn boundedTokens(text: []const u8, left_anchored: bool, right_anchored: bool, buf: []u32, from: usize) usize {
var n = from;
var i: usize = 0;
while (i < text.len) {
if (!isTokenChar(text[i])) {
@@ -340,22 +348,317 @@ fn collectTokens(filter: *const NetworkFilter, buf: []u32) []u32 {
const start = i;
while (i < text.len and isTokenChar(text[i])) i += 1;
const left_bounded = if (start == 0)
filter.left_anchor or filter.hostname_anchor
else
text[start - 1] != '*';
const right_bounded = if (i == text.len) filter.right_anchor else text[i] != '*';
const left_bounded = if (start == 0) left_anchored else text[start - 1] != '*';
const right_bounded = if (i == text.len) right_anchored else text[i] != '*';
if (left_bounded and right_bounded) {
if (n == buf.len) return buf[0..n];
if (n == buf.len) return n;
buf[n] = hash(text[start..i]);
n += 1;
}
}
return buf[0..n];
return n;
}
/// The tokens a regex is sure to have wherever it matches, uBO's
/// `tokenizableStrFromRegex`: the pattern is flattened into a plain pattern
/// where literal token characters stay and everything else becomes a marker
/// saying only whether it could be one. Anything the flattening does not
/// follow yields no token at all, which is never wrong: the filter then
/// rides the fallback bucket.
fn regexTokens(source: []const u8, buf: []u32) []u32 {
// A pattern's shape is never longer than the pattern.
var shape_buf: [8 * 1024]u8 = undefined;
if (source.len > shape_buf.len) return buf[0..0];
var shape: RegexShape = .{ .source = source, .out = &shape_buf };
const text = shape.flatten() catch return buf[0..0];
// Anchors are in the shape itself; its ends are open.
return buf[0..boundedTokens(text, false, false, buf, 0)];
}
const RegexShape = struct {
source: []const u8,
out: []u8,
i: usize = 0,
n: usize = 0,
/// Whatever matches here is not a token character: a non-token literal,
/// an anchor, `\b`.
const not_token = 0x00;
/// Whatever matches here may be a token character: `.`, `[a-z]`, `\d`, a
/// quantified literal. Spelled as a plain pattern's wildcard so the shape
/// reads as one.
const maybe_token = '*';
// The same two for a stretch that may match nothing at all; resolved
// once both neighbours are known.
const not_token_optional = 0x02;
const maybe_token_optional = 0x03;
const Error = error{Unsupported};
fn flatten(self: *RegexShape) Error![]const u8 {
try self.alternation(false);
if (self.i != self.source.len) return error.Unsupported;
self.resolveOptional();
return self.out[0..self.n];
}
/// Parses alternatives up to the closing parenthesis (or the end). More
/// than one collapses to two markers: all that is sure about `a|bc` is
/// how it starts and ends.
fn alternation(self: *RegexShape, nested: bool) Error!void {
const start = self.n;
var branches: usize = 1;
var branch_start = start;
var first = false;
var last = false;
while (true) {
try self.sequence(nested);
const branch = self.out[branch_start..self.n];
first = first or startsTokenish(branch);
last = last or endsTokenish(branch);
if (self.i == self.source.len or self.source[self.i] != '|') break;
self.i += 1;
branches += 1;
branch_start = self.n;
}
if (branches == 1) return;
self.n = start;
self.emit(if (first) maybe_token else not_token);
self.emit(if (last) maybe_token else not_token);
}
fn sequence(self: *RegexShape, nested: bool) Error!void {
while (self.i < self.source.len) {
const atom_start = self.n;
const c = self.source[self.i];
switch (c) {
'|' => return,
')' => {
if (!nested) return error.Unsupported;
return;
},
'(' => try self.group(),
'[' => try self.class(),
'\\' => try self.escape(),
'.' => {
self.i += 1;
self.emit(maybe_token);
},
'^', '$' => {
self.i += 1;
self.emit(not_token);
},
'*', '+', '?' => return error.Unsupported,
else => {
self.i += 1;
self.emitLiteral(c);
},
}
try self.quantifier(atom_start);
}
}
fn group(self: *RegexShape) Error!void {
const start = self.n;
self.i += 1;
var lookaround: enum { none, positive, negative } = .none;
if (self.i < self.source.len and self.source[self.i] == '?') {
self.i += 1;
const kind = self.take() orelse return error.Unsupported;
switch (kind) {
':' => {},
'=' => lookaround = .positive,
'!' => lookaround = .negative,
'<' => {
const next = self.take() orelse return error.Unsupported;
switch (next) {
'=' => lookaround = .positive,
'!' => lookaround = .negative,
else => {
// A named group is a plain group with a label.
const close = std.mem.indexOfScalarPos(u8, self.source, self.i, '>') orelse return error.Unsupported;
self.i = close + 1;
},
}
},
else => return error.Unsupported,
}
}
try self.alternation(true);
if (self.take() != ')') return error.Unsupported;
switch (lookaround) {
.none => {},
// Consumes nothing, so the neighbours touch; what it asserts
// could still be anything, and that is all a token may rely on.
.positive => {
self.n = start;
self.emit(maybe_token);
},
// Consumes nothing and rules text out: the neighbours touch.
.negative => self.n = start,
}
}
/// `[...]` is one character; a token character can come out of it if any
/// member is one, or if it is negated.
fn class(self: *RegexShape) Error!void {
self.i += 1;
var maybe = false;
if (self.i < self.source.len and self.source[self.i] == '^') {
self.i += 1;
maybe = true;
}
var first = true;
while (true) {
const c = self.take() orelse return error.Unsupported;
if (c == ']' and !first) break;
first = false;
if (c == '\\') {
const e = self.take() orelse return error.Unsupported;
switch (e) {
'd', 'D', 'w', 'W', 's', 'S' => maybe = true,
'b', 'n', 'r', 't', 'f', 'v' => {},
else => if (std.ascii.isAlphanumeric(e)) return error.Unsupported,
}
continue;
}
// A range is assumed to reach token characters.
if (c == '-' or std.ascii.isAlphanumeric(c)) maybe = true;
}
self.emit(if (maybe) maybe_token else not_token);
}
fn escape(self: *RegexShape) Error!void {
self.i += 1;
const e = self.take() orelse return error.Unsupported;
switch (e) {
'd', 'D', 'w', 'W', 's', 'S', 'B' => self.emit(maybe_token),
'b', 'n', 'r', 't', 'f', 'v' => self.emit(not_token),
// Code points, backreferences, properties: not worth following.
'x', 'u', 'c', 'k', 'p', 'P', '0'...'9' => return error.Unsupported,
// Anything else escaped is itself, as JavaScript reads it.
else => self.emitLiteral(e),
}
}
/// Applies a quantifier, if one follows, to what was just emitted. Only
/// the first and last character classes survive a repeat: `ab+` may match
/// "abbb", and its token is not "ab". Whether the atom may be absent is
/// all that matters beyond that.
fn quantifier(self: *RegexShape, atom_start: usize) Error!void {
if (self.i == self.source.len) return;
var optional = true;
switch (self.source[self.i]) {
'*', '?' => self.i += 1,
'+' => {
self.i += 1;
optional = false;
},
'{' => {
const close = std.mem.indexOfScalarPos(u8, self.source, self.i, '}') orelse return;
const body = self.source[self.i + 1 .. close];
const comma = std.mem.indexOfScalar(u8, body, ',') orelse body.len;
// Not a quantifier at all: JavaScript reads the `{` literally.
const min = std.fmt.parseUnsigned(usize, body[0..comma], 10) catch return;
if (comma + 1 < body.len) _ = std.fmt.parseUnsigned(usize, body[comma + 1 ..], 10) catch return;
optional = min == 0;
self.i = close + 1;
},
else => return,
}
// A lazy `?` changes nothing about what can match.
if (self.i < self.source.len and self.source[self.i] == '?') self.i += 1;
const atom = self.out[atom_start..self.n];
const first = startsTokenish(atom);
const last = endsTokenish(atom);
self.n = atom_start;
if (!optional) {
self.emit(if (first) maybe_token else not_token);
self.emit(if (last) maybe_token else not_token);
} else {
self.emit(if (first) maybe_token_optional else not_token_optional);
self.emit(if (last) maybe_token_optional else not_token_optional);
}
}
/// An optional stretch may vanish, letting its neighbours touch: it
/// resolves to markers that carry whichever side could be a token
/// character, so no run is read as bounded by something that may be
/// gone.
fn resolveOptional(self: *RegexShape) void {
var i: usize = 0;
while (i < self.n) {
if (!isOptional(self.out[i])) {
i += 1;
continue;
}
var end = i;
while (end < self.n and isOptional(self.out[end])) end += 1;
const left = self.out[0..i];
const middle = self.out[i..end];
const right = self.out[end..self.n];
const head: u8 = if (startsTokenish(right) or startsTokenish(middle)) maybe_token else not_token;
const tail: u8 = if (endsTokenish(left) or endsTokenish(middle)) maybe_token else not_token;
// Quantifiers emit markers in pairs, so a stretch is never shorter
// than what replaces it.
std.mem.copyForwards(u8, self.out[i + 2 .. self.n - (middle.len - 2)], right);
self.out[i] = head;
self.out[i + 1] = tail;
self.n -= middle.len - 2;
i += 2;
}
}
fn take(self: *RegexShape) ?u8 {
if (self.i == self.source.len) return null;
defer self.i += 1;
return self.source[self.i];
}
fn emit(self: *RegexShape, c: u8) void {
self.out[self.n] = c;
self.n += 1;
}
/// A literal is kept only as far as tokens care: a token character,
/// lowercased like the URL it is looked up in, or the fact that it is not
/// one.
fn emitLiteral(self: *RegexShape, c: u8) void {
self.emit(if (std.ascii.isAlphanumeric(c)) std.ascii.toLower(c) else not_token);
}
fn isOptional(c: u8) bool {
return c == not_token_optional or c == maybe_token_optional;
}
fn isTokenish(c: u8) bool {
return c == maybe_token or c == maybe_token_optional or isTokenChar(c);
}
/// Whether what `s` matches could start with a token character. An
/// optional non-token stretch may be absent, so whatever follows it
/// answers instead.
fn startsTokenish(s: []const u8) bool {
for (s) |c| {
if (isTokenish(c)) return true;
if (c != not_token_optional) return false;
}
return false;
}
fn endsTokenish(s: []const u8) bool {
var i = s.len;
while (i > 0) {
i -= 1;
if (isTokenish(s[i])) return true;
if (s[i] != not_token_optional) return false;
}
return false;
}
};
const testing = @import("../../testing.zig");
fn tokensOf(arena: Allocator, line: []const u8, buf: []u32) ![]u32 {
@@ -370,18 +673,80 @@ fn contains(tokens: []const u32, token: []const u8) bool {
return false;
}
test "adblock.Engine: regex filters yield the tokens every match carries" {
var arena_state = std.heap.ArenaAllocator.init(testing.allocator);
defer arena_state.deinit();
const arena = arena_state.allocator();
var buf: [TOKENS_MAX]u32 = undefined;
// Literal runs between literal non-token characters; the open end of an
// unanchored pattern bounds nothing.
var tokens = try tokensOf(arena, "/\\/[0-9a-f]{32}\\/invoke\\.js/", &buf);
try testing.expectEqual(1, tokens.len);
try testing.expect(contains(tokens, "invoke"));
// Anchors bound; `https?` may be either, so "http" is no token.
tokens = try tokensOf(arena, "/^https?:\\/\\/[0-9a-z]{5,}\\.com\\/.*/", &buf);
try testing.expectEqual(1, tokens.len);
try testing.expect(contains(tokens, "com"));
tokens = try tokensOf(arena, "/[a-z]{2,}\\.gif$/", &buf);
try testing.expectEqual(1, tokens.len);
try testing.expect(contains(tokens, "gif"));
tokens = try tokensOf(arena, "/about:blank.*/", &buf);
try testing.expectEqual(0, tokens.len);
// `\b` bounds, as uBO reads it: `/\bads\b/` is "ads", not "bads".
tokens = try tokensOf(arena, "/\\bads\\b/", &buf);
try testing.expect(contains(tokens, "ads"));
// Hashed lowercased, like the URL it is looked up in.
tokens = try tokensOf(arena, "/\\/Ads\\//$match-case", &buf);
try testing.expect(contains(tokens, "ads"));
// An optional stretch may vanish and glue its neighbours: "adsbanner".
tokens = try tokensOf(arena, "/\\/ads\\/?banner\\//", &buf);
try testing.expectEqual(0, tokens.len);
// ... also from inside a group: "adsx" is a match.
tokens = try tokensOf(arena, "/\\/ads(\\/?x|\\/y)/", &buf);
try testing.expectEqual(0, tokens.len);
// A repeat is not the literal it repeats.
tokens = try tokensOf(arena, "/\\/ab+c\\//", &buf);
try testing.expectEqual(0, tokens.len);
// Alternation and the text under a quantified group are uncertain.
tokens = try tokensOf(arena, "/^https?:\\/\\/(35|104)\\.(\\d){1,3}\\//", &buf);
try testing.expectEqual(0, tokens.len);
// ... but a group with one branch is transparent.
tokens = try tokensOf(arena, "/\\/(?:ads)\\//", &buf);
try testing.expect(contains(tokens, "ads"));
// A negative lookaround consumes nothing; a positive one is not
// trusted to spell anything.
tokens = try tokensOf(arena, "/\\/(?!ads)banner\\//", &buf);
try testing.expect(contains(tokens, "banner"));
tokens = try tokensOf(arena, "/\\/(?=ads)ads\\//", &buf);
try testing.expectEqual(0, tokens.len);
// What is not followed yields nothing rather than something wrong.
tokens = try tokensOf(arena, "/\\/\\x41ds\\//", &buf);
try testing.expectEqual(0, tokens.len);
tokens = try tokensOf(arena, "/\\/(ads\\//", &buf);
try testing.expectEqual(0, tokens.len);
}
test "adblock.Engine: a request keeps its first tokens, the rest as tail" {
const max = Request.URL_TOKENS_MAX;
const kind: NetworkFilter.ResourceTypes = .{ .script = true };
// Exactly as many tokens as the buffer holds: nothing is left to walk...
const full = "x/" ** (max - 1) ++ "x";
var request: Request = .init(full, "", kind);
var buf: [512]u8 = undefined;
var request: Request = Request.init(full, &buf, "", kind).?;
try testing.expectEqual(max, request.tokens_len);
try testing.expectEqual(0, request.tail.len);
// ...one more, and only that one is in the tail.
request = .init(full ++ "/y", "", kind);
request = Request.init(full ++ "/y", &buf, "", kind).?;
try testing.expectEqual(max, request.tokens_len);
try testing.expectString("/y", request.tail);
var it: Tokens = .{ .text = request.tail };
@@ -389,7 +754,7 @@ test "adblock.Engine: a request keeps its first tokens, the rest as tail" {
try testing.expect(it.next() == null);
// Fewer than the buffer holds: the tail is empty.
request = .init("https://example.com/a", "", kind);
request = Request.init("https://example.com/a", &buf, "", kind).?;
try testing.expectEqual(4, request.tokens_len);
try testing.expectEqual(0, request.tail.len);
}
+4
View File
@@ -18,6 +18,7 @@
const std = @import("std");
const domain = @import("domain.zig");
const Regex = @import("Regex.zig");
const NetworkFilter = @This();
@@ -38,6 +39,9 @@ exception: bool = false,
important: bool = false,
badfilter: bool = false,
match_case: bool = false,
/// The compiled `.regex` pattern, set by the blocker once it has read the
/// literal; the parser never runs one.
regex: ?*const Regex = null,
first_party: bool = true,
third_party: bool = true,
hostname_anchor: bool = false,
+224
View File
@@ -0,0 +1,224 @@
// Copyright (C) 2023-2026 Lightpanda (Selecy SAS)
//
// Francis Bouvier <francis@lightpanda.io>
// Pierre Tachoire <pierre@lightpanda.io>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
//! A compiled `/regex/` filter body. Filter lists write them in JavaScript
//! `RegExp` syntax and uBO runs them with `new RegExp(src, 'i')` against the
//! raw request URL (no flag under `$match-case`); PCRE2 reads that syntax
//! as-is, escapes like `\/` included.
//!
//! A compiled pattern and its `Context` are never modified after `compile`,
//! so one `Regex` can be shared by every HTTP client thread; the per-call
//! match data is what PCRE2 requires to be private.
const std = @import("std");
const lp = @import("lightpanda");
const pcre2 = @import("pcre2");
const Allocator = std.mem.Allocator;
const log = lp.log;
const Regex = @This();
code: *pcre2.pcre2_code_8,
context: *const Context,
pub const Error = error{ InvalidRegex, OutOfMemory };
/// What every `Regex` compiled through it shares: the allocator PCRE2 draws
/// from, and the compile and match settings. Outlives the regexes.
///
/// PCRE2 would happily use libc's malloc; it is handed the blocker's
/// allocator so that a compiled pattern nobody freed fails a test the way
/// any other leak does.
pub const Context = struct {
allocator: Allocator,
general: *pcre2.pcre2_general_context_8,
compile_context: *pcre2.pcre2_compile_context_8,
match_context: *pcre2.pcre2_match_context_8,
// A pattern from a list that backtracks this much on one URL is broken,
// not slow; giving up costs a false negative on that request, nothing
// more.
const MATCH_LIMIT = 100_000;
const DEPTH_LIMIT = 10_000;
pub fn init(allocator: Allocator) Allocator.Error!*Context {
const self = try allocator.create(Context);
errdefer allocator.destroy(self);
self.allocator = allocator;
// PCRE2 hands the allocator back to the callbacks by address, so the
// context has to be at its final one before anything is allocated
// through it.
const general = pcre2.pcre2_general_context_create_8(cMalloc, cFree, &self.allocator) orelse return error.OutOfMemory;
errdefer pcre2.pcre2_general_context_free_8(general);
const compile_context = pcre2.pcre2_compile_context_create_8(general) orelse return error.OutOfMemory;
errdefer pcre2.pcre2_compile_context_free_8(compile_context);
// JavaScript without the `u` flag reads an unknown escape as the
// literal character, and that is the mode uBO compiles filters in.
_ = pcre2.pcre2_set_compile_extra_options_8(compile_context, pcre2.PCRE2_EXTRA_BAD_ESCAPE_IS_LITERAL);
const match_context = pcre2.pcre2_match_context_create_8(general) orelse return error.OutOfMemory;
_ = pcre2.pcre2_set_match_limit_8(match_context, MATCH_LIMIT);
_ = pcre2.pcre2_set_depth_limit_8(match_context, DEPTH_LIMIT);
self.general = general;
self.compile_context = compile_context;
self.match_context = match_context;
return self;
}
pub fn deinit(self: *Context) void {
pcre2.pcre2_match_context_free_8(self.match_context);
pcre2.pcre2_compile_context_free_8(self.compile_context);
pcre2.pcre2_general_context_free_8(self.general);
self.allocator.destroy(self);
}
// PCRE2 frees without a size, so every block carries its own in a
// header that keeps the payload at malloc's alignment.
const HEADER = 16;
const alignment: std.mem.Alignment = .fromByteUnits(HEADER);
fn cMalloc(size: usize, data: ?*anyopaque) callconv(.c) ?*anyopaque {
const allocator: *const Allocator = @ptrCast(@alignCast(data.?));
const total = std.math.add(usize, size, HEADER) catch return null;
const block = allocator.alignedAlloc(u8, alignment, total) catch return null;
std.mem.writeInt(usize, block[0..@sizeOf(usize)], total, .little);
return block.ptr + HEADER;
}
fn cFree(ptr: ?*anyopaque, data: ?*anyopaque) callconv(.c) void {
const payload = ptr orelse return;
const allocator: *const Allocator = @ptrCast(@alignCast(data.?));
const base: [*]align(HEADER) u8 = @ptrCast(@alignCast(@as([*]u8, @ptrCast(payload)) - HEADER));
const total = std.mem.readInt(usize, base[0..@sizeOf(usize)], .little);
allocator.free(base[0..total]);
}
};
pub fn compile(context: *const Context, pattern: []const u8, case_insensitive: bool) Error!Regex {
const options: u32 = if (case_insensitive) pcre2.PCRE2_CASELESS else 0;
var err_code: c_int = 0;
var err_offset: usize = 0;
const code = pcre2.pcre2_compile_8(
pattern.ptr,
pattern.len,
options,
&err_code,
&err_offset,
context.compile_context,
) orelse {
// A failed allocation is ours, not the pattern's.
if (err_code == pcre2.PCRE2_ERROR_HEAP_FAILED) return error.OutOfMemory;
var buf: [256]u8 = undefined;
const len = pcre2.pcre2_get_error_message_8(err_code, &buf, buf.len);
const message: []const u8 = if (len < 0) "unknown error" else buf[0..@intCast(len)];
log.debug(.app, "adblock regex rejected", .{
.pattern = pattern,
.err = message,
.offset = err_offset,
});
return error.InvalidRegex;
};
return .{ .code = code, .context = context };
}
pub fn deinit(self: Regex) void {
pcre2.pcre2_code_free_8(self.code);
}
// What one match allocates: its match data and the 20KB of backtracking
// frames PCRE2 starts with, which only a deeply nested pattern outgrows.
const MATCH_SCRATCH = 24 * 1024;
/// Whether the pattern matches anywhere in `text`, as `RegExp.test` would
/// answer. A match that hits the backtracking limits counts as no match.
pub fn matches(self: Regex, text: []const u8) bool {
// This runs per request; the scratch keeps the common case off the heap.
var scratch = std.heap.stackFallback(MATCH_SCRATCH, self.context.allocator);
var allocator = scratch.get();
const general = pcre2.pcre2_general_context_create_8(Context.cMalloc, Context.cFree, &allocator) orelse return false;
defer pcre2.pcre2_general_context_free_8(general);
// One pair is the whole-match span, all a test needs; capture groups in
// the pattern are simply not recorded.
const match_data = pcre2.pcre2_match_data_create_8(1, general) orelse return false;
defer pcre2.pcre2_match_data_free_8(match_data);
const rc = pcre2.pcre2_match_8(self.code, text.ptr, text.len, 0, 0, match_data, self.context.match_context);
return rc >= 0;
}
const testing = @import("../../testing.zig");
test "adblock.Regex: JavaScript escapes and unanchored search" {
const context: *Context = try .init(testing.allocator);
defer context.deinit();
const regex = try Regex.compile(context, "^https?:\\/\\/[0-9a-z]{5,}\\.com\\/.*", true);
defer regex.deinit();
try testing.expect(regex.matches("https://abcde.com/x"));
try testing.expect(regex.matches("HTTPS://ABCDE.COM/X"));
try testing.expect(!regex.matches("https://abcd.com/x"));
try testing.expect(!regex.matches("https://abcde.org/x"));
const invoke = try Regex.compile(context, "\\/[0-9a-f]{32}\\/invoke\\.js", true);
defer invoke.deinit();
try testing.expect(invoke.matches("https://host.com/0123456789abcdef0123456789abcdef/invoke.js"));
try testing.expect(!invoke.matches("https://host.com/0123456789abcdef0123456789abcde/invoke.js"));
const dash = try Regex.compile(context, "[a-z\\-]+\\?s=", true);
defer dash.deinit();
try testing.expect(dash.matches("https://x.com/a-b?s=1"));
try testing.expect(!dash.matches("https://x.com/?s=1"));
}
test "adblock.Regex: $match-case keeps the case" {
const context: *Context = try .init(testing.allocator);
defer context.deinit();
const exact = try Regex.compile(context, "\\/[a-z0-9]{12}\\/[a-zA-Z0-9]{20,}$", false);
defer exact.deinit();
try testing.expect(exact.matches("https://x.com/abcdef123456/aBcDeFgHiJkLmNoPqRsTuV"));
try testing.expect(!exact.matches("https://x.com/ABCDEF123456/aBcDeFgHiJkLmNoPqRsTuV"));
}
test "adblock.Regex: invalid patterns are errors, runaway ones no match" {
const context: *Context = try .init(testing.allocator);
defer context.deinit();
try testing.expectError(error.InvalidRegex, Regex.compile(context, "(", true));
try testing.expectError(error.InvalidRegex, Regex.compile(context, "a{2,1}", true));
// An unknown alphanumeric escape is the literal, as in JavaScript.
const literal = try Regex.compile(context, "\\q", true);
defer literal.deinit();
try testing.expect(literal.matches("https://x.com/q"));
// Exponential backtracking stops at the match limit instead of stalling
// the request.
const runaway = try Regex.compile(context, "^(a+)+$", true);
defer runaway.deinit();
const subject = "a" ** 64 ++ "b";
try testing.expect(!runaway.matches(subject));
try testing.expect(runaway.matches("a" ** 64));
}
+15 -11
View File
@@ -28,8 +28,11 @@ const NetworkFilter = @import("NetworkFilter.zig");
/// The URL a request is matched against, with its hostname located once so
/// every filter can reuse the offsets.
pub const Url = struct {
/// Lowercased, fragment stripped.
/// Lowercased, fragment stripped: what patterns, stored lowercased, walk.
text: []const u8,
/// Fragment stripped, case kept: what a regex reads, since `$match-case`
/// is only meaningful on the original.
raw: []const u8,
host_start: u32,
host_end: u32,
@@ -40,7 +43,7 @@ pub const Url = struct {
/// Locates the hostname inside `url`: after "scheme://", up to the port,
/// path, query or end. Offsets, not a slice, because the matcher needs to
/// resume the pattern right where the hostname stops.
pub fn init(url: []const u8) Url {
pub fn init(url: []const u8, raw: []const u8) Url {
var start: usize = 0;
if (std.mem.indexOf(u8, url, "://")) |scheme| start = scheme + 3;
@@ -74,6 +77,7 @@ pub const Url = struct {
return .{
.text = url,
.raw = raw,
.host_start = @intCast(start),
.host_end = @intCast(host_end),
};
@@ -86,8 +90,8 @@ pub fn matches(filter: *const NetworkFilter, url: Url) bool {
switch (filter.kind) {
// Option-only filters ("$script,domain=x") match any URL.
.any => return true,
// Never indexed: there is no regex engine to run them with.
.regex => return false,
// uBO tests the raw URL; the case-insensitive flag is on the pattern.
.regex => return filter.regex.?.matches(url.raw),
.hostname, .plain, .wildcard => {},
}
@@ -235,28 +239,28 @@ const testing = @import("../../testing.zig");
fn testMatch(arena: std.mem.Allocator, line: []const u8, url: []const u8) !bool {
const filter = try NetworkFilter.parse(arena, line);
return matches(&filter, .init(url));
return matches(&filter, .init(url, url));
}
test "adblock.pattern: hostname location" {
var u: Url = .init("https://ads.example.com/x?y=1");
var u: Url = .init("https://ads.example.com/x?y=1", "");
try testing.expectString("ads.example.com", u.hostname());
u = .init("https://ads.example.com:8443/x");
u = .init("https://ads.example.com:8443/x", "");
try testing.expectString("ads.example.com", u.hostname());
u = .init("https://example.com");
u = .init("https://example.com", "");
try testing.expectString("example.com", u.hostname());
u = .init("http://[::1]:9222/json");
u = .init("http://[::1]:9222/json", "");
try testing.expectString("[::1]", u.hostname());
// Credentials are not part of the host.
u = .init("https://user:pass@ads.example.com/x");
u = .init("https://user:pass@ads.example.com/x", "");
try testing.expectString("ads.example.com", u.hostname());
// A pattern is matched against whatever it is given, even a bare path.
u = .init("/relative/path");
u = .init("/relative/path", "");
try testing.expectString("", u.hostname());
}