mirror of
https://github.com/lightpanda-io/browser.git
synced 2026-09-21 03:55:22 -04:00
Merge pull request #3463 from lightpanda-io/adblock-regex-pcre2
`AdBlocker`: run /regex/ filters with PCRE2
This commit is contained in:
8 files changed
+778
-65
No files matched your search
@@ -125,6 +125,7 @@ pub fn build(b: *Build) !void {
|
||||
linkZenai(b, lightpanda_module);
|
||||
linkIsocline(b, lightpanda_module);
|
||||
linkSqlite(b, lightpanda_module, enable_csan, enable_tsan, orderfile != null);
|
||||
linkPcre2(b, lightpanda_module, enable_csan, enable_tsan, orderfile != null);
|
||||
|
||||
// Check compilation
|
||||
const check = b.step("check", "Check if lightpanda compiles");
|
||||
@@ -486,6 +487,27 @@ fn linkSqlite(b: *Build, mod: *Build.Module, enable_csan: ?std.zig.SanitizeC, is
|
||||
mod.addImport("sqlite3", translate_c.createModule());
|
||||
}
|
||||
|
||||
fn linkPcre2(b: *Build, mod: *Build.Module, enable_csan: ?std.zig.SanitizeC, is_tsan: bool, section: bool) void {
|
||||
const dep = b.dependency("pcre2", .{
|
||||
.target = mod.resolved_target.?,
|
||||
.optimize = mod.optimize.?,
|
||||
.linkage = .static,
|
||||
});
|
||||
|
||||
const lib = sectionize(dep.artifact("pcre2-8"), section);
|
||||
lib.root_module.sanitize_c = enable_csan;
|
||||
lib.root_module.sanitize_thread = is_tsan;
|
||||
mod.linkLibrary(lib);
|
||||
|
||||
const translate_c = b.addTranslateC(.{
|
||||
.root_source_file = lib.getEmittedIncludeTree().path(b, "pcre2.h"),
|
||||
.target = mod.resolved_target.?,
|
||||
.optimize = mod.optimize.?,
|
||||
});
|
||||
translate_c.defineCMacro("PCRE2_CODE_UNIT_WIDTH", "8");
|
||||
mod.addImport("pcre2", translate_c.createModule());
|
||||
}
|
||||
|
||||
fn linkCurl(b: *Build, mod: *Build.Module, is_tsan: bool, section: bool) void {
|
||||
const target = mod.resolved_target.?;
|
||||
|
||||
|
||||
@@ -43,6 +43,10 @@
|
||||
.url = "git+https://github.com/arrufat/isocline#ec538faf435c616a6b38716f53980b5815c30f8a",
|
||||
.hash = "N-V-__8AAHhtEwBIqx5nOoiGo_FLAG8gpiVC6XzZn1teMKd0",
|
||||
},
|
||||
.pcre2 = .{
|
||||
.url = "https://github.com/PCRE2Project/pcre2/releases/download/pcre2-10.48/pcre2-10.48.tar.gz",
|
||||
.hash = "pcre2-10.48.0-IZ6r69wnegD6fc08FgRTKVcmyp4Ux2aVefaVvsRndgEl",
|
||||
},
|
||||
},
|
||||
.paths = .{""},
|
||||
}
|
||||
@@ -4390,12 +4390,27 @@ test "HttpClient: adblock verdicts apply per request" {
|
||||
\\||typed.example.com^$script
|
||||
\\||partied.example.com^$third-party
|
||||
\\||framed.example.com^$subdocument
|
||||
\\/\/[a-z]{4}\.js$/$match-case,script
|
||||
);
|
||||
try blocker.parse(&list);
|
||||
try blocker.build();
|
||||
client.network.adblocker = blocker;
|
||||
defer client.network.adblocker = null;
|
||||
|
||||
// A regex filter reads the URL as requested: case kept, fragment gone.
|
||||
try testing.expect(testIsUrlBlocked(&client, .{
|
||||
.url = "https://cdn.example.com/abcd.js",
|
||||
.resource_type = .script,
|
||||
}));
|
||||
try testing.expect(testIsUrlBlocked(&client, .{
|
||||
.url = "https://cdn.example.com/abcd.js#v2",
|
||||
.resource_type = .script,
|
||||
}));
|
||||
try testing.expect(!testIsUrlBlocked(&client, .{
|
||||
.url = "https://cdn.example.com/ABCD.js",
|
||||
.resource_type = .script,
|
||||
}));
|
||||
|
||||
try testing.expect(testIsUrlBlocked(&client, .{ .url = "https://ads.example.com/pixel.gif" }));
|
||||
// Hostnames are matched case-insensitively and without the port.
|
||||
try testing.expect(testIsUrlBlocked(&client, .{ .url = "https://SUB.ADS.EXAMPLE.COM:8443/x" }));
|
||||
|
||||
@@ -29,6 +29,7 @@ const Parser = @import("Parser.zig");
|
||||
const Engine = @import("Engine.zig");
|
||||
const HostnameTrie = @import("HostnameTrie.zig");
|
||||
const NetworkFilter = @import("NetworkFilter.zig");
|
||||
const Regex = @import("Regex.zig");
|
||||
|
||||
const log = lp.log;
|
||||
|
||||
@@ -47,6 +48,11 @@ arena: std.heap.ArenaAllocator,
|
||||
filters: std.ArrayList(NetworkFilter),
|
||||
/// Filled while parsing, consumed by `build`.
|
||||
badfilters: std.AutoHashMapUnmanaged(u64, void),
|
||||
/// Every regex compiled for a filter, whether or not `build` kept the
|
||||
/// filter; they are freed here, not through `filters`.
|
||||
regexes: std.ArrayList(*const Regex),
|
||||
/// What the regexes are compiled and run with; PCRE2 allocates through it.
|
||||
regex_context: *Regex.Context,
|
||||
built: bool,
|
||||
trie: HostnameTrie,
|
||||
blocked: u32,
|
||||
@@ -64,7 +70,7 @@ exceptions: Engine,
|
||||
/// Rules that reached a trie or an index, across every list parsed so far.
|
||||
rules_loaded: usize,
|
||||
/// Rules the lists carry and we do not apply, across those same lists: the
|
||||
/// ones the parser could not read, plus the ones no matcher can express.
|
||||
/// ones the parser could not read, regex literals PCRE2 rejects included.
|
||||
rules_skipped: usize,
|
||||
/// Domain-scoped element-hiding rules and cosmetic-realm exceptions, across
|
||||
/// those same lists. Not ours to apply, but not rules we failed at either, so
|
||||
@@ -77,6 +83,8 @@ rules_cosmetic: usize,
|
||||
const LINE_MAX = 8 * 1024;
|
||||
|
||||
pub fn init(allocator: Allocator) Allocator.Error!AdBlocker {
|
||||
const regex_context: *Regex.Context = try .init(allocator);
|
||||
errdefer regex_context.deinit();
|
||||
var trie: HostnameTrie = try .init(allocator);
|
||||
errdefer trie.deinit(allocator);
|
||||
const blocked = try trie.createTrie(allocator);
|
||||
@@ -89,6 +97,8 @@ pub fn init(allocator: Allocator) Allocator.Error!AdBlocker {
|
||||
.arena = std.heap.ArenaAllocator.init(allocator),
|
||||
.filters = .empty,
|
||||
.badfilters = .empty,
|
||||
.regexes = .empty,
|
||||
.regex_context = regex_context,
|
||||
.built = false,
|
||||
.trie = trie,
|
||||
.blocked = blocked,
|
||||
@@ -109,6 +119,9 @@ pub fn deinit(self: *AdBlocker) void {
|
||||
self.blocking_important.deinit(self.allocator);
|
||||
self.exceptions.deinit(self.allocator);
|
||||
self.badfilters.deinit(self.allocator);
|
||||
for (self.regexes.items) |regex| regex.deinit();
|
||||
self.regexes.deinit(self.allocator);
|
||||
self.regex_context.deinit();
|
||||
self.filters.deinit(self.allocator);
|
||||
self.trie.deinit(self.allocator);
|
||||
self.arena.deinit();
|
||||
@@ -180,7 +193,7 @@ pub fn parse(self: *AdBlocker, reader: *Io.Reader) !void {
|
||||
// both of which the next call reuses, so what we keep is copied out.
|
||||
defer _ = scratch_instance.reset(.retain_capacity);
|
||||
|
||||
const filter = switch (item) {
|
||||
var filter = switch (item) {
|
||||
// The parser already counted it; all we want is whether it was an
|
||||
// exception, in which case its hostname stops being blockable.
|
||||
.dropped => |dropped| {
|
||||
@@ -201,11 +214,6 @@ pub fn parse(self: *AdBlocker, reader: *Io.Reader) !void {
|
||||
self.rules_cosmetic += 1;
|
||||
continue;
|
||||
}
|
||||
if (!isSupported(&filter)) {
|
||||
self.rules_skipped += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (filter.badfilter) {
|
||||
// It cancels a rule that may not have been read yet, so it can
|
||||
// only be resolved once every list is in. It is not a rule we
|
||||
@@ -215,6 +223,22 @@ pub fn parse(self: *AdBlocker, reader: *Io.Reader) !void {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (filter.kind == .regex) {
|
||||
const body = filter.pattern[1 .. filter.pattern.len - 1];
|
||||
const compiled = Regex.compile(self.regex_context, body, !filter.match_case) catch |err| switch (err) {
|
||||
error.OutOfMemory => return error.OutOfMemory,
|
||||
error.InvalidRegex => {
|
||||
self.rules_skipped += 1;
|
||||
continue;
|
||||
},
|
||||
};
|
||||
errdefer compiled.deinit();
|
||||
const regex = try arena.create(Regex);
|
||||
regex.* = compiled;
|
||||
try self.regexes.append(self.allocator, regex);
|
||||
filter.regex = regex;
|
||||
}
|
||||
|
||||
try self.filters.append(self.allocator, try dupe(arena, &filter));
|
||||
self.rules_loaded += 1;
|
||||
}
|
||||
@@ -361,12 +385,6 @@ pub fn match(self: *const AdBlocker, request: *const Request) Verdict {
|
||||
return .blocked;
|
||||
}
|
||||
|
||||
/// Whether we can evaluate this filter at all.
|
||||
fn isSupported(filter: *const NetworkFilter) bool {
|
||||
// No regex engine to run them with.
|
||||
return filter.kind != .regex;
|
||||
}
|
||||
|
||||
/// Whether the filter's whole effect is "every request to this hostname and
|
||||
/// its subdomains", which is all a trie can express.
|
||||
fn isWholeHostname(filter: *const NetworkFilter) bool {
|
||||
@@ -451,11 +469,8 @@ fn expectVerdict(
|
||||
source: []const u8,
|
||||
kind: ResourceTypes,
|
||||
) !void {
|
||||
// `match` takes a lowercased URL, as the real caller hands it; patterns
|
||||
// are lowercased too, so a rule spelled "/embed/C-iDzdvIg1Y" still lands.
|
||||
var buf: [512]u8 = undefined;
|
||||
const lowered = std.ascii.lowerString(buf[0..url.len], url);
|
||||
const request: Request = .init(lowered, source, kind);
|
||||
const request: Request = Request.init(url, &buf, source, kind).?;
|
||||
try testing.expectEqual(expected, blocker.match(&request));
|
||||
}
|
||||
|
||||
@@ -591,7 +606,8 @@ test "adblock.AdBlocker: tokens past the request buffer still match" {
|
||||
// 140 tokens of query noise push the filter's token ("utm", its rarest)
|
||||
// past what the request holds; the engine walks the rest of the URL.
|
||||
const noise = "https://example.com/?" ++ "a=1&" ** 70;
|
||||
const overflowing: Request = .init(noise ++ "utm_tracker=1", "a.com", script);
|
||||
var buf: [512]u8 = undefined;
|
||||
const overflowing: Request = Request.init(noise ++ "utm_tracker=1", &buf, "a.com", script).?;
|
||||
try testing.expect(overflowing.tail.len != 0);
|
||||
try testing.expectEqual(.blocked, blocker.match(&overflowing));
|
||||
// A token past the buffer finds its bucket, but the pattern does not fit.
|
||||
@@ -608,7 +624,7 @@ test "adblock.AdBlocker: cosmetic-realm rules are not skipped rules" {
|
||||
\\@@||example.com^$generichide
|
||||
\\@@||example.com^$elemhide
|
||||
\\example.com##.ad-banner
|
||||
\\/regex-we-cannot-run/
|
||||
\\/regex-we-cannot-run(/
|
||||
);
|
||||
|
||||
try testing.expectEqual(1, blocker.rules_loaded);
|
||||
@@ -616,6 +632,65 @@ test "adblock.AdBlocker: cosmetic-realm rules are not skipped rules" {
|
||||
try testing.expectEqual(3, blocker.rules_cosmetic);
|
||||
}
|
||||
|
||||
test "adblock.AdBlocker: the regex rules from EasyList" {
|
||||
var blocker: AdBlocker = try .init(testing.allocator);
|
||||
defer blocker.deinit();
|
||||
|
||||
try testLoad(&blocker,
|
||||
\\/\/[0-9a-f]{32}\/invoke\.js/$script,third-party
|
||||
\\/^https?:\/\/[0-9a-z]{5,}\.com\/.*/$script,third-party,xmlhttprequest,domain=dood.to
|
||||
\\@@/\/invoke\.js/$domain=trusted.com
|
||||
\\/\/[a-z]{4}\.js$/$match-case,script
|
||||
);
|
||||
try testing.expectEqual(4, blocker.rules_loaded);
|
||||
try testing.expectEqual(0, blocker.rules_skipped);
|
||||
// Each carries a literal every match has, so none rides the fallback.
|
||||
try testing.expectEqual(0, blocker.blocking.fallback.len);
|
||||
try testing.expectEqual(0, blocker.exceptions.fallback.len);
|
||||
|
||||
const invoke = "https://host.com/0123456789abcdef0123456789abcdef/invoke.js";
|
||||
try expectVerdict(&blocker, .blocked, invoke, "site.com", script);
|
||||
try expectVerdict(&blocker, .blocked, "https://HOST.com/0123456789ABCDEF0123456789abcdef/invoke.js", "site.com", script);
|
||||
try expectVerdict(&blocker, .none, invoke, "host.com", script);
|
||||
try expectVerdict(&blocker, .none, invoke, "site.com", image);
|
||||
try expectVerdict(&blocker, .none, "https://host.com/0123456789abcdef0123456789abcde/invoke.js", "site.com", script);
|
||||
try expectVerdict(&blocker, .allowed, invoke, "trusted.com", script);
|
||||
|
||||
try expectVerdict(&blocker, .blocked, "https://abcde.com/x", "dood.to", xhr);
|
||||
try expectVerdict(&blocker, .none, "https://abcde.com/x", "other.to", xhr);
|
||||
try expectVerdict(&blocker, .none, "https://abcd.com/x", "dood.to", xhr);
|
||||
|
||||
// `$match-case` reads the URL as requested, not the lowercased copy.
|
||||
try expectVerdict(&blocker, .blocked, "https://x.com/abcd.js", "x.com", script);
|
||||
try expectVerdict(&blocker, .none, "https://x.com/ABCD.js", "x.com", script);
|
||||
}
|
||||
|
||||
test "adblock.AdBlocker: a regex is found under every token it may match" {
|
||||
var blocker: AdBlocker = try .init(testing.allocator);
|
||||
defer blocker.deinit();
|
||||
|
||||
try testLoad(&blocker,
|
||||
\\/\/ads(\/?x|\/y)/$script
|
||||
);
|
||||
try expectVerdict(&blocker, .blocked, "https://example.com/adsx", "example.com", script);
|
||||
try expectVerdict(&blocker, .blocked, "https://example.com/ads/x", "example.com", script);
|
||||
try expectVerdict(&blocker, .blocked, "https://example.com/ads/y", "example.com", script);
|
||||
try expectVerdict(&blocker, .none, "https://example.com/ads/z", "example.com", script);
|
||||
}
|
||||
|
||||
test "adblock.AdBlocker: $badfilter removes a regex rule" {
|
||||
var blocker: AdBlocker = try .init(testing.allocator);
|
||||
defer blocker.deinit();
|
||||
|
||||
try testLoad(&blocker,
|
||||
\\/\/invoke\.js/$script
|
||||
\\/\/invoke\.js/$script,badfilter
|
||||
);
|
||||
try testing.expectEqual(0, blocker.rules_loaded);
|
||||
try testing.expectEqual(2, blocker.rules_skipped);
|
||||
try expectVerdict(&blocker, .none, "https://host.com/invoke.js", "site.com", script);
|
||||
}
|
||||
|
||||
test "adblock.AdBlocker: verdict precedence" {
|
||||
var blocker: AdBlocker = try .init(testing.allocator);
|
||||
defer blocker.deinit();
|
||||
|
||||
+400
-35
@@ -95,14 +95,18 @@ pub const Request = struct {
|
||||
source: [SOURCE_MAX]u8,
|
||||
};
|
||||
|
||||
/// `url` must already be lowercased and fragment-free; `source_hostname`
|
||||
/// may be empty when there is no document context.
|
||||
/// `buf` backs the lowercased URL; null when `url` is longer than it.
|
||||
/// `source_hostname` may be empty when there is no document context.
|
||||
pub fn init(
|
||||
url: []const u8,
|
||||
buf: []u8,
|
||||
source_hostname: []const u8,
|
||||
kind: NetworkFilter.ResourceTypes,
|
||||
) Request {
|
||||
const parsed: pattern.Url = .init(url);
|
||||
) ?Request {
|
||||
// No request URL carries a fragment onto the wire.
|
||||
const raw = URL.stripFragment(url);
|
||||
const lowered = lowercase(raw, buf) orelse return null;
|
||||
const parsed: pattern.Url = .init(lowered, raw);
|
||||
const source = if (source_hostname.len == 0) parsed.hostname() else source_hostname;
|
||||
var request: Request = .{
|
||||
.url = parsed,
|
||||
@@ -113,19 +117,18 @@ pub const Request = struct {
|
||||
.tokens_len = 0,
|
||||
.tail = "",
|
||||
};
|
||||
var it: Tokens = .{ .text = url };
|
||||
var it: Tokens = .{ .text = lowered };
|
||||
while (request.tokens_len < request.tokens_buf.len) {
|
||||
const token = it.next() orelse break;
|
||||
request.tokens_buf[request.tokens_len] = token;
|
||||
request.tokens_len += 1;
|
||||
}
|
||||
request.tail = url[it.i..];
|
||||
request.tail = lowered[it.i..];
|
||||
return request;
|
||||
}
|
||||
|
||||
pub fn fromHttp(transfer: *const HttpClient.Transfer, buffers: *Buffers) ?Request {
|
||||
const req = &transfer.req;
|
||||
const url = normalizeUrl(req.url, &buffers.url) orelse return null;
|
||||
|
||||
var owner: ?*const HttpClient.Owner = transfer.owner;
|
||||
var subframe = false;
|
||||
@@ -148,27 +151,23 @@ pub const Request = struct {
|
||||
.worker => .{ .script = true },
|
||||
};
|
||||
|
||||
return .init(url, source, resource_type);
|
||||
return .init(req.url, &buffers.url, source, resource_type);
|
||||
}
|
||||
|
||||
inline fn tokens(self: *const Request) []const u32 {
|
||||
return self.tokens_buf[0..self.tokens_len];
|
||||
}
|
||||
|
||||
/// Lowercases `url` into `buf` with its fragment stripped; patterns are
|
||||
/// stored lowercased, and no request URL carries a fragment onto the wire.
|
||||
fn normalizeUrl(url: []const u8, buf: []u8) ?[]const u8 {
|
||||
const end = std.mem.indexOfScalar(u8, url, '#') orelse url.len;
|
||||
const trimmed = url[0..end];
|
||||
|
||||
const upper = for (trimmed, 0..) |c, i| {
|
||||
/// Lowercases `url` into `buf`, as patterns are stored lowercased.
|
||||
fn lowercase(url: []const u8, buf: []u8) ?[]const u8 {
|
||||
const upper = for (url, 0..) |c, i| {
|
||||
if (std.ascii.isUpper(c)) break i;
|
||||
} else return trimmed;
|
||||
} else return url;
|
||||
|
||||
if (trimmed.len > buf.len) return null;
|
||||
const out = buf[0..trimmed.len];
|
||||
@memcpy(out[0..upper], trimmed[0..upper]);
|
||||
_ = std.ascii.lowerString(out[upper..], trimmed[upper..]);
|
||||
if (url.len > buf.len) return null;
|
||||
const out = buf[0..url.len];
|
||||
@memcpy(out[0..upper], url[0..upper]);
|
||||
_ = std.ascii.lowerString(out[upper..], url[upper..]);
|
||||
return out;
|
||||
}
|
||||
};
|
||||
@@ -326,11 +325,20 @@ fn collectTokens(filter: *const NetworkFilter, buf: []u32) []u32 {
|
||||
}
|
||||
}
|
||||
|
||||
// A /regex/ literal is never indexed (nothing can run it), and `.any`
|
||||
// has no pattern at all.
|
||||
if (filter.kind == .regex or filter.pattern.len == 0) return buf[0..n];
|
||||
// A /regex/ literal has no hostname, so `n` is still 0 here.
|
||||
if (filter.kind == .regex) return regexTokens(filter.pattern[1 .. filter.pattern.len - 1], buf);
|
||||
// `.any` has no pattern at all.
|
||||
if (filter.pattern.len == 0) return buf[0..n];
|
||||
|
||||
const text = filter.pattern;
|
||||
const left_anchored = filter.left_anchor or filter.hostname_anchor;
|
||||
return buf[0..boundedTokens(filter.pattern, left_anchored, filter.right_anchor, buf, n)];
|
||||
}
|
||||
|
||||
/// Appends the tokens of `text` that any URL matching it carries whole: the
|
||||
/// alphanumeric runs no `*` may extend, and that the open ends of the text
|
||||
/// may not extend either unless anchored. Returns the new count.
|
||||
fn boundedTokens(text: []const u8, left_anchored: bool, right_anchored: bool, buf: []u32, from: usize) usize {
|
||||
var n = from;
|
||||
var i: usize = 0;
|
||||
while (i < text.len) {
|
||||
if (!isTokenChar(text[i])) {
|
||||
@@ -340,22 +348,317 @@ fn collectTokens(filter: *const NetworkFilter, buf: []u32) []u32 {
|
||||
const start = i;
|
||||
while (i < text.len and isTokenChar(text[i])) i += 1;
|
||||
|
||||
const left_bounded = if (start == 0)
|
||||
filter.left_anchor or filter.hostname_anchor
|
||||
else
|
||||
text[start - 1] != '*';
|
||||
const right_bounded = if (i == text.len) filter.right_anchor else text[i] != '*';
|
||||
const left_bounded = if (start == 0) left_anchored else text[start - 1] != '*';
|
||||
const right_bounded = if (i == text.len) right_anchored else text[i] != '*';
|
||||
|
||||
if (left_bounded and right_bounded) {
|
||||
if (n == buf.len) return buf[0..n];
|
||||
if (n == buf.len) return n;
|
||||
buf[n] = hash(text[start..i]);
|
||||
n += 1;
|
||||
}
|
||||
}
|
||||
|
||||
return buf[0..n];
|
||||
return n;
|
||||
}
|
||||
|
||||
/// The tokens a regex is sure to have wherever it matches, uBO's
|
||||
/// `tokenizableStrFromRegex`: the pattern is flattened into a plain pattern
|
||||
/// where literal token characters stay and everything else becomes a marker
|
||||
/// saying only whether it could be one. Anything the flattening does not
|
||||
/// follow yields no token at all, which is never wrong: the filter then
|
||||
/// rides the fallback bucket.
|
||||
fn regexTokens(source: []const u8, buf: []u32) []u32 {
|
||||
// A pattern's shape is never longer than the pattern.
|
||||
var shape_buf: [8 * 1024]u8 = undefined;
|
||||
if (source.len > shape_buf.len) return buf[0..0];
|
||||
var shape: RegexShape = .{ .source = source, .out = &shape_buf };
|
||||
const text = shape.flatten() catch return buf[0..0];
|
||||
// Anchors are in the shape itself; its ends are open.
|
||||
return buf[0..boundedTokens(text, false, false, buf, 0)];
|
||||
}
|
||||
|
||||
const RegexShape = struct {
|
||||
source: []const u8,
|
||||
out: []u8,
|
||||
i: usize = 0,
|
||||
n: usize = 0,
|
||||
|
||||
/// Whatever matches here is not a token character: a non-token literal,
|
||||
/// an anchor, `\b`.
|
||||
const not_token = 0x00;
|
||||
/// Whatever matches here may be a token character: `.`, `[a-z]`, `\d`, a
|
||||
/// quantified literal. Spelled as a plain pattern's wildcard so the shape
|
||||
/// reads as one.
|
||||
const maybe_token = '*';
|
||||
// The same two for a stretch that may match nothing at all; resolved
|
||||
// once both neighbours are known.
|
||||
const not_token_optional = 0x02;
|
||||
const maybe_token_optional = 0x03;
|
||||
|
||||
const Error = error{Unsupported};
|
||||
|
||||
fn flatten(self: *RegexShape) Error![]const u8 {
|
||||
try self.alternation(false);
|
||||
if (self.i != self.source.len) return error.Unsupported;
|
||||
self.resolveOptional();
|
||||
return self.out[0..self.n];
|
||||
}
|
||||
|
||||
/// Parses alternatives up to the closing parenthesis (or the end). More
|
||||
/// than one collapses to two markers: all that is sure about `a|bc` is
|
||||
/// how it starts and ends.
|
||||
fn alternation(self: *RegexShape, nested: bool) Error!void {
|
||||
const start = self.n;
|
||||
var branches: usize = 1;
|
||||
var branch_start = start;
|
||||
var first = false;
|
||||
var last = false;
|
||||
while (true) {
|
||||
try self.sequence(nested);
|
||||
const branch = self.out[branch_start..self.n];
|
||||
first = first or startsTokenish(branch);
|
||||
last = last or endsTokenish(branch);
|
||||
if (self.i == self.source.len or self.source[self.i] != '|') break;
|
||||
self.i += 1;
|
||||
branches += 1;
|
||||
branch_start = self.n;
|
||||
}
|
||||
if (branches == 1) return;
|
||||
self.n = start;
|
||||
self.emit(if (first) maybe_token else not_token);
|
||||
self.emit(if (last) maybe_token else not_token);
|
||||
}
|
||||
|
||||
fn sequence(self: *RegexShape, nested: bool) Error!void {
|
||||
while (self.i < self.source.len) {
|
||||
const atom_start = self.n;
|
||||
const c = self.source[self.i];
|
||||
switch (c) {
|
||||
'|' => return,
|
||||
')' => {
|
||||
if (!nested) return error.Unsupported;
|
||||
return;
|
||||
},
|
||||
'(' => try self.group(),
|
||||
'[' => try self.class(),
|
||||
'\\' => try self.escape(),
|
||||
'.' => {
|
||||
self.i += 1;
|
||||
self.emit(maybe_token);
|
||||
},
|
||||
'^', '$' => {
|
||||
self.i += 1;
|
||||
self.emit(not_token);
|
||||
},
|
||||
'*', '+', '?' => return error.Unsupported,
|
||||
else => {
|
||||
self.i += 1;
|
||||
self.emitLiteral(c);
|
||||
},
|
||||
}
|
||||
try self.quantifier(atom_start);
|
||||
}
|
||||
}
|
||||
|
||||
fn group(self: *RegexShape) Error!void {
|
||||
const start = self.n;
|
||||
self.i += 1;
|
||||
var lookaround: enum { none, positive, negative } = .none;
|
||||
if (self.i < self.source.len and self.source[self.i] == '?') {
|
||||
self.i += 1;
|
||||
const kind = self.take() orelse return error.Unsupported;
|
||||
switch (kind) {
|
||||
':' => {},
|
||||
'=' => lookaround = .positive,
|
||||
'!' => lookaround = .negative,
|
||||
'<' => {
|
||||
const next = self.take() orelse return error.Unsupported;
|
||||
switch (next) {
|
||||
'=' => lookaround = .positive,
|
||||
'!' => lookaround = .negative,
|
||||
else => {
|
||||
// A named group is a plain group with a label.
|
||||
const close = std.mem.indexOfScalarPos(u8, self.source, self.i, '>') orelse return error.Unsupported;
|
||||
self.i = close + 1;
|
||||
},
|
||||
}
|
||||
},
|
||||
else => return error.Unsupported,
|
||||
}
|
||||
}
|
||||
try self.alternation(true);
|
||||
if (self.take() != ')') return error.Unsupported;
|
||||
switch (lookaround) {
|
||||
.none => {},
|
||||
// Consumes nothing, so the neighbours touch; what it asserts
|
||||
// could still be anything, and that is all a token may rely on.
|
||||
.positive => {
|
||||
self.n = start;
|
||||
self.emit(maybe_token);
|
||||
},
|
||||
// Consumes nothing and rules text out: the neighbours touch.
|
||||
.negative => self.n = start,
|
||||
}
|
||||
}
|
||||
|
||||
/// `[...]` is one character; a token character can come out of it if any
|
||||
/// member is one, or if it is negated.
|
||||
fn class(self: *RegexShape) Error!void {
|
||||
self.i += 1;
|
||||
var maybe = false;
|
||||
if (self.i < self.source.len and self.source[self.i] == '^') {
|
||||
self.i += 1;
|
||||
maybe = true;
|
||||
}
|
||||
var first = true;
|
||||
while (true) {
|
||||
const c = self.take() orelse return error.Unsupported;
|
||||
if (c == ']' and !first) break;
|
||||
first = false;
|
||||
if (c == '\\') {
|
||||
const e = self.take() orelse return error.Unsupported;
|
||||
switch (e) {
|
||||
'd', 'D', 'w', 'W', 's', 'S' => maybe = true,
|
||||
'b', 'n', 'r', 't', 'f', 'v' => {},
|
||||
else => if (std.ascii.isAlphanumeric(e)) return error.Unsupported,
|
||||
}
|
||||
continue;
|
||||
}
|
||||
// A range is assumed to reach token characters.
|
||||
if (c == '-' or std.ascii.isAlphanumeric(c)) maybe = true;
|
||||
}
|
||||
self.emit(if (maybe) maybe_token else not_token);
|
||||
}
|
||||
|
||||
fn escape(self: *RegexShape) Error!void {
|
||||
self.i += 1;
|
||||
const e = self.take() orelse return error.Unsupported;
|
||||
switch (e) {
|
||||
'd', 'D', 'w', 'W', 's', 'S', 'B' => self.emit(maybe_token),
|
||||
'b', 'n', 'r', 't', 'f', 'v' => self.emit(not_token),
|
||||
// Code points, backreferences, properties: not worth following.
|
||||
'x', 'u', 'c', 'k', 'p', 'P', '0'...'9' => return error.Unsupported,
|
||||
// Anything else escaped is itself, as JavaScript reads it.
|
||||
else => self.emitLiteral(e),
|
||||
}
|
||||
}
|
||||
|
||||
/// Applies a quantifier, if one follows, to what was just emitted. Only
|
||||
/// the first and last character classes survive a repeat: `ab+` may match
|
||||
/// "abbb", and its token is not "ab". Whether the atom may be absent is
|
||||
/// all that matters beyond that.
|
||||
fn quantifier(self: *RegexShape, atom_start: usize) Error!void {
|
||||
if (self.i == self.source.len) return;
|
||||
var optional = true;
|
||||
switch (self.source[self.i]) {
|
||||
'*', '?' => self.i += 1,
|
||||
'+' => {
|
||||
self.i += 1;
|
||||
optional = false;
|
||||
},
|
||||
'{' => {
|
||||
const close = std.mem.indexOfScalarPos(u8, self.source, self.i, '}') orelse return;
|
||||
const body = self.source[self.i + 1 .. close];
|
||||
const comma = std.mem.indexOfScalar(u8, body, ',') orelse body.len;
|
||||
// Not a quantifier at all: JavaScript reads the `{` literally.
|
||||
const min = std.fmt.parseUnsigned(usize, body[0..comma], 10) catch return;
|
||||
if (comma + 1 < body.len) _ = std.fmt.parseUnsigned(usize, body[comma + 1 ..], 10) catch return;
|
||||
optional = min == 0;
|
||||
self.i = close + 1;
|
||||
},
|
||||
else => return,
|
||||
}
|
||||
// A lazy `?` changes nothing about what can match.
|
||||
if (self.i < self.source.len and self.source[self.i] == '?') self.i += 1;
|
||||
|
||||
const atom = self.out[atom_start..self.n];
|
||||
const first = startsTokenish(atom);
|
||||
const last = endsTokenish(atom);
|
||||
self.n = atom_start;
|
||||
if (!optional) {
|
||||
self.emit(if (first) maybe_token else not_token);
|
||||
self.emit(if (last) maybe_token else not_token);
|
||||
} else {
|
||||
self.emit(if (first) maybe_token_optional else not_token_optional);
|
||||
self.emit(if (last) maybe_token_optional else not_token_optional);
|
||||
}
|
||||
}
|
||||
|
||||
/// An optional stretch may vanish, letting its neighbours touch: it
|
||||
/// resolves to markers that carry whichever side could be a token
|
||||
/// character, so no run is read as bounded by something that may be
|
||||
/// gone.
|
||||
fn resolveOptional(self: *RegexShape) void {
|
||||
var i: usize = 0;
|
||||
while (i < self.n) {
|
||||
if (!isOptional(self.out[i])) {
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
var end = i;
|
||||
while (end < self.n and isOptional(self.out[end])) end += 1;
|
||||
const left = self.out[0..i];
|
||||
const middle = self.out[i..end];
|
||||
const right = self.out[end..self.n];
|
||||
const head: u8 = if (startsTokenish(right) or startsTokenish(middle)) maybe_token else not_token;
|
||||
const tail: u8 = if (endsTokenish(left) or endsTokenish(middle)) maybe_token else not_token;
|
||||
// Quantifiers emit markers in pairs, so a stretch is never shorter
|
||||
// than what replaces it.
|
||||
std.mem.copyForwards(u8, self.out[i + 2 .. self.n - (middle.len - 2)], right);
|
||||
self.out[i] = head;
|
||||
self.out[i + 1] = tail;
|
||||
self.n -= middle.len - 2;
|
||||
i += 2;
|
||||
}
|
||||
}
|
||||
|
||||
fn take(self: *RegexShape) ?u8 {
|
||||
if (self.i == self.source.len) return null;
|
||||
defer self.i += 1;
|
||||
return self.source[self.i];
|
||||
}
|
||||
|
||||
fn emit(self: *RegexShape, c: u8) void {
|
||||
self.out[self.n] = c;
|
||||
self.n += 1;
|
||||
}
|
||||
|
||||
/// A literal is kept only as far as tokens care: a token character,
|
||||
/// lowercased like the URL it is looked up in, or the fact that it is not
|
||||
/// one.
|
||||
fn emitLiteral(self: *RegexShape, c: u8) void {
|
||||
self.emit(if (std.ascii.isAlphanumeric(c)) std.ascii.toLower(c) else not_token);
|
||||
}
|
||||
|
||||
fn isOptional(c: u8) bool {
|
||||
return c == not_token_optional or c == maybe_token_optional;
|
||||
}
|
||||
|
||||
fn isTokenish(c: u8) bool {
|
||||
return c == maybe_token or c == maybe_token_optional or isTokenChar(c);
|
||||
}
|
||||
|
||||
/// Whether what `s` matches could start with a token character. An
|
||||
/// optional non-token stretch may be absent, so whatever follows it
|
||||
/// answers instead.
|
||||
fn startsTokenish(s: []const u8) bool {
|
||||
for (s) |c| {
|
||||
if (isTokenish(c)) return true;
|
||||
if (c != not_token_optional) return false;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
fn endsTokenish(s: []const u8) bool {
|
||||
var i = s.len;
|
||||
while (i > 0) {
|
||||
i -= 1;
|
||||
if (isTokenish(s[i])) return true;
|
||||
if (s[i] != not_token_optional) return false;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
const testing = @import("../../testing.zig");
|
||||
|
||||
fn tokensOf(arena: Allocator, line: []const u8, buf: []u32) ![]u32 {
|
||||
@@ -370,18 +673,80 @@ fn contains(tokens: []const u32, token: []const u8) bool {
|
||||
return false;
|
||||
}
|
||||
|
||||
test "adblock.Engine: regex filters yield the tokens every match carries" {
|
||||
var arena_state = std.heap.ArenaAllocator.init(testing.allocator);
|
||||
defer arena_state.deinit();
|
||||
const arena = arena_state.allocator();
|
||||
var buf: [TOKENS_MAX]u32 = undefined;
|
||||
|
||||
// Literal runs between literal non-token characters; the open end of an
|
||||
// unanchored pattern bounds nothing.
|
||||
var tokens = try tokensOf(arena, "/\\/[0-9a-f]{32}\\/invoke\\.js/", &buf);
|
||||
try testing.expectEqual(1, tokens.len);
|
||||
try testing.expect(contains(tokens, "invoke"));
|
||||
|
||||
// Anchors bound; `https?` may be either, so "http" is no token.
|
||||
tokens = try tokensOf(arena, "/^https?:\\/\\/[0-9a-z]{5,}\\.com\\/.*/", &buf);
|
||||
try testing.expectEqual(1, tokens.len);
|
||||
try testing.expect(contains(tokens, "com"));
|
||||
tokens = try tokensOf(arena, "/[a-z]{2,}\\.gif$/", &buf);
|
||||
try testing.expectEqual(1, tokens.len);
|
||||
try testing.expect(contains(tokens, "gif"));
|
||||
tokens = try tokensOf(arena, "/about:blank.*/", &buf);
|
||||
try testing.expectEqual(0, tokens.len);
|
||||
|
||||
// `\b` bounds, as uBO reads it: `/\bads\b/` is "ads", not "bads".
|
||||
tokens = try tokensOf(arena, "/\\bads\\b/", &buf);
|
||||
try testing.expect(contains(tokens, "ads"));
|
||||
|
||||
// Hashed lowercased, like the URL it is looked up in.
|
||||
tokens = try tokensOf(arena, "/\\/Ads\\//$match-case", &buf);
|
||||
try testing.expect(contains(tokens, "ads"));
|
||||
|
||||
// An optional stretch may vanish and glue its neighbours: "adsbanner".
|
||||
tokens = try tokensOf(arena, "/\\/ads\\/?banner\\//", &buf);
|
||||
try testing.expectEqual(0, tokens.len);
|
||||
// ... also from inside a group: "adsx" is a match.
|
||||
tokens = try tokensOf(arena, "/\\/ads(\\/?x|\\/y)/", &buf);
|
||||
try testing.expectEqual(0, tokens.len);
|
||||
// A repeat is not the literal it repeats.
|
||||
tokens = try tokensOf(arena, "/\\/ab+c\\//", &buf);
|
||||
try testing.expectEqual(0, tokens.len);
|
||||
|
||||
// Alternation and the text under a quantified group are uncertain.
|
||||
tokens = try tokensOf(arena, "/^https?:\\/\\/(35|104)\\.(\\d){1,3}\\//", &buf);
|
||||
try testing.expectEqual(0, tokens.len);
|
||||
// ... but a group with one branch is transparent.
|
||||
tokens = try tokensOf(arena, "/\\/(?:ads)\\//", &buf);
|
||||
try testing.expect(contains(tokens, "ads"));
|
||||
|
||||
// A negative lookaround consumes nothing; a positive one is not
|
||||
// trusted to spell anything.
|
||||
tokens = try tokensOf(arena, "/\\/(?!ads)banner\\//", &buf);
|
||||
try testing.expect(contains(tokens, "banner"));
|
||||
tokens = try tokensOf(arena, "/\\/(?=ads)ads\\//", &buf);
|
||||
try testing.expectEqual(0, tokens.len);
|
||||
|
||||
// What is not followed yields nothing rather than something wrong.
|
||||
tokens = try tokensOf(arena, "/\\/\\x41ds\\//", &buf);
|
||||
try testing.expectEqual(0, tokens.len);
|
||||
tokens = try tokensOf(arena, "/\\/(ads\\//", &buf);
|
||||
try testing.expectEqual(0, tokens.len);
|
||||
}
|
||||
|
||||
test "adblock.Engine: a request keeps its first tokens, the rest as tail" {
|
||||
const max = Request.URL_TOKENS_MAX;
|
||||
const kind: NetworkFilter.ResourceTypes = .{ .script = true };
|
||||
|
||||
// Exactly as many tokens as the buffer holds: nothing is left to walk...
|
||||
const full = "x/" ** (max - 1) ++ "x";
|
||||
var request: Request = .init(full, "", kind);
|
||||
var buf: [512]u8 = undefined;
|
||||
var request: Request = Request.init(full, &buf, "", kind).?;
|
||||
try testing.expectEqual(max, request.tokens_len);
|
||||
try testing.expectEqual(0, request.tail.len);
|
||||
|
||||
// ...one more, and only that one is in the tail.
|
||||
request = .init(full ++ "/y", "", kind);
|
||||
request = Request.init(full ++ "/y", &buf, "", kind).?;
|
||||
try testing.expectEqual(max, request.tokens_len);
|
||||
try testing.expectString("/y", request.tail);
|
||||
var it: Tokens = .{ .text = request.tail };
|
||||
@@ -389,7 +754,7 @@ test "adblock.Engine: a request keeps its first tokens, the rest as tail" {
|
||||
try testing.expect(it.next() == null);
|
||||
|
||||
// Fewer than the buffer holds: the tail is empty.
|
||||
request = .init("https://example.com/a", "", kind);
|
||||
request = Request.init("https://example.com/a", &buf, "", kind).?;
|
||||
try testing.expectEqual(4, request.tokens_len);
|
||||
try testing.expectEqual(0, request.tail.len);
|
||||
}
|
||||
|
||||
@@ -18,6 +18,7 @@
|
||||
|
||||
const std = @import("std");
|
||||
const domain = @import("domain.zig");
|
||||
const Regex = @import("Regex.zig");
|
||||
|
||||
const NetworkFilter = @This();
|
||||
|
||||
@@ -38,6 +39,9 @@ exception: bool = false,
|
||||
important: bool = false,
|
||||
badfilter: bool = false,
|
||||
match_case: bool = false,
|
||||
/// The compiled `.regex` pattern, set by the blocker once it has read the
|
||||
/// literal; the parser never runs one.
|
||||
regex: ?*const Regex = null,
|
||||
first_party: bool = true,
|
||||
third_party: bool = true,
|
||||
hostname_anchor: bool = false,
|
||||
|
||||
@@ -0,0 +1,224 @@
|
||||
// Copyright (C) 2023-2026 Lightpanda (Selecy SAS)
|
||||
//
|
||||
// Francis Bouvier <francis@lightpanda.io>
|
||||
// Pierre Tachoire <pierre@lightpanda.io>
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as
|
||||
// published by the Free Software Foundation, either version 3 of the
|
||||
// License, or (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
//! A compiled `/regex/` filter body. Filter lists write them in JavaScript
|
||||
//! `RegExp` syntax and uBO runs them with `new RegExp(src, 'i')` against the
|
||||
//! raw request URL (no flag under `$match-case`); PCRE2 reads that syntax
|
||||
//! as-is, escapes like `\/` included.
|
||||
//!
|
||||
//! A compiled pattern and its `Context` are never modified after `compile`,
|
||||
//! so one `Regex` can be shared by every HTTP client thread; the per-call
|
||||
//! match data is what PCRE2 requires to be private.
|
||||
|
||||
const std = @import("std");
|
||||
const lp = @import("lightpanda");
|
||||
const pcre2 = @import("pcre2");
|
||||
|
||||
const Allocator = std.mem.Allocator;
|
||||
|
||||
const log = lp.log;
|
||||
|
||||
const Regex = @This();
|
||||
|
||||
code: *pcre2.pcre2_code_8,
|
||||
context: *const Context,
|
||||
|
||||
pub const Error = error{ InvalidRegex, OutOfMemory };
|
||||
|
||||
/// What every `Regex` compiled through it shares: the allocator PCRE2 draws
|
||||
/// from, and the compile and match settings. Outlives the regexes.
|
||||
///
|
||||
/// PCRE2 would happily use libc's malloc; it is handed the blocker's
|
||||
/// allocator so that a compiled pattern nobody freed fails a test the way
|
||||
/// any other leak does.
|
||||
pub const Context = struct {
|
||||
allocator: Allocator,
|
||||
general: *pcre2.pcre2_general_context_8,
|
||||
compile_context: *pcre2.pcre2_compile_context_8,
|
||||
match_context: *pcre2.pcre2_match_context_8,
|
||||
|
||||
// A pattern from a list that backtracks this much on one URL is broken,
|
||||
// not slow; giving up costs a false negative on that request, nothing
|
||||
// more.
|
||||
const MATCH_LIMIT = 100_000;
|
||||
const DEPTH_LIMIT = 10_000;
|
||||
|
||||
pub fn init(allocator: Allocator) Allocator.Error!*Context {
|
||||
const self = try allocator.create(Context);
|
||||
errdefer allocator.destroy(self);
|
||||
self.allocator = allocator;
|
||||
|
||||
// PCRE2 hands the allocator back to the callbacks by address, so the
|
||||
// context has to be at its final one before anything is allocated
|
||||
// through it.
|
||||
const general = pcre2.pcre2_general_context_create_8(cMalloc, cFree, &self.allocator) orelse return error.OutOfMemory;
|
||||
errdefer pcre2.pcre2_general_context_free_8(general);
|
||||
|
||||
const compile_context = pcre2.pcre2_compile_context_create_8(general) orelse return error.OutOfMemory;
|
||||
errdefer pcre2.pcre2_compile_context_free_8(compile_context);
|
||||
// JavaScript without the `u` flag reads an unknown escape as the
|
||||
// literal character, and that is the mode uBO compiles filters in.
|
||||
_ = pcre2.pcre2_set_compile_extra_options_8(compile_context, pcre2.PCRE2_EXTRA_BAD_ESCAPE_IS_LITERAL);
|
||||
|
||||
const match_context = pcre2.pcre2_match_context_create_8(general) orelse return error.OutOfMemory;
|
||||
_ = pcre2.pcre2_set_match_limit_8(match_context, MATCH_LIMIT);
|
||||
_ = pcre2.pcre2_set_depth_limit_8(match_context, DEPTH_LIMIT);
|
||||
|
||||
self.general = general;
|
||||
self.compile_context = compile_context;
|
||||
self.match_context = match_context;
|
||||
return self;
|
||||
}
|
||||
|
||||
pub fn deinit(self: *Context) void {
|
||||
pcre2.pcre2_match_context_free_8(self.match_context);
|
||||
pcre2.pcre2_compile_context_free_8(self.compile_context);
|
||||
pcre2.pcre2_general_context_free_8(self.general);
|
||||
self.allocator.destroy(self);
|
||||
}
|
||||
|
||||
// PCRE2 frees without a size, so every block carries its own in a
|
||||
// header that keeps the payload at malloc's alignment.
|
||||
const HEADER = 16;
|
||||
const alignment: std.mem.Alignment = .fromByteUnits(HEADER);
|
||||
|
||||
fn cMalloc(size: usize, data: ?*anyopaque) callconv(.c) ?*anyopaque {
|
||||
const allocator: *const Allocator = @ptrCast(@alignCast(data.?));
|
||||
const total = std.math.add(usize, size, HEADER) catch return null;
|
||||
const block = allocator.alignedAlloc(u8, alignment, total) catch return null;
|
||||
std.mem.writeInt(usize, block[0..@sizeOf(usize)], total, .little);
|
||||
return block.ptr + HEADER;
|
||||
}
|
||||
|
||||
fn cFree(ptr: ?*anyopaque, data: ?*anyopaque) callconv(.c) void {
|
||||
const payload = ptr orelse return;
|
||||
const allocator: *const Allocator = @ptrCast(@alignCast(data.?));
|
||||
const base: [*]align(HEADER) u8 = @ptrCast(@alignCast(@as([*]u8, @ptrCast(payload)) - HEADER));
|
||||
const total = std.mem.readInt(usize, base[0..@sizeOf(usize)], .little);
|
||||
allocator.free(base[0..total]);
|
||||
}
|
||||
};
|
||||
|
||||
pub fn compile(context: *const Context, pattern: []const u8, case_insensitive: bool) Error!Regex {
|
||||
const options: u32 = if (case_insensitive) pcre2.PCRE2_CASELESS else 0;
|
||||
var err_code: c_int = 0;
|
||||
var err_offset: usize = 0;
|
||||
const code = pcre2.pcre2_compile_8(
|
||||
pattern.ptr,
|
||||
pattern.len,
|
||||
options,
|
||||
&err_code,
|
||||
&err_offset,
|
||||
context.compile_context,
|
||||
) orelse {
|
||||
// A failed allocation is ours, not the pattern's.
|
||||
if (err_code == pcre2.PCRE2_ERROR_HEAP_FAILED) return error.OutOfMemory;
|
||||
var buf: [256]u8 = undefined;
|
||||
const len = pcre2.pcre2_get_error_message_8(err_code, &buf, buf.len);
|
||||
const message: []const u8 = if (len < 0) "unknown error" else buf[0..@intCast(len)];
|
||||
log.debug(.app, "adblock regex rejected", .{
|
||||
.pattern = pattern,
|
||||
.err = message,
|
||||
.offset = err_offset,
|
||||
});
|
||||
return error.InvalidRegex;
|
||||
};
|
||||
return .{ .code = code, .context = context };
|
||||
}
|
||||
|
||||
pub fn deinit(self: Regex) void {
|
||||
pcre2.pcre2_code_free_8(self.code);
|
||||
}
|
||||
|
||||
// What one match allocates: its match data and the 20KB of backtracking
|
||||
// frames PCRE2 starts with, which only a deeply nested pattern outgrows.
|
||||
const MATCH_SCRATCH = 24 * 1024;
|
||||
|
||||
/// Whether the pattern matches anywhere in `text`, as `RegExp.test` would
|
||||
/// answer. A match that hits the backtracking limits counts as no match.
|
||||
pub fn matches(self: Regex, text: []const u8) bool {
|
||||
// This runs per request; the scratch keeps the common case off the heap.
|
||||
var scratch = std.heap.stackFallback(MATCH_SCRATCH, self.context.allocator);
|
||||
var allocator = scratch.get();
|
||||
const general = pcre2.pcre2_general_context_create_8(Context.cMalloc, Context.cFree, &allocator) orelse return false;
|
||||
defer pcre2.pcre2_general_context_free_8(general);
|
||||
|
||||
// One pair is the whole-match span, all a test needs; capture groups in
|
||||
// the pattern are simply not recorded.
|
||||
const match_data = pcre2.pcre2_match_data_create_8(1, general) orelse return false;
|
||||
defer pcre2.pcre2_match_data_free_8(match_data);
|
||||
|
||||
const rc = pcre2.pcre2_match_8(self.code, text.ptr, text.len, 0, 0, match_data, self.context.match_context);
|
||||
return rc >= 0;
|
||||
}
|
||||
|
||||
const testing = @import("../../testing.zig");
|
||||
|
||||
test "adblock.Regex: JavaScript escapes and unanchored search" {
|
||||
const context: *Context = try .init(testing.allocator);
|
||||
defer context.deinit();
|
||||
|
||||
const regex = try Regex.compile(context, "^https?:\\/\\/[0-9a-z]{5,}\\.com\\/.*", true);
|
||||
defer regex.deinit();
|
||||
|
||||
try testing.expect(regex.matches("https://abcde.com/x"));
|
||||
try testing.expect(regex.matches("HTTPS://ABCDE.COM/X"));
|
||||
try testing.expect(!regex.matches("https://abcd.com/x"));
|
||||
try testing.expect(!regex.matches("https://abcde.org/x"));
|
||||
|
||||
const invoke = try Regex.compile(context, "\\/[0-9a-f]{32}\\/invoke\\.js", true);
|
||||
defer invoke.deinit();
|
||||
try testing.expect(invoke.matches("https://host.com/0123456789abcdef0123456789abcdef/invoke.js"));
|
||||
try testing.expect(!invoke.matches("https://host.com/0123456789abcdef0123456789abcde/invoke.js"));
|
||||
|
||||
const dash = try Regex.compile(context, "[a-z\\-]+\\?s=", true);
|
||||
defer dash.deinit();
|
||||
try testing.expect(dash.matches("https://x.com/a-b?s=1"));
|
||||
try testing.expect(!dash.matches("https://x.com/?s=1"));
|
||||
}
|
||||
|
||||
test "adblock.Regex: $match-case keeps the case" {
|
||||
const context: *Context = try .init(testing.allocator);
|
||||
defer context.deinit();
|
||||
|
||||
const exact = try Regex.compile(context, "\\/[a-z0-9]{12}\\/[a-zA-Z0-9]{20,}$", false);
|
||||
defer exact.deinit();
|
||||
try testing.expect(exact.matches("https://x.com/abcdef123456/aBcDeFgHiJkLmNoPqRsTuV"));
|
||||
try testing.expect(!exact.matches("https://x.com/ABCDEF123456/aBcDeFgHiJkLmNoPqRsTuV"));
|
||||
}
|
||||
|
||||
test "adblock.Regex: invalid patterns are errors, runaway ones no match" {
|
||||
const context: *Context = try .init(testing.allocator);
|
||||
defer context.deinit();
|
||||
|
||||
try testing.expectError(error.InvalidRegex, Regex.compile(context, "(", true));
|
||||
try testing.expectError(error.InvalidRegex, Regex.compile(context, "a{2,1}", true));
|
||||
|
||||
// An unknown alphanumeric escape is the literal, as in JavaScript.
|
||||
const literal = try Regex.compile(context, "\\q", true);
|
||||
defer literal.deinit();
|
||||
try testing.expect(literal.matches("https://x.com/q"));
|
||||
|
||||
// Exponential backtracking stops at the match limit instead of stalling
|
||||
// the request.
|
||||
const runaway = try Regex.compile(context, "^(a+)+$", true);
|
||||
defer runaway.deinit();
|
||||
const subject = "a" ** 64 ++ "b";
|
||||
try testing.expect(!runaway.matches(subject));
|
||||
try testing.expect(runaway.matches("a" ** 64));
|
||||
}
|
||||
@@ -28,8 +28,11 @@ const NetworkFilter = @import("NetworkFilter.zig");
|
||||
/// The URL a request is matched against, with its hostname located once so
|
||||
/// every filter can reuse the offsets.
|
||||
pub const Url = struct {
|
||||
/// Lowercased, fragment stripped.
|
||||
/// Lowercased, fragment stripped: what patterns, stored lowercased, walk.
|
||||
text: []const u8,
|
||||
/// Fragment stripped, case kept: what a regex reads, since `$match-case`
|
||||
/// is only meaningful on the original.
|
||||
raw: []const u8,
|
||||
host_start: u32,
|
||||
host_end: u32,
|
||||
|
||||
@@ -40,7 +43,7 @@ pub const Url = struct {
|
||||
/// Locates the hostname inside `url`: after "scheme://", up to the port,
|
||||
/// path, query or end. Offsets, not a slice, because the matcher needs to
|
||||
/// resume the pattern right where the hostname stops.
|
||||
pub fn init(url: []const u8) Url {
|
||||
pub fn init(url: []const u8, raw: []const u8) Url {
|
||||
var start: usize = 0;
|
||||
if (std.mem.indexOf(u8, url, "://")) |scheme| start = scheme + 3;
|
||||
|
||||
@@ -74,6 +77,7 @@ pub const Url = struct {
|
||||
|
||||
return .{
|
||||
.text = url,
|
||||
.raw = raw,
|
||||
.host_start = @intCast(start),
|
||||
.host_end = @intCast(host_end),
|
||||
};
|
||||
@@ -86,8 +90,8 @@ pub fn matches(filter: *const NetworkFilter, url: Url) bool {
|
||||
switch (filter.kind) {
|
||||
// Option-only filters ("$script,domain=x") match any URL.
|
||||
.any => return true,
|
||||
// Never indexed: there is no regex engine to run them with.
|
||||
.regex => return false,
|
||||
// uBO tests the raw URL; the case-insensitive flag is on the pattern.
|
||||
.regex => return filter.regex.?.matches(url.raw),
|
||||
.hostname, .plain, .wildcard => {},
|
||||
}
|
||||
|
||||
@@ -235,28 +239,28 @@ const testing = @import("../../testing.zig");
|
||||
|
||||
fn testMatch(arena: std.mem.Allocator, line: []const u8, url: []const u8) !bool {
|
||||
const filter = try NetworkFilter.parse(arena, line);
|
||||
return matches(&filter, .init(url));
|
||||
return matches(&filter, .init(url, url));
|
||||
}
|
||||
|
||||
test "adblock.pattern: hostname location" {
|
||||
var u: Url = .init("https://ads.example.com/x?y=1");
|
||||
var u: Url = .init("https://ads.example.com/x?y=1", "");
|
||||
try testing.expectString("ads.example.com", u.hostname());
|
||||
|
||||
u = .init("https://ads.example.com:8443/x");
|
||||
u = .init("https://ads.example.com:8443/x", "");
|
||||
try testing.expectString("ads.example.com", u.hostname());
|
||||
|
||||
u = .init("https://example.com");
|
||||
u = .init("https://example.com", "");
|
||||
try testing.expectString("example.com", u.hostname());
|
||||
|
||||
u = .init("http://[::1]:9222/json");
|
||||
u = .init("http://[::1]:9222/json", "");
|
||||
try testing.expectString("[::1]", u.hostname());
|
||||
|
||||
// Credentials are not part of the host.
|
||||
u = .init("https://user:pass@ads.example.com/x");
|
||||
u = .init("https://user:pass@ads.example.com/x", "");
|
||||
try testing.expectString("ads.example.com", u.hostname());
|
||||
|
||||
// A pattern is matched against whatever it is given, even a bare path.
|
||||
u = .init("/relative/path");
|
||||
u = .init("/relative/path", "");
|
||||
try testing.expectString("", u.hostname());
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user