diff --git a/build.zig b/build.zig index be9d47485..42a58eba2 100644 --- a/build.zig +++ b/build.zig @@ -125,6 +125,7 @@ pub fn build(b: *Build) !void { linkZenai(b, lightpanda_module); linkIsocline(b, lightpanda_module); linkSqlite(b, lightpanda_module, enable_csan, enable_tsan, orderfile != null); + linkPcre2(b, lightpanda_module, enable_csan, enable_tsan, orderfile != null); // Check compilation const check = b.step("check", "Check if lightpanda compiles"); @@ -486,6 +487,27 @@ fn linkSqlite(b: *Build, mod: *Build.Module, enable_csan: ?std.zig.SanitizeC, is mod.addImport("sqlite3", translate_c.createModule()); } +fn linkPcre2(b: *Build, mod: *Build.Module, enable_csan: ?std.zig.SanitizeC, is_tsan: bool, section: bool) void { + const dep = b.dependency("pcre2", .{ + .target = mod.resolved_target.?, + .optimize = mod.optimize.?, + .linkage = .static, + }); + + const lib = sectionize(dep.artifact("pcre2-8"), section); + lib.root_module.sanitize_c = enable_csan; + lib.root_module.sanitize_thread = is_tsan; + mod.linkLibrary(lib); + + const translate_c = b.addTranslateC(.{ + .root_source_file = lib.getEmittedIncludeTree().path(b, "pcre2.h"), + .target = mod.resolved_target.?, + .optimize = mod.optimize.?, + }); + translate_c.defineCMacro("PCRE2_CODE_UNIT_WIDTH", "8"); + mod.addImport("pcre2", translate_c.createModule()); +} + fn linkCurl(b: *Build, mod: *Build.Module, is_tsan: bool, section: bool) void { const target = mod.resolved_target.?; diff --git a/build.zig.zon b/build.zig.zon index 1b953147d..d0bf21fcb 100644 --- a/build.zig.zon +++ b/build.zig.zon @@ -43,6 +43,10 @@ .url = "git+https://github.com/arrufat/isocline#ec538faf435c616a6b38716f53980b5815c30f8a", .hash = "N-V-__8AAHhtEwBIqx5nOoiGo_FLAG8gpiVC6XzZn1teMKd0", }, + .pcre2 = .{ + .url = "https://github.com/PCRE2Project/pcre2/releases/download/pcre2-10.48/pcre2-10.48.tar.gz", + .hash = "pcre2-10.48.0-IZ6r69wnegD6fc08FgRTKVcmyp4Ux2aVefaVvsRndgEl", + }, }, .paths = .{""}, } diff --git a/src/network/HttpClient.zig b/src/network/HttpClient.zig index 54af81941..e1ba6683b 100644 --- a/src/network/HttpClient.zig +++ b/src/network/HttpClient.zig @@ -4390,12 +4390,27 @@ test "HttpClient: adblock verdicts apply per request" { \\||typed.example.com^$script \\||partied.example.com^$third-party \\||framed.example.com^$subdocument + \\/\/[a-z]{4}\.js$/$match-case,script ); try blocker.parse(&list); try blocker.build(); client.network.adblocker = blocker; defer client.network.adblocker = null; + // A regex filter reads the URL as requested: case kept, fragment gone. + try testing.expect(testIsUrlBlocked(&client, .{ + .url = "https://cdn.example.com/abcd.js", + .resource_type = .script, + })); + try testing.expect(testIsUrlBlocked(&client, .{ + .url = "https://cdn.example.com/abcd.js#v2", + .resource_type = .script, + })); + try testing.expect(!testIsUrlBlocked(&client, .{ + .url = "https://cdn.example.com/ABCD.js", + .resource_type = .script, + })); + try testing.expect(testIsUrlBlocked(&client, .{ .url = "https://ads.example.com/pixel.gif" })); // Hostnames are matched case-insensitively and without the port. try testing.expect(testIsUrlBlocked(&client, .{ .url = "https://SUB.ADS.EXAMPLE.COM:8443/x" })); diff --git a/src/network/adblock/AdBlocker.zig b/src/network/adblock/AdBlocker.zig index 44ebcb265..9699783aa 100644 --- a/src/network/adblock/AdBlocker.zig +++ b/src/network/adblock/AdBlocker.zig @@ -29,6 +29,7 @@ const Parser = @import("Parser.zig"); const Engine = @import("Engine.zig"); const HostnameTrie = @import("HostnameTrie.zig"); const NetworkFilter = @import("NetworkFilter.zig"); +const Regex = @import("Regex.zig"); const log = lp.log; @@ -47,6 +48,11 @@ arena: std.heap.ArenaAllocator, filters: std.ArrayList(NetworkFilter), /// Filled while parsing, consumed by `build`. badfilters: std.AutoHashMapUnmanaged(u64, void), +/// Every regex compiled for a filter, whether or not `build` kept the +/// filter; they are freed here, not through `filters`. +regexes: std.ArrayList(*const Regex), +/// What the regexes are compiled and run with; PCRE2 allocates through it. +regex_context: *Regex.Context, built: bool, trie: HostnameTrie, blocked: u32, @@ -64,7 +70,7 @@ exceptions: Engine, /// Rules that reached a trie or an index, across every list parsed so far. rules_loaded: usize, /// Rules the lists carry and we do not apply, across those same lists: the -/// ones the parser could not read, plus the ones no matcher can express. +/// ones the parser could not read, regex literals PCRE2 rejects included. rules_skipped: usize, /// Domain-scoped element-hiding rules and cosmetic-realm exceptions, across /// those same lists. Not ours to apply, but not rules we failed at either, so @@ -77,6 +83,8 @@ rules_cosmetic: usize, const LINE_MAX = 8 * 1024; pub fn init(allocator: Allocator) Allocator.Error!AdBlocker { + const regex_context: *Regex.Context = try .init(allocator); + errdefer regex_context.deinit(); var trie: HostnameTrie = try .init(allocator); errdefer trie.deinit(allocator); const blocked = try trie.createTrie(allocator); @@ -89,6 +97,8 @@ pub fn init(allocator: Allocator) Allocator.Error!AdBlocker { .arena = std.heap.ArenaAllocator.init(allocator), .filters = .empty, .badfilters = .empty, + .regexes = .empty, + .regex_context = regex_context, .built = false, .trie = trie, .blocked = blocked, @@ -109,6 +119,9 @@ pub fn deinit(self: *AdBlocker) void { self.blocking_important.deinit(self.allocator); self.exceptions.deinit(self.allocator); self.badfilters.deinit(self.allocator); + for (self.regexes.items) |regex| regex.deinit(); + self.regexes.deinit(self.allocator); + self.regex_context.deinit(); self.filters.deinit(self.allocator); self.trie.deinit(self.allocator); self.arena.deinit(); @@ -180,7 +193,7 @@ pub fn parse(self: *AdBlocker, reader: *Io.Reader) !void { // both of which the next call reuses, so what we keep is copied out. defer _ = scratch_instance.reset(.retain_capacity); - const filter = switch (item) { + var filter = switch (item) { // The parser already counted it; all we want is whether it was an // exception, in which case its hostname stops being blockable. .dropped => |dropped| { @@ -201,11 +214,6 @@ pub fn parse(self: *AdBlocker, reader: *Io.Reader) !void { self.rules_cosmetic += 1; continue; } - if (!isSupported(&filter)) { - self.rules_skipped += 1; - continue; - } - if (filter.badfilter) { // It cancels a rule that may not have been read yet, so it can // only be resolved once every list is in. It is not a rule we @@ -215,6 +223,22 @@ pub fn parse(self: *AdBlocker, reader: *Io.Reader) !void { continue; } + if (filter.kind == .regex) { + const body = filter.pattern[1 .. filter.pattern.len - 1]; + const compiled = Regex.compile(self.regex_context, body, !filter.match_case) catch |err| switch (err) { + error.OutOfMemory => return error.OutOfMemory, + error.InvalidRegex => { + self.rules_skipped += 1; + continue; + }, + }; + errdefer compiled.deinit(); + const regex = try arena.create(Regex); + regex.* = compiled; + try self.regexes.append(self.allocator, regex); + filter.regex = regex; + } + try self.filters.append(self.allocator, try dupe(arena, &filter)); self.rules_loaded += 1; } @@ -361,12 +385,6 @@ pub fn match(self: *const AdBlocker, request: *const Request) Verdict { return .blocked; } -/// Whether we can evaluate this filter at all. -fn isSupported(filter: *const NetworkFilter) bool { - // No regex engine to run them with. - return filter.kind != .regex; -} - /// Whether the filter's whole effect is "every request to this hostname and /// its subdomains", which is all a trie can express. fn isWholeHostname(filter: *const NetworkFilter) bool { @@ -451,11 +469,8 @@ fn expectVerdict( source: []const u8, kind: ResourceTypes, ) !void { - // `match` takes a lowercased URL, as the real caller hands it; patterns - // are lowercased too, so a rule spelled "/embed/C-iDzdvIg1Y" still lands. var buf: [512]u8 = undefined; - const lowered = std.ascii.lowerString(buf[0..url.len], url); - const request: Request = .init(lowered, source, kind); + const request: Request = Request.init(url, &buf, source, kind).?; try testing.expectEqual(expected, blocker.match(&request)); } @@ -591,7 +606,8 @@ test "adblock.AdBlocker: tokens past the request buffer still match" { // 140 tokens of query noise push the filter's token ("utm", its rarest) // past what the request holds; the engine walks the rest of the URL. const noise = "https://example.com/?" ++ "a=1&" ** 70; - const overflowing: Request = .init(noise ++ "utm_tracker=1", "a.com", script); + var buf: [512]u8 = undefined; + const overflowing: Request = Request.init(noise ++ "utm_tracker=1", &buf, "a.com", script).?; try testing.expect(overflowing.tail.len != 0); try testing.expectEqual(.blocked, blocker.match(&overflowing)); // A token past the buffer finds its bucket, but the pattern does not fit. @@ -608,7 +624,7 @@ test "adblock.AdBlocker: cosmetic-realm rules are not skipped rules" { \\@@||example.com^$generichide \\@@||example.com^$elemhide \\example.com##.ad-banner - \\/regex-we-cannot-run/ + \\/regex-we-cannot-run(/ ); try testing.expectEqual(1, blocker.rules_loaded); @@ -616,6 +632,65 @@ test "adblock.AdBlocker: cosmetic-realm rules are not skipped rules" { try testing.expectEqual(3, blocker.rules_cosmetic); } +test "adblock.AdBlocker: the regex rules from EasyList" { + var blocker: AdBlocker = try .init(testing.allocator); + defer blocker.deinit(); + + try testLoad(&blocker, + \\/\/[0-9a-f]{32}\/invoke\.js/$script,third-party + \\/^https?:\/\/[0-9a-z]{5,}\.com\/.*/$script,third-party,xmlhttprequest,domain=dood.to + \\@@/\/invoke\.js/$domain=trusted.com + \\/\/[a-z]{4}\.js$/$match-case,script + ); + try testing.expectEqual(4, blocker.rules_loaded); + try testing.expectEqual(0, blocker.rules_skipped); + // Each carries a literal every match has, so none rides the fallback. + try testing.expectEqual(0, blocker.blocking.fallback.len); + try testing.expectEqual(0, blocker.exceptions.fallback.len); + + const invoke = "https://host.com/0123456789abcdef0123456789abcdef/invoke.js"; + try expectVerdict(&blocker, .blocked, invoke, "site.com", script); + try expectVerdict(&blocker, .blocked, "https://HOST.com/0123456789ABCDEF0123456789abcdef/invoke.js", "site.com", script); + try expectVerdict(&blocker, .none, invoke, "host.com", script); + try expectVerdict(&blocker, .none, invoke, "site.com", image); + try expectVerdict(&blocker, .none, "https://host.com/0123456789abcdef0123456789abcde/invoke.js", "site.com", script); + try expectVerdict(&blocker, .allowed, invoke, "trusted.com", script); + + try expectVerdict(&blocker, .blocked, "https://abcde.com/x", "dood.to", xhr); + try expectVerdict(&blocker, .none, "https://abcde.com/x", "other.to", xhr); + try expectVerdict(&blocker, .none, "https://abcd.com/x", "dood.to", xhr); + + // `$match-case` reads the URL as requested, not the lowercased copy. + try expectVerdict(&blocker, .blocked, "https://x.com/abcd.js", "x.com", script); + try expectVerdict(&blocker, .none, "https://x.com/ABCD.js", "x.com", script); +} + +test "adblock.AdBlocker: a regex is found under every token it may match" { + var blocker: AdBlocker = try .init(testing.allocator); + defer blocker.deinit(); + + try testLoad(&blocker, + \\/\/ads(\/?x|\/y)/$script + ); + try expectVerdict(&blocker, .blocked, "https://example.com/adsx", "example.com", script); + try expectVerdict(&blocker, .blocked, "https://example.com/ads/x", "example.com", script); + try expectVerdict(&blocker, .blocked, "https://example.com/ads/y", "example.com", script); + try expectVerdict(&blocker, .none, "https://example.com/ads/z", "example.com", script); +} + +test "adblock.AdBlocker: $badfilter removes a regex rule" { + var blocker: AdBlocker = try .init(testing.allocator); + defer blocker.deinit(); + + try testLoad(&blocker, + \\/\/invoke\.js/$script + \\/\/invoke\.js/$script,badfilter + ); + try testing.expectEqual(0, blocker.rules_loaded); + try testing.expectEqual(2, blocker.rules_skipped); + try expectVerdict(&blocker, .none, "https://host.com/invoke.js", "site.com", script); +} + test "adblock.AdBlocker: verdict precedence" { var blocker: AdBlocker = try .init(testing.allocator); defer blocker.deinit(); diff --git a/src/network/adblock/Engine.zig b/src/network/adblock/Engine.zig index 8a6159070..ff19e3684 100644 --- a/src/network/adblock/Engine.zig +++ b/src/network/adblock/Engine.zig @@ -95,14 +95,18 @@ pub const Request = struct { source: [SOURCE_MAX]u8, }; - /// `url` must already be lowercased and fragment-free; `source_hostname` - /// may be empty when there is no document context. + /// `buf` backs the lowercased URL; null when `url` is longer than it. + /// `source_hostname` may be empty when there is no document context. pub fn init( url: []const u8, + buf: []u8, source_hostname: []const u8, kind: NetworkFilter.ResourceTypes, - ) Request { - const parsed: pattern.Url = .init(url); + ) ?Request { + // No request URL carries a fragment onto the wire. + const raw = URL.stripFragment(url); + const lowered = lowercase(raw, buf) orelse return null; + const parsed: pattern.Url = .init(lowered, raw); const source = if (source_hostname.len == 0) parsed.hostname() else source_hostname; var request: Request = .{ .url = parsed, @@ -113,19 +117,18 @@ pub const Request = struct { .tokens_len = 0, .tail = "", }; - var it: Tokens = .{ .text = url }; + var it: Tokens = .{ .text = lowered }; while (request.tokens_len < request.tokens_buf.len) { const token = it.next() orelse break; request.tokens_buf[request.tokens_len] = token; request.tokens_len += 1; } - request.tail = url[it.i..]; + request.tail = lowered[it.i..]; return request; } pub fn fromHttp(transfer: *const HttpClient.Transfer, buffers: *Buffers) ?Request { const req = &transfer.req; - const url = normalizeUrl(req.url, &buffers.url) orelse return null; var owner: ?*const HttpClient.Owner = transfer.owner; var subframe = false; @@ -148,27 +151,23 @@ pub const Request = struct { .worker => .{ .script = true }, }; - return .init(url, source, resource_type); + return .init(req.url, &buffers.url, source, resource_type); } inline fn tokens(self: *const Request) []const u32 { return self.tokens_buf[0..self.tokens_len]; } - /// Lowercases `url` into `buf` with its fragment stripped; patterns are - /// stored lowercased, and no request URL carries a fragment onto the wire. - fn normalizeUrl(url: []const u8, buf: []u8) ?[]const u8 { - const end = std.mem.indexOfScalar(u8, url, '#') orelse url.len; - const trimmed = url[0..end]; - - const upper = for (trimmed, 0..) |c, i| { + /// Lowercases `url` into `buf`, as patterns are stored lowercased. + fn lowercase(url: []const u8, buf: []u8) ?[]const u8 { + const upper = for (url, 0..) |c, i| { if (std.ascii.isUpper(c)) break i; - } else return trimmed; + } else return url; - if (trimmed.len > buf.len) return null; - const out = buf[0..trimmed.len]; - @memcpy(out[0..upper], trimmed[0..upper]); - _ = std.ascii.lowerString(out[upper..], trimmed[upper..]); + if (url.len > buf.len) return null; + const out = buf[0..url.len]; + @memcpy(out[0..upper], url[0..upper]); + _ = std.ascii.lowerString(out[upper..], url[upper..]); return out; } }; @@ -326,11 +325,20 @@ fn collectTokens(filter: *const NetworkFilter, buf: []u32) []u32 { } } - // A /regex/ literal is never indexed (nothing can run it), and `.any` - // has no pattern at all. - if (filter.kind == .regex or filter.pattern.len == 0) return buf[0..n]; + // A /regex/ literal has no hostname, so `n` is still 0 here. + if (filter.kind == .regex) return regexTokens(filter.pattern[1 .. filter.pattern.len - 1], buf); + // `.any` has no pattern at all. + if (filter.pattern.len == 0) return buf[0..n]; - const text = filter.pattern; + const left_anchored = filter.left_anchor or filter.hostname_anchor; + return buf[0..boundedTokens(filter.pattern, left_anchored, filter.right_anchor, buf, n)]; +} + +/// Appends the tokens of `text` that any URL matching it carries whole: the +/// alphanumeric runs no `*` may extend, and that the open ends of the text +/// may not extend either unless anchored. Returns the new count. +fn boundedTokens(text: []const u8, left_anchored: bool, right_anchored: bool, buf: []u32, from: usize) usize { + var n = from; var i: usize = 0; while (i < text.len) { if (!isTokenChar(text[i])) { @@ -340,22 +348,317 @@ fn collectTokens(filter: *const NetworkFilter, buf: []u32) []u32 { const start = i; while (i < text.len and isTokenChar(text[i])) i += 1; - const left_bounded = if (start == 0) - filter.left_anchor or filter.hostname_anchor - else - text[start - 1] != '*'; - const right_bounded = if (i == text.len) filter.right_anchor else text[i] != '*'; + const left_bounded = if (start == 0) left_anchored else text[start - 1] != '*'; + const right_bounded = if (i == text.len) right_anchored else text[i] != '*'; if (left_bounded and right_bounded) { - if (n == buf.len) return buf[0..n]; + if (n == buf.len) return n; buf[n] = hash(text[start..i]); n += 1; } } - - return buf[0..n]; + return n; } +/// The tokens a regex is sure to have wherever it matches, uBO's +/// `tokenizableStrFromRegex`: the pattern is flattened into a plain pattern +/// where literal token characters stay and everything else becomes a marker +/// saying only whether it could be one. Anything the flattening does not +/// follow yields no token at all, which is never wrong: the filter then +/// rides the fallback bucket. +fn regexTokens(source: []const u8, buf: []u32) []u32 { + // A pattern's shape is never longer than the pattern. + var shape_buf: [8 * 1024]u8 = undefined; + if (source.len > shape_buf.len) return buf[0..0]; + var shape: RegexShape = .{ .source = source, .out = &shape_buf }; + const text = shape.flatten() catch return buf[0..0]; + // Anchors are in the shape itself; its ends are open. + return buf[0..boundedTokens(text, false, false, buf, 0)]; +} + +const RegexShape = struct { + source: []const u8, + out: []u8, + i: usize = 0, + n: usize = 0, + + /// Whatever matches here is not a token character: a non-token literal, + /// an anchor, `\b`. + const not_token = 0x00; + /// Whatever matches here may be a token character: `.`, `[a-z]`, `\d`, a + /// quantified literal. Spelled as a plain pattern's wildcard so the shape + /// reads as one. + const maybe_token = '*'; + // The same two for a stretch that may match nothing at all; resolved + // once both neighbours are known. + const not_token_optional = 0x02; + const maybe_token_optional = 0x03; + + const Error = error{Unsupported}; + + fn flatten(self: *RegexShape) Error![]const u8 { + try self.alternation(false); + if (self.i != self.source.len) return error.Unsupported; + self.resolveOptional(); + return self.out[0..self.n]; + } + + /// Parses alternatives up to the closing parenthesis (or the end). More + /// than one collapses to two markers: all that is sure about `a|bc` is + /// how it starts and ends. + fn alternation(self: *RegexShape, nested: bool) Error!void { + const start = self.n; + var branches: usize = 1; + var branch_start = start; + var first = false; + var last = false; + while (true) { + try self.sequence(nested); + const branch = self.out[branch_start..self.n]; + first = first or startsTokenish(branch); + last = last or endsTokenish(branch); + if (self.i == self.source.len or self.source[self.i] != '|') break; + self.i += 1; + branches += 1; + branch_start = self.n; + } + if (branches == 1) return; + self.n = start; + self.emit(if (first) maybe_token else not_token); + self.emit(if (last) maybe_token else not_token); + } + + fn sequence(self: *RegexShape, nested: bool) Error!void { + while (self.i < self.source.len) { + const atom_start = self.n; + const c = self.source[self.i]; + switch (c) { + '|' => return, + ')' => { + if (!nested) return error.Unsupported; + return; + }, + '(' => try self.group(), + '[' => try self.class(), + '\\' => try self.escape(), + '.' => { + self.i += 1; + self.emit(maybe_token); + }, + '^', '$' => { + self.i += 1; + self.emit(not_token); + }, + '*', '+', '?' => return error.Unsupported, + else => { + self.i += 1; + self.emitLiteral(c); + }, + } + try self.quantifier(atom_start); + } + } + + fn group(self: *RegexShape) Error!void { + const start = self.n; + self.i += 1; + var lookaround: enum { none, positive, negative } = .none; + if (self.i < self.source.len and self.source[self.i] == '?') { + self.i += 1; + const kind = self.take() orelse return error.Unsupported; + switch (kind) { + ':' => {}, + '=' => lookaround = .positive, + '!' => lookaround = .negative, + '<' => { + const next = self.take() orelse return error.Unsupported; + switch (next) { + '=' => lookaround = .positive, + '!' => lookaround = .negative, + else => { + // A named group is a plain group with a label. + const close = std.mem.indexOfScalarPos(u8, self.source, self.i, '>') orelse return error.Unsupported; + self.i = close + 1; + }, + } + }, + else => return error.Unsupported, + } + } + try self.alternation(true); + if (self.take() != ')') return error.Unsupported; + switch (lookaround) { + .none => {}, + // Consumes nothing, so the neighbours touch; what it asserts + // could still be anything, and that is all a token may rely on. + .positive => { + self.n = start; + self.emit(maybe_token); + }, + // Consumes nothing and rules text out: the neighbours touch. + .negative => self.n = start, + } + } + + /// `[...]` is one character; a token character can come out of it if any + /// member is one, or if it is negated. + fn class(self: *RegexShape) Error!void { + self.i += 1; + var maybe = false; + if (self.i < self.source.len and self.source[self.i] == '^') { + self.i += 1; + maybe = true; + } + var first = true; + while (true) { + const c = self.take() orelse return error.Unsupported; + if (c == ']' and !first) break; + first = false; + if (c == '\\') { + const e = self.take() orelse return error.Unsupported; + switch (e) { + 'd', 'D', 'w', 'W', 's', 'S' => maybe = true, + 'b', 'n', 'r', 't', 'f', 'v' => {}, + else => if (std.ascii.isAlphanumeric(e)) return error.Unsupported, + } + continue; + } + // A range is assumed to reach token characters. + if (c == '-' or std.ascii.isAlphanumeric(c)) maybe = true; + } + self.emit(if (maybe) maybe_token else not_token); + } + + fn escape(self: *RegexShape) Error!void { + self.i += 1; + const e = self.take() orelse return error.Unsupported; + switch (e) { + 'd', 'D', 'w', 'W', 's', 'S', 'B' => self.emit(maybe_token), + 'b', 'n', 'r', 't', 'f', 'v' => self.emit(not_token), + // Code points, backreferences, properties: not worth following. + 'x', 'u', 'c', 'k', 'p', 'P', '0'...'9' => return error.Unsupported, + // Anything else escaped is itself, as JavaScript reads it. + else => self.emitLiteral(e), + } + } + + /// Applies a quantifier, if one follows, to what was just emitted. Only + /// the first and last character classes survive a repeat: `ab+` may match + /// "abbb", and its token is not "ab". Whether the atom may be absent is + /// all that matters beyond that. + fn quantifier(self: *RegexShape, atom_start: usize) Error!void { + if (self.i == self.source.len) return; + var optional = true; + switch (self.source[self.i]) { + '*', '?' => self.i += 1, + '+' => { + self.i += 1; + optional = false; + }, + '{' => { + const close = std.mem.indexOfScalarPos(u8, self.source, self.i, '}') orelse return; + const body = self.source[self.i + 1 .. close]; + const comma = std.mem.indexOfScalar(u8, body, ',') orelse body.len; + // Not a quantifier at all: JavaScript reads the `{` literally. + const min = std.fmt.parseUnsigned(usize, body[0..comma], 10) catch return; + if (comma + 1 < body.len) _ = std.fmt.parseUnsigned(usize, body[comma + 1 ..], 10) catch return; + optional = min == 0; + self.i = close + 1; + }, + else => return, + } + // A lazy `?` changes nothing about what can match. + if (self.i < self.source.len and self.source[self.i] == '?') self.i += 1; + + const atom = self.out[atom_start..self.n]; + const first = startsTokenish(atom); + const last = endsTokenish(atom); + self.n = atom_start; + if (!optional) { + self.emit(if (first) maybe_token else not_token); + self.emit(if (last) maybe_token else not_token); + } else { + self.emit(if (first) maybe_token_optional else not_token_optional); + self.emit(if (last) maybe_token_optional else not_token_optional); + } + } + + /// An optional stretch may vanish, letting its neighbours touch: it + /// resolves to markers that carry whichever side could be a token + /// character, so no run is read as bounded by something that may be + /// gone. + fn resolveOptional(self: *RegexShape) void { + var i: usize = 0; + while (i < self.n) { + if (!isOptional(self.out[i])) { + i += 1; + continue; + } + var end = i; + while (end < self.n and isOptional(self.out[end])) end += 1; + const left = self.out[0..i]; + const middle = self.out[i..end]; + const right = self.out[end..self.n]; + const head: u8 = if (startsTokenish(right) or startsTokenish(middle)) maybe_token else not_token; + const tail: u8 = if (endsTokenish(left) or endsTokenish(middle)) maybe_token else not_token; + // Quantifiers emit markers in pairs, so a stretch is never shorter + // than what replaces it. + std.mem.copyForwards(u8, self.out[i + 2 .. self.n - (middle.len - 2)], right); + self.out[i] = head; + self.out[i + 1] = tail; + self.n -= middle.len - 2; + i += 2; + } + } + + fn take(self: *RegexShape) ?u8 { + if (self.i == self.source.len) return null; + defer self.i += 1; + return self.source[self.i]; + } + + fn emit(self: *RegexShape, c: u8) void { + self.out[self.n] = c; + self.n += 1; + } + + /// A literal is kept only as far as tokens care: a token character, + /// lowercased like the URL it is looked up in, or the fact that it is not + /// one. + fn emitLiteral(self: *RegexShape, c: u8) void { + self.emit(if (std.ascii.isAlphanumeric(c)) std.ascii.toLower(c) else not_token); + } + + fn isOptional(c: u8) bool { + return c == not_token_optional or c == maybe_token_optional; + } + + fn isTokenish(c: u8) bool { + return c == maybe_token or c == maybe_token_optional or isTokenChar(c); + } + + /// Whether what `s` matches could start with a token character. An + /// optional non-token stretch may be absent, so whatever follows it + /// answers instead. + fn startsTokenish(s: []const u8) bool { + for (s) |c| { + if (isTokenish(c)) return true; + if (c != not_token_optional) return false; + } + return false; + } + + fn endsTokenish(s: []const u8) bool { + var i = s.len; + while (i > 0) { + i -= 1; + if (isTokenish(s[i])) return true; + if (s[i] != not_token_optional) return false; + } + return false; + } +}; + const testing = @import("../../testing.zig"); fn tokensOf(arena: Allocator, line: []const u8, buf: []u32) ![]u32 { @@ -370,18 +673,80 @@ fn contains(tokens: []const u32, token: []const u8) bool { return false; } +test "adblock.Engine: regex filters yield the tokens every match carries" { + var arena_state = std.heap.ArenaAllocator.init(testing.allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + var buf: [TOKENS_MAX]u32 = undefined; + + // Literal runs between literal non-token characters; the open end of an + // unanchored pattern bounds nothing. + var tokens = try tokensOf(arena, "/\\/[0-9a-f]{32}\\/invoke\\.js/", &buf); + try testing.expectEqual(1, tokens.len); + try testing.expect(contains(tokens, "invoke")); + + // Anchors bound; `https?` may be either, so "http" is no token. + tokens = try tokensOf(arena, "/^https?:\\/\\/[0-9a-z]{5,}\\.com\\/.*/", &buf); + try testing.expectEqual(1, tokens.len); + try testing.expect(contains(tokens, "com")); + tokens = try tokensOf(arena, "/[a-z]{2,}\\.gif$/", &buf); + try testing.expectEqual(1, tokens.len); + try testing.expect(contains(tokens, "gif")); + tokens = try tokensOf(arena, "/about:blank.*/", &buf); + try testing.expectEqual(0, tokens.len); + + // `\b` bounds, as uBO reads it: `/\bads\b/` is "ads", not "bads". + tokens = try tokensOf(arena, "/\\bads\\b/", &buf); + try testing.expect(contains(tokens, "ads")); + + // Hashed lowercased, like the URL it is looked up in. + tokens = try tokensOf(arena, "/\\/Ads\\//$match-case", &buf); + try testing.expect(contains(tokens, "ads")); + + // An optional stretch may vanish and glue its neighbours: "adsbanner". + tokens = try tokensOf(arena, "/\\/ads\\/?banner\\//", &buf); + try testing.expectEqual(0, tokens.len); + // ... also from inside a group: "adsx" is a match. + tokens = try tokensOf(arena, "/\\/ads(\\/?x|\\/y)/", &buf); + try testing.expectEqual(0, tokens.len); + // A repeat is not the literal it repeats. + tokens = try tokensOf(arena, "/\\/ab+c\\//", &buf); + try testing.expectEqual(0, tokens.len); + + // Alternation and the text under a quantified group are uncertain. + tokens = try tokensOf(arena, "/^https?:\\/\\/(35|104)\\.(\\d){1,3}\\//", &buf); + try testing.expectEqual(0, tokens.len); + // ... but a group with one branch is transparent. + tokens = try tokensOf(arena, "/\\/(?:ads)\\//", &buf); + try testing.expect(contains(tokens, "ads")); + + // A negative lookaround consumes nothing; a positive one is not + // trusted to spell anything. + tokens = try tokensOf(arena, "/\\/(?!ads)banner\\//", &buf); + try testing.expect(contains(tokens, "banner")); + tokens = try tokensOf(arena, "/\\/(?=ads)ads\\//", &buf); + try testing.expectEqual(0, tokens.len); + + // What is not followed yields nothing rather than something wrong. + tokens = try tokensOf(arena, "/\\/\\x41ds\\//", &buf); + try testing.expectEqual(0, tokens.len); + tokens = try tokensOf(arena, "/\\/(ads\\//", &buf); + try testing.expectEqual(0, tokens.len); +} + test "adblock.Engine: a request keeps its first tokens, the rest as tail" { const max = Request.URL_TOKENS_MAX; const kind: NetworkFilter.ResourceTypes = .{ .script = true }; // Exactly as many tokens as the buffer holds: nothing is left to walk... const full = "x/" ** (max - 1) ++ "x"; - var request: Request = .init(full, "", kind); + var buf: [512]u8 = undefined; + var request: Request = Request.init(full, &buf, "", kind).?; try testing.expectEqual(max, request.tokens_len); try testing.expectEqual(0, request.tail.len); // ...one more, and only that one is in the tail. - request = .init(full ++ "/y", "", kind); + request = Request.init(full ++ "/y", &buf, "", kind).?; try testing.expectEqual(max, request.tokens_len); try testing.expectString("/y", request.tail); var it: Tokens = .{ .text = request.tail }; @@ -389,7 +754,7 @@ test "adblock.Engine: a request keeps its first tokens, the rest as tail" { try testing.expect(it.next() == null); // Fewer than the buffer holds: the tail is empty. - request = .init("https://example.com/a", "", kind); + request = Request.init("https://example.com/a", &buf, "", kind).?; try testing.expectEqual(4, request.tokens_len); try testing.expectEqual(0, request.tail.len); } diff --git a/src/network/adblock/NetworkFilter.zig b/src/network/adblock/NetworkFilter.zig index 0cae510b6..ab1ef8e9d 100644 --- a/src/network/adblock/NetworkFilter.zig +++ b/src/network/adblock/NetworkFilter.zig @@ -18,6 +18,7 @@ const std = @import("std"); const domain = @import("domain.zig"); +const Regex = @import("Regex.zig"); const NetworkFilter = @This(); @@ -38,6 +39,9 @@ exception: bool = false, important: bool = false, badfilter: bool = false, match_case: bool = false, +/// The compiled `.regex` pattern, set by the blocker once it has read the +/// literal; the parser never runs one. +regex: ?*const Regex = null, first_party: bool = true, third_party: bool = true, hostname_anchor: bool = false, diff --git a/src/network/adblock/Regex.zig b/src/network/adblock/Regex.zig new file mode 100644 index 000000000..76806f15d --- /dev/null +++ b/src/network/adblock/Regex.zig @@ -0,0 +1,224 @@ +// Copyright (C) 2023-2026 Lightpanda (Selecy SAS) +// +// Francis Bouvier +// Pierre Tachoire +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +//! A compiled `/regex/` filter body. Filter lists write them in JavaScript +//! `RegExp` syntax and uBO runs them with `new RegExp(src, 'i')` against the +//! raw request URL (no flag under `$match-case`); PCRE2 reads that syntax +//! as-is, escapes like `\/` included. +//! +//! A compiled pattern and its `Context` are never modified after `compile`, +//! so one `Regex` can be shared by every HTTP client thread; the per-call +//! match data is what PCRE2 requires to be private. + +const std = @import("std"); +const lp = @import("lightpanda"); +const pcre2 = @import("pcre2"); + +const Allocator = std.mem.Allocator; + +const log = lp.log; + +const Regex = @This(); + +code: *pcre2.pcre2_code_8, +context: *const Context, + +pub const Error = error{ InvalidRegex, OutOfMemory }; + +/// What every `Regex` compiled through it shares: the allocator PCRE2 draws +/// from, and the compile and match settings. Outlives the regexes. +/// +/// PCRE2 would happily use libc's malloc; it is handed the blocker's +/// allocator so that a compiled pattern nobody freed fails a test the way +/// any other leak does. +pub const Context = struct { + allocator: Allocator, + general: *pcre2.pcre2_general_context_8, + compile_context: *pcre2.pcre2_compile_context_8, + match_context: *pcre2.pcre2_match_context_8, + + // A pattern from a list that backtracks this much on one URL is broken, + // not slow; giving up costs a false negative on that request, nothing + // more. + const MATCH_LIMIT = 100_000; + const DEPTH_LIMIT = 10_000; + + pub fn init(allocator: Allocator) Allocator.Error!*Context { + const self = try allocator.create(Context); + errdefer allocator.destroy(self); + self.allocator = allocator; + + // PCRE2 hands the allocator back to the callbacks by address, so the + // context has to be at its final one before anything is allocated + // through it. + const general = pcre2.pcre2_general_context_create_8(cMalloc, cFree, &self.allocator) orelse return error.OutOfMemory; + errdefer pcre2.pcre2_general_context_free_8(general); + + const compile_context = pcre2.pcre2_compile_context_create_8(general) orelse return error.OutOfMemory; + errdefer pcre2.pcre2_compile_context_free_8(compile_context); + // JavaScript without the `u` flag reads an unknown escape as the + // literal character, and that is the mode uBO compiles filters in. + _ = pcre2.pcre2_set_compile_extra_options_8(compile_context, pcre2.PCRE2_EXTRA_BAD_ESCAPE_IS_LITERAL); + + const match_context = pcre2.pcre2_match_context_create_8(general) orelse return error.OutOfMemory; + _ = pcre2.pcre2_set_match_limit_8(match_context, MATCH_LIMIT); + _ = pcre2.pcre2_set_depth_limit_8(match_context, DEPTH_LIMIT); + + self.general = general; + self.compile_context = compile_context; + self.match_context = match_context; + return self; + } + + pub fn deinit(self: *Context) void { + pcre2.pcre2_match_context_free_8(self.match_context); + pcre2.pcre2_compile_context_free_8(self.compile_context); + pcre2.pcre2_general_context_free_8(self.general); + self.allocator.destroy(self); + } + + // PCRE2 frees without a size, so every block carries its own in a + // header that keeps the payload at malloc's alignment. + const HEADER = 16; + const alignment: std.mem.Alignment = .fromByteUnits(HEADER); + + fn cMalloc(size: usize, data: ?*anyopaque) callconv(.c) ?*anyopaque { + const allocator: *const Allocator = @ptrCast(@alignCast(data.?)); + const total = std.math.add(usize, size, HEADER) catch return null; + const block = allocator.alignedAlloc(u8, alignment, total) catch return null; + std.mem.writeInt(usize, block[0..@sizeOf(usize)], total, .little); + return block.ptr + HEADER; + } + + fn cFree(ptr: ?*anyopaque, data: ?*anyopaque) callconv(.c) void { + const payload = ptr orelse return; + const allocator: *const Allocator = @ptrCast(@alignCast(data.?)); + const base: [*]align(HEADER) u8 = @ptrCast(@alignCast(@as([*]u8, @ptrCast(payload)) - HEADER)); + const total = std.mem.readInt(usize, base[0..@sizeOf(usize)], .little); + allocator.free(base[0..total]); + } +}; + +pub fn compile(context: *const Context, pattern: []const u8, case_insensitive: bool) Error!Regex { + const options: u32 = if (case_insensitive) pcre2.PCRE2_CASELESS else 0; + var err_code: c_int = 0; + var err_offset: usize = 0; + const code = pcre2.pcre2_compile_8( + pattern.ptr, + pattern.len, + options, + &err_code, + &err_offset, + context.compile_context, + ) orelse { + // A failed allocation is ours, not the pattern's. + if (err_code == pcre2.PCRE2_ERROR_HEAP_FAILED) return error.OutOfMemory; + var buf: [256]u8 = undefined; + const len = pcre2.pcre2_get_error_message_8(err_code, &buf, buf.len); + const message: []const u8 = if (len < 0) "unknown error" else buf[0..@intCast(len)]; + log.debug(.app, "adblock regex rejected", .{ + .pattern = pattern, + .err = message, + .offset = err_offset, + }); + return error.InvalidRegex; + }; + return .{ .code = code, .context = context }; +} + +pub fn deinit(self: Regex) void { + pcre2.pcre2_code_free_8(self.code); +} + +// What one match allocates: its match data and the 20KB of backtracking +// frames PCRE2 starts with, which only a deeply nested pattern outgrows. +const MATCH_SCRATCH = 24 * 1024; + +/// Whether the pattern matches anywhere in `text`, as `RegExp.test` would +/// answer. A match that hits the backtracking limits counts as no match. +pub fn matches(self: Regex, text: []const u8) bool { + // This runs per request; the scratch keeps the common case off the heap. + var scratch = std.heap.stackFallback(MATCH_SCRATCH, self.context.allocator); + var allocator = scratch.get(); + const general = pcre2.pcre2_general_context_create_8(Context.cMalloc, Context.cFree, &allocator) orelse return false; + defer pcre2.pcre2_general_context_free_8(general); + + // One pair is the whole-match span, all a test needs; capture groups in + // the pattern are simply not recorded. + const match_data = pcre2.pcre2_match_data_create_8(1, general) orelse return false; + defer pcre2.pcre2_match_data_free_8(match_data); + + const rc = pcre2.pcre2_match_8(self.code, text.ptr, text.len, 0, 0, match_data, self.context.match_context); + return rc >= 0; +} + +const testing = @import("../../testing.zig"); + +test "adblock.Regex: JavaScript escapes and unanchored search" { + const context: *Context = try .init(testing.allocator); + defer context.deinit(); + + const regex = try Regex.compile(context, "^https?:\\/\\/[0-9a-z]{5,}\\.com\\/.*", true); + defer regex.deinit(); + + try testing.expect(regex.matches("https://abcde.com/x")); + try testing.expect(regex.matches("HTTPS://ABCDE.COM/X")); + try testing.expect(!regex.matches("https://abcd.com/x")); + try testing.expect(!regex.matches("https://abcde.org/x")); + + const invoke = try Regex.compile(context, "\\/[0-9a-f]{32}\\/invoke\\.js", true); + defer invoke.deinit(); + try testing.expect(invoke.matches("https://host.com/0123456789abcdef0123456789abcdef/invoke.js")); + try testing.expect(!invoke.matches("https://host.com/0123456789abcdef0123456789abcde/invoke.js")); + + const dash = try Regex.compile(context, "[a-z\\-]+\\?s=", true); + defer dash.deinit(); + try testing.expect(dash.matches("https://x.com/a-b?s=1")); + try testing.expect(!dash.matches("https://x.com/?s=1")); +} + +test "adblock.Regex: $match-case keeps the case" { + const context: *Context = try .init(testing.allocator); + defer context.deinit(); + + const exact = try Regex.compile(context, "\\/[a-z0-9]{12}\\/[a-zA-Z0-9]{20,}$", false); + defer exact.deinit(); + try testing.expect(exact.matches("https://x.com/abcdef123456/aBcDeFgHiJkLmNoPqRsTuV")); + try testing.expect(!exact.matches("https://x.com/ABCDEF123456/aBcDeFgHiJkLmNoPqRsTuV")); +} + +test "adblock.Regex: invalid patterns are errors, runaway ones no match" { + const context: *Context = try .init(testing.allocator); + defer context.deinit(); + + try testing.expectError(error.InvalidRegex, Regex.compile(context, "(", true)); + try testing.expectError(error.InvalidRegex, Regex.compile(context, "a{2,1}", true)); + + // An unknown alphanumeric escape is the literal, as in JavaScript. + const literal = try Regex.compile(context, "\\q", true); + defer literal.deinit(); + try testing.expect(literal.matches("https://x.com/q")); + + // Exponential backtracking stops at the match limit instead of stalling + // the request. + const runaway = try Regex.compile(context, "^(a+)+$", true); + defer runaway.deinit(); + const subject = "a" ** 64 ++ "b"; + try testing.expect(!runaway.matches(subject)); + try testing.expect(runaway.matches("a" ** 64)); +} diff --git a/src/network/adblock/pattern.zig b/src/network/adblock/pattern.zig index 6bd2d085d..e04c6a10c 100644 --- a/src/network/adblock/pattern.zig +++ b/src/network/adblock/pattern.zig @@ -28,8 +28,11 @@ const NetworkFilter = @import("NetworkFilter.zig"); /// The URL a request is matched against, with its hostname located once so /// every filter can reuse the offsets. pub const Url = struct { - /// Lowercased, fragment stripped. + /// Lowercased, fragment stripped: what patterns, stored lowercased, walk. text: []const u8, + /// Fragment stripped, case kept: what a regex reads, since `$match-case` + /// is only meaningful on the original. + raw: []const u8, host_start: u32, host_end: u32, @@ -40,7 +43,7 @@ pub const Url = struct { /// Locates the hostname inside `url`: after "scheme://", up to the port, /// path, query or end. Offsets, not a slice, because the matcher needs to /// resume the pattern right where the hostname stops. - pub fn init(url: []const u8) Url { + pub fn init(url: []const u8, raw: []const u8) Url { var start: usize = 0; if (std.mem.indexOf(u8, url, "://")) |scheme| start = scheme + 3; @@ -74,6 +77,7 @@ pub const Url = struct { return .{ .text = url, + .raw = raw, .host_start = @intCast(start), .host_end = @intCast(host_end), }; @@ -86,8 +90,8 @@ pub fn matches(filter: *const NetworkFilter, url: Url) bool { switch (filter.kind) { // Option-only filters ("$script,domain=x") match any URL. .any => return true, - // Never indexed: there is no regex engine to run them with. - .regex => return false, + // uBO tests the raw URL; the case-insensitive flag is on the pattern. + .regex => return filter.regex.?.matches(url.raw), .hostname, .plain, .wildcard => {}, } @@ -235,28 +239,28 @@ const testing = @import("../../testing.zig"); fn testMatch(arena: std.mem.Allocator, line: []const u8, url: []const u8) !bool { const filter = try NetworkFilter.parse(arena, line); - return matches(&filter, .init(url)); + return matches(&filter, .init(url, url)); } test "adblock.pattern: hostname location" { - var u: Url = .init("https://ads.example.com/x?y=1"); + var u: Url = .init("https://ads.example.com/x?y=1", ""); try testing.expectString("ads.example.com", u.hostname()); - u = .init("https://ads.example.com:8443/x"); + u = .init("https://ads.example.com:8443/x", ""); try testing.expectString("ads.example.com", u.hostname()); - u = .init("https://example.com"); + u = .init("https://example.com", ""); try testing.expectString("example.com", u.hostname()); - u = .init("http://[::1]:9222/json"); + u = .init("http://[::1]:9222/json", ""); try testing.expectString("[::1]", u.hostname()); // Credentials are not part of the host. - u = .init("https://user:pass@ads.example.com/x"); + u = .init("https://user:pass@ads.example.com/x", ""); try testing.expectString("ads.example.com", u.hostname()); // A pattern is matched against whatever it is given, even a bare path. - u = .init("/relative/path"); + u = .init("/relative/path", ""); try testing.expectString("", u.hostname()); }