Engine: index regex filters by the literals every match carries

A regex filter rode the fallback bucket, which every request pays for.
uBO reads a token out of one by flattening the pattern into a string
where literal characters stay and anything else becomes a marker that
says only whether a token character may be there, then taking the
alphanumeric runs bounded on both sides by something that is surely
not one; `tokenizableStrFromRegex`, ported here as `RegexShape`.

One departure: a positive lookaround becomes a marker rather than
being inlined, since a token must never come from text the regex does
not consume. Anything the flattening does not follow yields no token
at all, which is never wrong.

On the 79 regex rules the parser accepts across EasyList, EasyPrivacy
and uBO's lists, 72 now land in a bucket.

src/network/adblock/AdBlocker.zig

@@ -645,6 +645,9 @@ test "adblock.AdBlocker: the regex rules from EasyList" {
     );
     try testing.expectEqual(4, blocker.rules_loaded);
     try testing.expectEqual(0, blocker.rules_skipped);
+    // Each carries a literal every match has, so none rides the fallback.
+    try testing.expectEqual(0, blocker.blocking.fallback.len);
+    try testing.expectEqual(0, blocker.exceptions.fallback.len);
 
     const invoke = "https://host.com/0123456789abcdef0123456789abcdef/invoke.js";
     try expectVerdict(&blocker, .blocked, invoke, "site.com", script);

src/network/adblock/Engine.zig

@@ -333,9 +333,10 @@ fn collectTokens(filter: *const NetworkFilter, buf: []u32) []u32 {
         }
     }
 
-    // A /regex/ literal is never indexed (no token is pulled out of one),
-    // and `.any` has no pattern at all.
-    if (filter.kind == .regex or filter.pattern.len == 0) return buf[0..n];
+    // A /regex/ literal has no hostname, so `n` is still 0 here.
+    if (filter.kind == .regex) return regexTokens(filter.pattern[1 .. filter.pattern.len - 1], buf);
+    // `.any` has no pattern at all.
+    if (filter.pattern.len == 0) return buf[0..n];
 
     const text = filter.pattern;
     var i: usize = 0;
@@ -363,6 +364,312 @@ fn collectTokens(filter: *const NetworkFilter, buf: []u32) []u32 {
     return buf[0..n];
 }
 
+/// The tokens a regex is sure to have wherever it matches, uBO's
+/// `tokenizableStrFromRegex`: the pattern is flattened into a string where
+/// literal characters stay and everything else becomes a marker saying only
+/// whether it could be a token character, then read like a plain pattern.
+/// Anything the flattening does not follow yields no token at all, which is
+/// never wrong: the filter then rides the fallback bucket.
+fn regexTokens(source: []const u8, buf: []u32) []u32 {
+    // A pattern's shape is never longer than the pattern.
+    var shape_buf: [8 * 1024]u8 = undefined;
+    if (source.len > shape_buf.len) return buf[0..0];
+    var shape: RegexShape = .{ .source = source, .out = &shape_buf };
+    const text = shape.flatten() catch return buf[0..0];
+
+    var n: usize = 0;
+    var i: usize = 0;
+    while (i < text.len) {
+        if (!isTokenChar(text[i])) {
+            i += 1;
+            continue;
+        }
+        const start = i;
+        while (i < text.len and isTokenChar(text[i])) i += 1;
+
+        // The pattern's ends are open unless anchored, and a marker that may
+        // be a token character would extend the run.
+        const left_bounded = start != 0 and text[start - 1] != RegexShape.maybe_token;
+        const right_bounded = i != text.len and text[i] != RegexShape.maybe_token;
+        if (left_bounded and right_bounded) {
+            if (n == buf.len) return buf[0..n];
+            buf[n] = hash(text[start..i]);
+            n += 1;
+        }
+    }
+    return buf[0..n];
+}
+
+const RegexShape = struct {
+    source: []const u8,
+    out: []u8,
+    i: usize = 0,
+    n: usize = 0,
+
+    /// Whatever matches here is not a token character: an anchor, `\b`, a
+    /// quantified non-token literal.
+    const not_token = 0x00;
+    /// Whatever matches here may be a token character: `.`, `[a-z]`, `\d`, a
+    /// quantified literal.
+    const maybe_token = 0x01;
+    // The same two for a stretch that may match nothing at all; resolved
+    // once both neighbours are known.
+    const not_token_optional = 0x02;
+    const maybe_token_optional = 0x03;
+
+    const Error = error{Unsupported};
+
+    fn flatten(self: *RegexShape) Error![]const u8 {
+        try self.alternation(false);
+        if (self.i != self.source.len) return error.Unsupported;
+        self.resolveOptional();
+        return self.out[0..self.n];
+    }
+
+    /// Parses alternatives up to the closing parenthesis (or the end). More
+    /// than one collapses to two markers: all that is sure about `a|bc` is
+    /// how it starts and ends.
+    fn alternation(self: *RegexShape, nested: bool) Error!void {
+        const start = self.n;
+        var branches: usize = 1;
+        var branch_start = start;
+        var first = false;
+        var last = false;
+        while (true) {
+            try self.sequence(nested);
+            const branch = self.out[branch_start..self.n];
+            first = first or startsTokenish(branch);
+            last = last or endsTokenish(branch);
+            if (self.i == self.source.len or self.source[self.i] != '|') break;
+            self.i += 1;
+            branches += 1;
+            branch_start = self.n;
+        }
+        if (branches == 1) return;
+        self.n = start;
+        self.emit(if (first) maybe_token else not_token);
+        self.emit(if (last) maybe_token else not_token);
+    }
+
+    fn sequence(self: *RegexShape, nested: bool) Error!void {
+        while (self.i < self.source.len) {
+            const atom_start = self.n;
+            const c = self.source[self.i];
+            switch (c) {
+                '|' => return,
+                ')' => {
+                    if (!nested) return error.Unsupported;
+                    return;
+                },
+                '(' => try self.group(),
+                '[' => try self.class(),
+                '\\' => try self.escape(),
+                '.' => {
+                    self.i += 1;
+                    self.emit(maybe_token);
+                },
+                '^', '$' => {
+                    self.i += 1;
+                    self.emit(not_token);
+                },
+                '*', '+', '?' => return error.Unsupported,
+                else => {
+                    self.i += 1;
+                    self.emit(std.ascii.toLower(c));
+                },
+            }
+            try self.quantifier(atom_start);
+        }
+    }
+
+    fn group(self: *RegexShape) Error!void {
+        const start = self.n;
+        self.i += 1;
+        var lookaround: enum { none, positive, negative } = .none;
+        if (self.i < self.source.len and self.source[self.i] == '?') {
+            self.i += 1;
+            const kind = self.take() orelse return error.Unsupported;
+            switch (kind) {
+                ':' => {},
+                '=' => lookaround = .positive,
+                '!' => lookaround = .negative,
+                '<' => {
+                    const next = self.take() orelse return error.Unsupported;
+                    switch (next) {
+                        '=' => lookaround = .positive,
+                        '!' => lookaround = .negative,
+                        else => {
+                            // A named group is a plain group with a label.
+                            const close = std.mem.indexOfScalarPos(u8, self.source, self.i, '>') orelse return error.Unsupported;
+                            self.i = close + 1;
+                        },
+                    }
+                },
+                else => return error.Unsupported,
+            }
+        }
+        try self.alternation(true);
+        if (self.take() != ')') return error.Unsupported;
+        switch (lookaround) {
+            .none => {},
+            // Consumes nothing, so the neighbours touch; what it asserts
+            // could still be anything, and that is all a token may rely on.
+            .positive => {
+                self.n = start;
+                self.emit(maybe_token);
+            },
+            // Consumes nothing and rules text out: the neighbours touch.
+            .negative => self.n = start,
+        }
+    }
+
+    /// `[...]` is one character; a token character can come out of it if any
+    /// member is one, or if it is negated.
+    fn class(self: *RegexShape) Error!void {
+        self.i += 1;
+        var maybe = false;
+        if (self.i < self.source.len and self.source[self.i] == '^') {
+            self.i += 1;
+            maybe = true;
+        }
+        var first = true;
+        while (true) {
+            const c = self.take() orelse return error.Unsupported;
+            if (c == ']' and !first) break;
+            first = false;
+            if (c == '\\') {
+                const e = self.take() orelse return error.Unsupported;
+                switch (e) {
+                    'd', 'D', 'w', 'W', 's', 'S' => maybe = true,
+                    'b', 'n', 'r', 't', 'f', 'v' => {},
+                    else => if (std.ascii.isAlphanumeric(e)) return error.Unsupported,
+                }
+                continue;
+            }
+            // A range is assumed to reach token characters.
+            if (c == '-' or std.ascii.isAlphanumeric(c)) maybe = true;
+        }
+        self.emit(if (maybe) maybe_token else not_token);
+    }
+
+    fn escape(self: *RegexShape) Error!void {
+        self.i += 1;
+        const e = self.take() orelse return error.Unsupported;
+        switch (e) {
+            'd', 'D', 'w', 'W', 's', 'S', 'B' => self.emit(maybe_token),
+            'b', 'n', 'r', 't', 'f', 'v' => self.emit(not_token),
+            // Code points, backreferences, properties: not worth following.
+            'x', 'u', 'c', 'k', 'p', 'P', '0'...'9' => return error.Unsupported,
+            // Anything else escaped is itself, as JavaScript reads it.
+            else => self.emit(std.ascii.toLower(e)),
+        }
+    }
+
+    /// Applies a quantifier, if one follows, to what was just emitted. Only
+    /// the first and last character classes survive a repeat: `ab+` may match
+    /// "abbb", and its token is not "ab".
+    fn quantifier(self: *RegexShape, atom_start: usize) Error!void {
+        if (self.i == self.source.len) return;
+        var min: usize = 0;
+        var max: ?usize = null;
+        switch (self.source[self.i]) {
+            '*' => self.i += 1,
+            '+' => {
+                self.i += 1;
+                min = 1;
+            },
+            '?' => {
+                self.i += 1;
+                max = 1;
+            },
+            '{' => {
+                const close = std.mem.indexOfScalarPos(u8, self.source, self.i, '}') orelse return;
+                const body = self.source[self.i + 1 .. close];
+                const comma = std.mem.indexOfScalar(u8, body, ',');
+                const min_text = if (comma) |at| body[0..at] else body;
+                // Not a quantifier at all: JavaScript reads the `{` literally.
+                min = std.fmt.parseUnsigned(usize, min_text, 10) catch return;
+                max = if (comma) |at|
+                    (if (at + 1 == body.len) null else std.fmt.parseUnsigned(usize, body[at + 1 ..], 10) catch return)
+                else
+                    min;
+                self.i = close + 1;
+            },
+            else => return,
+        }
+        // A lazy `?` changes nothing about what can match.
+        if (self.i < self.source.len and self.source[self.i] == '?') self.i += 1;
+
+        const atom = self.out[atom_start..self.n];
+        const first = startsTokenish(atom);
+        const last = endsTokenish(atom);
+        self.n = atom_start;
+        if (max == 0) return;
+        if (min != 0) {
+            self.emit(if (first) maybe_token else not_token);
+            self.emit(if (last) maybe_token else not_token);
+        } else {
+            self.emit(if (first) maybe_token_optional else not_token_optional);
+            self.emit(if (last) maybe_token_optional else not_token_optional);
+        }
+    }
+
+    /// An optional stretch may vanish, letting its neighbours touch: it
+    /// resolves to markers that carry whichever side could be a token
+    /// character, so no run is read as bounded by something that may be
+    /// gone.
+    fn resolveOptional(self: *RegexShape) void {
+        var i: usize = 0;
+        while (i < self.n) {
+            if (!isOptional(self.out[i])) {
+                i += 1;
+                continue;
+            }
+            var end = i;
+            while (end < self.n and isOptional(self.out[end])) end += 1;
+            const left = self.out[0..i];
+            const middle = self.out[i..end];
+            const right = self.out[end..self.n];
+            const head: u8 = if (startsTokenish(right) or startsTokenish(middle)) maybe_token else not_token;
+            const tail: u8 = if (endsTokenish(left) or endsTokenish(middle)) maybe_token else not_token;
+            // Quantifiers emit markers in pairs, so a stretch is never shorter
+            // than what replaces it.
+            std.mem.copyForwards(u8, self.out[i + 2 .. self.n - (middle.len - 2)], right);
+            self.out[i] = head;
+            self.out[i + 1] = tail;
+            self.n -= middle.len - 2;
+            i += 2;
+        }
+    }
+
+    fn take(self: *RegexShape) ?u8 {
+        if (self.i == self.source.len) return null;
+        defer self.i += 1;
+        return self.source[self.i];
+    }
+
+    fn emit(self: *RegexShape, c: u8) void {
+        self.out[self.n] = c;
+        self.n += 1;
+    }
+
+    fn isOptional(c: u8) bool {
+        return c == not_token_optional or c == maybe_token_optional;
+    }
+
+    fn isTokenish(c: u8) bool {
+        return c == maybe_token or c == maybe_token_optional or isTokenChar(c);
+    }
+
+    fn startsTokenish(s: []const u8) bool {
+        return s.len != 0 and isTokenish(s[0]);
+    }
+
+    fn endsTokenish(s: []const u8) bool {
+        return s.len != 0 and isTokenish(s[s.len - 1]);
+    }
+};
+
 const testing = @import("../../testing.zig");
 
 fn tokensOf(arena: Allocator, line: []const u8, buf: []u32) ![]u32 {
@@ -377,6 +684,66 @@ fn contains(tokens: []const u32, token: []const u8) bool {
     return false;
 }
 
+test "adblock.Engine: regex filters yield the tokens every match carries" {
+    var arena_state = std.heap.ArenaAllocator.init(testing.allocator);
+    defer arena_state.deinit();
+    const arena = arena_state.allocator();
+    var buf: [TOKENS_MAX]u32 = undefined;
+
+    // Literal runs between literal non-token characters; the open end of an
+    // unanchored pattern bounds nothing.
+    var tokens = try tokensOf(arena, "/\\/[0-9a-f]{32}\\/invoke\\.js/", &buf);
+    try testing.expectEqual(1, tokens.len);
+    try testing.expect(contains(tokens, "invoke"));
+
+    // Anchors bound; `https?` may be either, so "http" is no token.
+    tokens = try tokensOf(arena, "/^https?:\\/\\/[0-9a-z]{5,}\\.com\\/.*/", &buf);
+    try testing.expectEqual(1, tokens.len);
+    try testing.expect(contains(tokens, "com"));
+    tokens = try tokensOf(arena, "/[a-z]{2,}\\.gif$/", &buf);
+    try testing.expectEqual(1, tokens.len);
+    try testing.expect(contains(tokens, "gif"));
+
+    // `\b` bounds, as uBO reads it: `/\bads\b/` is "ads", not "bads".
+    tokens = try tokensOf(arena, "/\\bads\\b/", &buf);
+    try testing.expect(contains(tokens, "ads"));
+
+    // Hashed lowercased, like the URL it is looked up in.
+    tokens = try tokensOf(arena, "/\\/Ads\\//$match-case", &buf);
+    try testing.expect(contains(tokens, "ads"));
+
+    // An optional stretch may vanish and glue its neighbours: "adsbanner".
+    tokens = try tokensOf(arena, "/\\/ads\\/?banner\\//", &buf);
+    try testing.expectEqual(0, tokens.len);
+    // A repeat is not the literal it repeats.
+    tokens = try tokensOf(arena, "/\\/ab+c\\//", &buf);
+    try testing.expectEqual(0, tokens.len);
+
+    // Alternation and the text under a quantified group are uncertain.
+    tokens = try tokensOf(arena, "/^https?:\\/\\/(35|104)\\.(\\d){1,3}\\//", &buf);
+    try testing.expectEqual(0, tokens.len);
+    tokens = try tokensOf(arena, "/ads|banner/", &buf);
+    try testing.expectEqual(0, tokens.len);
+    // ... but a group with one branch is transparent.
+    tokens = try tokensOf(arena, "/\\/(?:ads)\\//", &buf);
+    try testing.expect(contains(tokens, "ads"));
+
+    // A negative lookaround consumes nothing; a positive one is not
+    // trusted to spell anything.
+    tokens = try tokensOf(arena, "/\\/(?!ads)banner\\//", &buf);
+    try testing.expect(contains(tokens, "banner"));
+    tokens = try tokensOf(arena, "/\\/(?=ads)ads\\//", &buf);
+    try testing.expectEqual(0, tokens.len);
+
+    // What is not followed yields nothing rather than something wrong.
+    tokens = try tokensOf(arena, "/\\/\\x41ds\\//", &buf);
+    try testing.expectEqual(0, tokens.len);
+    tokens = try tokensOf(arena, "/\\/(ads\\//", &buf);
+    try testing.expectEqual(0, tokens.len);
+    tokens = try tokensOf(arena, "/about:blank.*/", &buf);
+    try testing.expectEqual(0, tokens.len);
+}
+
 test "adblock.Engine: a request keeps its first tokens, the rest as tail" {
     const max = Request.URL_TOKENS_MAX;
     const kind: NetworkFilter.ResourceTypes = .{ .script = true };