mirror of
https://github.com/lightpanda-io/browser.git
synced 2026-09-16 16:01:30 -04:00
Adrià Arrufat
433ca9b747
The raw URL lives on `pattern.Url` next to the lowercased one, so `pattern.matches` owns the `.regex` arm like every other kind and the engine stops special-casing it. `Request.init` does the lowercasing itself, as `fromHttp` already had to, instead of asking callers for both spellings. The regex shape now spells its uncertain marker as `*` and keeps non-token literals as one marker, so it is read by the same bounded-token loop as a plain pattern rather than a copy of it. The quantifier parser keeps only what it uses: whether the atom may be absent. `Regex.matches` runs on a stack-first allocator: PCRE2 wants a match data block and 20KB of backtracking frames per call, which no longer touches the heap in the common case. A filter holds a pointer to its regex, keeping `NetworkFilter` at its previous size.
786 lines
30 KiB
Zig
786 lines
30 KiB
Zig
// Copyright (C) 2023-2026 Lightpanda (Selecy SAS)
|
|
//
|
|
// Francis Bouvier <francis@lightpanda.io>
|
|
// Pierre Tachoire <pierre@lightpanda.io>
|
|
//
|
|
// This program is free software: you can redistribute it and/or modify
|
|
// it under the terms of the GNU Affero General Public License as
|
|
// published by the Free Software Foundation, either version 3 of the
|
|
// License, or (at your option) any later version.
|
|
//
|
|
// This program is distributed in the hope that it will be useful,
|
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
// GNU Affero General Public License for more details.
|
|
//
|
|
// You should have received a copy of the GNU Affero General Public License
|
|
// along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
|
|
//! One indexed set of network filters — the blocking ones, the `$important`
|
|
//! ones or the exceptions — and the lookup that finds which of them a request
|
|
//! matches. A port of adblock-rust's `NetworkFilterList`.
|
|
//!
|
|
//! Checking every filter against every request is not affordable: EasyList
|
|
//! alone carries tens of thousands. Instead each filter is filed under one
|
|
//! token of its pattern — a run of alphanumerics that any URL it matches is
|
|
//! bound to contain — and a request only ever pays for the filters filed
|
|
//! under the tokens its own URL happens to have.
|
|
//!
|
|
//! Which token decides how much that costs, so each filter takes its rarest
|
|
//! one across the whole loaded corpus: "gampad" narrows to a handful of
|
|
//! filters where "com" would narrow to nothing. Filters whose pattern offers
|
|
//! no token that is guaranteed to survive into the URL (see `collectTokens`)
|
|
//! go to `fallback`, which every request pays for.
|
|
|
|
const std = @import("std");
|
|
|
|
const URL = @import("../../browser/URL.zig");
|
|
const HttpClient = @import("../HttpClient.zig");
|
|
|
|
const domain = @import("domain.zig");
|
|
const pattern = @import("pattern.zig");
|
|
const NetworkFilter = @import("NetworkFilter.zig");
|
|
|
|
const Allocator = std.mem.Allocator;
|
|
|
|
const Engine = @This();
|
|
|
|
/// The shared filter array; `buckets` and `fallback` hold indices into it.
|
|
filters: []const NetworkFilter,
|
|
buckets: std.AutoHashMapUnmanaged(u32, []const u32),
|
|
/// Filters no token can stand for; checked on every request.
|
|
fallback: []const u32,
|
|
|
|
pub const empty: Engine = .{
|
|
.filters = &.{},
|
|
.buckets = .empty,
|
|
.fallback = &.{},
|
|
};
|
|
|
|
pub fn deinit(self: *Engine, allocator: Allocator) void {
|
|
self.buckets.deinit(allocator);
|
|
self.* = .empty;
|
|
}
|
|
|
|
/// Everything the filters need to know about a request. Built once per
|
|
/// request and shared by all three engines.
|
|
pub const Request = struct {
|
|
url: pattern.Url,
|
|
/// The hostname of the document the request belongs to. Falls back to the
|
|
/// request's own hostname, which is what uBO does for top-level loads.
|
|
source_hostname: []const u8,
|
|
/// Exactly one bit set.
|
|
kind: NetworkFilter.ResourceTypes,
|
|
third_party: bool,
|
|
/// The URL's first tokens, hashed once here so no engine retokenizes.
|
|
tokens_buf: [URL_TOKENS_MAX]u32,
|
|
tokens_len: usize,
|
|
/// The rest of the URL, past the last token that fit.
|
|
/// Each engine tokenizes it itself, so no token is ever lost. Token-free for
|
|
/// all but the longest URLs.
|
|
tail: []const u8,
|
|
|
|
const URL_TOKENS_MAX = 128;
|
|
|
|
/// Longest URL `fromHttp` will normalize. Anything past this is not a
|
|
/// resource a filter list has an opinion about.
|
|
const URL_MAX = 8 * 1024;
|
|
/// DNS's own hostname limit.
|
|
const SOURCE_MAX = 253;
|
|
|
|
/// Backs the normalized text of a `fromHttp` request, which stays valid
|
|
/// only as long as the buffers do.
|
|
pub const Buffers = struct {
|
|
url: [URL_MAX]u8,
|
|
source: [SOURCE_MAX]u8,
|
|
};
|
|
|
|
/// `buf` backs the lowercased URL; null when `url` is longer than it.
|
|
/// `source_hostname` may be empty when there is no document context.
|
|
pub fn init(
|
|
url: []const u8,
|
|
buf: []u8,
|
|
source_hostname: []const u8,
|
|
kind: NetworkFilter.ResourceTypes,
|
|
) ?Request {
|
|
// No request URL carries a fragment onto the wire.
|
|
const raw = URL.stripFragment(url);
|
|
const lowered = lowercase(raw, buf) orelse return null;
|
|
const parsed: pattern.Url = .init(lowered, raw);
|
|
const source = if (source_hostname.len == 0) parsed.hostname() else source_hostname;
|
|
var request: Request = .{
|
|
.url = parsed,
|
|
.source_hostname = source,
|
|
.kind = kind,
|
|
.third_party = domain.isThirdParty(parsed.hostname(), source),
|
|
.tokens_buf = undefined,
|
|
.tokens_len = 0,
|
|
.tail = "",
|
|
};
|
|
var it: Tokens = .{ .text = lowered };
|
|
while (request.tokens_len < request.tokens_buf.len) {
|
|
const token = it.next() orelse break;
|
|
request.tokens_buf[request.tokens_len] = token;
|
|
request.tokens_len += 1;
|
|
}
|
|
request.tail = lowered[it.i..];
|
|
return request;
|
|
}
|
|
|
|
pub fn fromHttp(transfer: *const HttpClient.Transfer, buffers: *Buffers) ?Request {
|
|
const req = &transfer.req;
|
|
|
|
var owner: ?*const HttpClient.Owner = transfer.owner;
|
|
var subframe = false;
|
|
if (req.resource_type == .document) {
|
|
owner = if (owner) |o| o.parent else null;
|
|
subframe = owner != null;
|
|
}
|
|
const source_url = if (owner) |o| o.documentUrl() else null;
|
|
const source_host = if (source_url) |u| URL.getOriginHostname(u) else "";
|
|
if (source_host.len > buffers.source.len) return null;
|
|
const source = std.ascii.lowerString(&buffers.source, source_host);
|
|
|
|
const resource_type: NetworkFilter.ResourceTypes = switch (req.resource_type) {
|
|
.document => if (subframe) .{ .subdocument = true } else .{ .document = true },
|
|
.script => .{ .script = true },
|
|
.stylesheet => .{ .stylesheet = true },
|
|
.xhr, .fetch => .{ .xmlhttprequest = true },
|
|
.image => .{ .image = true },
|
|
.eventsource => .{ .other = true },
|
|
.worker => .{ .script = true },
|
|
};
|
|
|
|
return .init(req.url, &buffers.url, source, resource_type);
|
|
}
|
|
|
|
inline fn tokens(self: *const Request) []const u32 {
|
|
return self.tokens_buf[0..self.tokens_len];
|
|
}
|
|
|
|
/// Lowercases `url` into `buf`, as patterns are stored lowercased.
|
|
fn lowercase(url: []const u8, buf: []u8) ?[]const u8 {
|
|
const upper = for (url, 0..) |c, i| {
|
|
if (std.ascii.isUpper(c)) break i;
|
|
} else return url;
|
|
|
|
if (url.len > buf.len) return null;
|
|
const out = buf[0..url.len];
|
|
@memcpy(out[0..upper], url[0..upper]);
|
|
_ = std.ascii.lowerString(out[upper..], url[upper..]);
|
|
return out;
|
|
}
|
|
};
|
|
|
|
/// How often each token appears across every indexed filter. Shared by the
|
|
/// engines so that "rarest" means rarest overall, not within one class.
|
|
pub const Histogram = std.AutoHashMapUnmanaged(u32, u32);
|
|
|
|
/// Tokens considered per filter. Long query-string patterns can exceed this;
|
|
/// taking the first few costs a wider bucket, never a wrong answer.
|
|
const TOKENS_MAX = 32;
|
|
|
|
/// The first filter that matches, or null.
|
|
pub fn match(self: *const Engine, request: *const Request) ?*const NetworkFilter {
|
|
if (self.buckets.count() != 0) {
|
|
for (request.tokens()) |token| {
|
|
if (self.matchToken(token, request)) |filter| return filter;
|
|
}
|
|
// Whatever the request could not hold is hashed again.
|
|
var it: Tokens = .{ .text = request.tail };
|
|
while (it.next()) |token| {
|
|
if (self.matchToken(token, request)) |filter| return filter;
|
|
}
|
|
}
|
|
return self.matchIn(self.fallback, request);
|
|
}
|
|
|
|
fn matchToken(self: *const Engine, token: u32, request: *const Request) ?*const NetworkFilter {
|
|
const bucket = self.buckets.get(token) orelse return null;
|
|
return self.matchIn(bucket, request);
|
|
}
|
|
|
|
fn matchIn(self: *const Engine, bucket: []const u32, request: *const Request) ?*const NetworkFilter {
|
|
for (bucket) |index| {
|
|
const filter = &self.filters[index];
|
|
if (matchesFilter(filter, request)) return filter;
|
|
}
|
|
return null;
|
|
}
|
|
|
|
/// Cheapest constraint first: the type and party bits are two comparisons,
|
|
/// the pattern walk is the expensive one.
|
|
fn matchesFilter(filter: *const NetworkFilter, request: *const Request) bool {
|
|
if (filter.types.bits() & request.kind.bits() == 0) return false;
|
|
if (request.third_party) {
|
|
if (!filter.third_party) return false;
|
|
} else if (!filter.first_party) {
|
|
return false;
|
|
}
|
|
if (!filter.domains.matches(request.source_hostname)) return false;
|
|
return pattern.matches(filter, request.url);
|
|
}
|
|
|
|
/// Adds `filter`'s tokens to the corpus counts.
|
|
pub fn count(
|
|
histogram: *Histogram,
|
|
allocator: Allocator,
|
|
filter: *const NetworkFilter,
|
|
) Allocator.Error!void {
|
|
var buf: [TOKENS_MAX]u32 = undefined;
|
|
for (collectTokens(filter, &buf)) |token| {
|
|
const gop = try histogram.getOrPut(allocator, token);
|
|
gop.value_ptr.* = if (gop.found_existing) gop.value_ptr.* + 1 else 1;
|
|
}
|
|
}
|
|
|
|
/// Indexes `members` (indices into `filters`) by their rarest token.
|
|
/// `allocator` backs the bucket map, `arena` the index slices it points at.
|
|
pub fn build(
|
|
allocator: Allocator,
|
|
arena: Allocator,
|
|
filters: []const NetworkFilter,
|
|
members: []const u32,
|
|
histogram: *const Histogram,
|
|
) Allocator.Error!Engine {
|
|
var groups: std.AutoHashMapUnmanaged(u32, std.ArrayList(u32)) = .empty;
|
|
defer {
|
|
var it = groups.valueIterator();
|
|
while (it.next()) |list| list.deinit(allocator);
|
|
groups.deinit(allocator);
|
|
}
|
|
|
|
var fallback: std.ArrayList(u32) = .empty;
|
|
defer fallback.deinit(allocator);
|
|
|
|
var buf: [TOKENS_MAX]u32 = undefined;
|
|
for (members) |index| {
|
|
var rarest: ?u32 = null;
|
|
var lowest: u32 = std.math.maxInt(u32);
|
|
for (collectTokens(&filters[index], &buf)) |token| {
|
|
const seen = histogram.get(token) orelse 0;
|
|
if (seen < lowest) {
|
|
lowest = seen;
|
|
rarest = token;
|
|
}
|
|
}
|
|
|
|
const token = rarest orelse {
|
|
try fallback.append(allocator, index);
|
|
continue;
|
|
};
|
|
const gop = try groups.getOrPut(allocator, token);
|
|
if (!gop.found_existing) gop.value_ptr.* = .empty;
|
|
try gop.value_ptr.append(allocator, index);
|
|
}
|
|
|
|
var buckets: std.AutoHashMapUnmanaged(u32, []const u32) = .empty;
|
|
errdefer buckets.deinit(allocator);
|
|
try buckets.ensureTotalCapacity(allocator, groups.count());
|
|
|
|
var it = groups.iterator();
|
|
while (it.next()) |entry| {
|
|
buckets.putAssumeCapacity(entry.key_ptr.*, try arena.dupe(u32, entry.value_ptr.items));
|
|
}
|
|
|
|
return .{
|
|
.filters = filters,
|
|
.buckets = buckets,
|
|
.fallback = try arena.dupe(u32, fallback.items),
|
|
};
|
|
}
|
|
|
|
/// Runs of alphanumerics. Everything else ('.', '/', '-', '%', '_', '?') is a boundary.
|
|
const Tokens = struct {
|
|
text: []const u8,
|
|
i: usize = 0,
|
|
|
|
pub fn next(self: *Tokens) ?u32 {
|
|
while (self.i < self.text.len and !isTokenChar(self.text[self.i])) self.i += 1;
|
|
if (self.i == self.text.len) return null;
|
|
|
|
const start = self.i;
|
|
while (self.i < self.text.len and isTokenChar(self.text[self.i])) self.i += 1;
|
|
return hash(self.text[start..self.i]);
|
|
}
|
|
};
|
|
|
|
fn isTokenChar(c: u8) bool {
|
|
return std.ascii.isAlphanumeric(c);
|
|
}
|
|
|
|
fn hash(token: []const u8) u32 {
|
|
return @truncate(std.hash.Wyhash.hash(0, token));
|
|
}
|
|
|
|
fn collectTokens(filter: *const NetworkFilter, buf: []u32) []u32 {
|
|
var n: usize = 0;
|
|
|
|
if (filter.hostname.len != 0) {
|
|
var it: Tokens = .{ .text = filter.hostname };
|
|
while (it.next()) |token| {
|
|
if (n == buf.len) return buf[0..n];
|
|
buf[n] = token;
|
|
n += 1;
|
|
}
|
|
}
|
|
|
|
// A /regex/ literal has no hostname, so `n` is still 0 here.
|
|
if (filter.kind == .regex) return regexTokens(filter.pattern[1 .. filter.pattern.len - 1], buf);
|
|
// `.any` has no pattern at all.
|
|
if (filter.pattern.len == 0) return buf[0..n];
|
|
|
|
const left_anchored = filter.left_anchor or filter.hostname_anchor;
|
|
return buf[0..boundedTokens(filter.pattern, left_anchored, filter.right_anchor, buf, n)];
|
|
}
|
|
|
|
/// Appends the tokens of `text` that any URL matching it carries whole: the
|
|
/// alphanumeric runs no `*` may extend, and that the open ends of the text
|
|
/// may not extend either unless anchored. Returns the new count.
|
|
fn boundedTokens(text: []const u8, left_anchored: bool, right_anchored: bool, buf: []u32, from: usize) usize {
|
|
var n = from;
|
|
var i: usize = 0;
|
|
while (i < text.len) {
|
|
if (!isTokenChar(text[i])) {
|
|
i += 1;
|
|
continue;
|
|
}
|
|
const start = i;
|
|
while (i < text.len and isTokenChar(text[i])) i += 1;
|
|
|
|
const left_bounded = if (start == 0) left_anchored else text[start - 1] != '*';
|
|
const right_bounded = if (i == text.len) right_anchored else text[i] != '*';
|
|
|
|
if (left_bounded and right_bounded) {
|
|
if (n == buf.len) return n;
|
|
buf[n] = hash(text[start..i]);
|
|
n += 1;
|
|
}
|
|
}
|
|
return n;
|
|
}
|
|
|
|
/// The tokens a regex is sure to have wherever it matches, uBO's
|
|
/// `tokenizableStrFromRegex`: the pattern is flattened into a plain pattern
|
|
/// where literal token characters stay and everything else becomes a marker
|
|
/// saying only whether it could be one. Anything the flattening does not
|
|
/// follow yields no token at all, which is never wrong: the filter then
|
|
/// rides the fallback bucket.
|
|
fn regexTokens(source: []const u8, buf: []u32) []u32 {
|
|
// A pattern's shape is never longer than the pattern.
|
|
var shape_buf: [8 * 1024]u8 = undefined;
|
|
if (source.len > shape_buf.len) return buf[0..0];
|
|
var shape: RegexShape = .{ .source = source, .out = &shape_buf };
|
|
const text = shape.flatten() catch return buf[0..0];
|
|
// Anchors are in the shape itself; its ends are open.
|
|
return buf[0..boundedTokens(text, false, false, buf, 0)];
|
|
}
|
|
|
|
const RegexShape = struct {
|
|
source: []const u8,
|
|
out: []u8,
|
|
i: usize = 0,
|
|
n: usize = 0,
|
|
|
|
/// Whatever matches here is not a token character: a non-token literal,
|
|
/// an anchor, `\b`.
|
|
const not_token = 0x00;
|
|
/// Whatever matches here may be a token character: `.`, `[a-z]`, `\d`, a
|
|
/// quantified literal. Spelled as a plain pattern's wildcard so the shape
|
|
/// reads as one.
|
|
const maybe_token = '*';
|
|
// The same two for a stretch that may match nothing at all; resolved
|
|
// once both neighbours are known.
|
|
const not_token_optional = 0x02;
|
|
const maybe_token_optional = 0x03;
|
|
|
|
const Error = error{Unsupported};
|
|
|
|
fn flatten(self: *RegexShape) Error![]const u8 {
|
|
try self.alternation(false);
|
|
if (self.i != self.source.len) return error.Unsupported;
|
|
self.resolveOptional();
|
|
return self.out[0..self.n];
|
|
}
|
|
|
|
/// Parses alternatives up to the closing parenthesis (or the end). More
|
|
/// than one collapses to two markers: all that is sure about `a|bc` is
|
|
/// how it starts and ends.
|
|
fn alternation(self: *RegexShape, nested: bool) Error!void {
|
|
const start = self.n;
|
|
var branches: usize = 1;
|
|
var branch_start = start;
|
|
var first = false;
|
|
var last = false;
|
|
while (true) {
|
|
try self.sequence(nested);
|
|
const branch = self.out[branch_start..self.n];
|
|
first = first or startsTokenish(branch);
|
|
last = last or endsTokenish(branch);
|
|
if (self.i == self.source.len or self.source[self.i] != '|') break;
|
|
self.i += 1;
|
|
branches += 1;
|
|
branch_start = self.n;
|
|
}
|
|
if (branches == 1) return;
|
|
self.n = start;
|
|
self.emit(if (first) maybe_token else not_token);
|
|
self.emit(if (last) maybe_token else not_token);
|
|
}
|
|
|
|
fn sequence(self: *RegexShape, nested: bool) Error!void {
|
|
while (self.i < self.source.len) {
|
|
const atom_start = self.n;
|
|
const c = self.source[self.i];
|
|
switch (c) {
|
|
'|' => return,
|
|
')' => {
|
|
if (!nested) return error.Unsupported;
|
|
return;
|
|
},
|
|
'(' => try self.group(),
|
|
'[' => try self.class(),
|
|
'\\' => try self.escape(),
|
|
'.' => {
|
|
self.i += 1;
|
|
self.emit(maybe_token);
|
|
},
|
|
'^', '$' => {
|
|
self.i += 1;
|
|
self.emit(not_token);
|
|
},
|
|
'*', '+', '?' => return error.Unsupported,
|
|
else => {
|
|
self.i += 1;
|
|
self.emitLiteral(c);
|
|
},
|
|
}
|
|
try self.quantifier(atom_start);
|
|
}
|
|
}
|
|
|
|
fn group(self: *RegexShape) Error!void {
|
|
const start = self.n;
|
|
self.i += 1;
|
|
var lookaround: enum { none, positive, negative } = .none;
|
|
if (self.i < self.source.len and self.source[self.i] == '?') {
|
|
self.i += 1;
|
|
const kind = self.take() orelse return error.Unsupported;
|
|
switch (kind) {
|
|
':' => {},
|
|
'=' => lookaround = .positive,
|
|
'!' => lookaround = .negative,
|
|
'<' => {
|
|
const next = self.take() orelse return error.Unsupported;
|
|
switch (next) {
|
|
'=' => lookaround = .positive,
|
|
'!' => lookaround = .negative,
|
|
else => {
|
|
// A named group is a plain group with a label.
|
|
const close = std.mem.indexOfScalarPos(u8, self.source, self.i, '>') orelse return error.Unsupported;
|
|
self.i = close + 1;
|
|
},
|
|
}
|
|
},
|
|
else => return error.Unsupported,
|
|
}
|
|
}
|
|
try self.alternation(true);
|
|
if (self.take() != ')') return error.Unsupported;
|
|
switch (lookaround) {
|
|
.none => {},
|
|
// Consumes nothing, so the neighbours touch; what it asserts
|
|
// could still be anything, and that is all a token may rely on.
|
|
.positive => {
|
|
self.n = start;
|
|
self.emit(maybe_token);
|
|
},
|
|
// Consumes nothing and rules text out: the neighbours touch.
|
|
.negative => self.n = start,
|
|
}
|
|
}
|
|
|
|
/// `[...]` is one character; a token character can come out of it if any
|
|
/// member is one, or if it is negated.
|
|
fn class(self: *RegexShape) Error!void {
|
|
self.i += 1;
|
|
var maybe = false;
|
|
if (self.i < self.source.len and self.source[self.i] == '^') {
|
|
self.i += 1;
|
|
maybe = true;
|
|
}
|
|
var first = true;
|
|
while (true) {
|
|
const c = self.take() orelse return error.Unsupported;
|
|
if (c == ']' and !first) break;
|
|
first = false;
|
|
if (c == '\\') {
|
|
const e = self.take() orelse return error.Unsupported;
|
|
switch (e) {
|
|
'd', 'D', 'w', 'W', 's', 'S' => maybe = true,
|
|
'b', 'n', 'r', 't', 'f', 'v' => {},
|
|
else => if (std.ascii.isAlphanumeric(e)) return error.Unsupported,
|
|
}
|
|
continue;
|
|
}
|
|
// A range is assumed to reach token characters.
|
|
if (c == '-' or std.ascii.isAlphanumeric(c)) maybe = true;
|
|
}
|
|
self.emit(if (maybe) maybe_token else not_token);
|
|
}
|
|
|
|
fn escape(self: *RegexShape) Error!void {
|
|
self.i += 1;
|
|
const e = self.take() orelse return error.Unsupported;
|
|
switch (e) {
|
|
'd', 'D', 'w', 'W', 's', 'S', 'B' => self.emit(maybe_token),
|
|
'b', 'n', 'r', 't', 'f', 'v' => self.emit(not_token),
|
|
// Code points, backreferences, properties: not worth following.
|
|
'x', 'u', 'c', 'k', 'p', 'P', '0'...'9' => return error.Unsupported,
|
|
// Anything else escaped is itself, as JavaScript reads it.
|
|
else => self.emitLiteral(e),
|
|
}
|
|
}
|
|
|
|
/// Applies a quantifier, if one follows, to what was just emitted. Only
|
|
/// the first and last character classes survive a repeat: `ab+` may match
|
|
/// "abbb", and its token is not "ab". Whether the atom may be absent is
|
|
/// all that matters beyond that.
|
|
fn quantifier(self: *RegexShape, atom_start: usize) Error!void {
|
|
if (self.i == self.source.len) return;
|
|
var optional = true;
|
|
switch (self.source[self.i]) {
|
|
'*', '?' => self.i += 1,
|
|
'+' => {
|
|
self.i += 1;
|
|
optional = false;
|
|
},
|
|
'{' => {
|
|
const close = std.mem.indexOfScalarPos(u8, self.source, self.i, '}') orelse return;
|
|
const body = self.source[self.i + 1 .. close];
|
|
const comma = std.mem.indexOfScalar(u8, body, ',') orelse body.len;
|
|
// Not a quantifier at all: JavaScript reads the `{` literally.
|
|
const min = std.fmt.parseUnsigned(usize, body[0..comma], 10) catch return;
|
|
if (comma + 1 < body.len) _ = std.fmt.parseUnsigned(usize, body[comma + 1 ..], 10) catch return;
|
|
optional = min == 0;
|
|
self.i = close + 1;
|
|
},
|
|
else => return,
|
|
}
|
|
// A lazy `?` changes nothing about what can match.
|
|
if (self.i < self.source.len and self.source[self.i] == '?') self.i += 1;
|
|
|
|
const atom = self.out[atom_start..self.n];
|
|
const first = startsTokenish(atom);
|
|
const last = endsTokenish(atom);
|
|
self.n = atom_start;
|
|
if (!optional) {
|
|
self.emit(if (first) maybe_token else not_token);
|
|
self.emit(if (last) maybe_token else not_token);
|
|
} else {
|
|
self.emit(if (first) maybe_token_optional else not_token_optional);
|
|
self.emit(if (last) maybe_token_optional else not_token_optional);
|
|
}
|
|
}
|
|
|
|
/// An optional stretch may vanish, letting its neighbours touch: it
|
|
/// resolves to markers that carry whichever side could be a token
|
|
/// character, so no run is read as bounded by something that may be
|
|
/// gone.
|
|
fn resolveOptional(self: *RegexShape) void {
|
|
var i: usize = 0;
|
|
while (i < self.n) {
|
|
if (!isOptional(self.out[i])) {
|
|
i += 1;
|
|
continue;
|
|
}
|
|
var end = i;
|
|
while (end < self.n and isOptional(self.out[end])) end += 1;
|
|
const left = self.out[0..i];
|
|
const middle = self.out[i..end];
|
|
const right = self.out[end..self.n];
|
|
const head: u8 = if (startsTokenish(right) or startsTokenish(middle)) maybe_token else not_token;
|
|
const tail: u8 = if (endsTokenish(left) or endsTokenish(middle)) maybe_token else not_token;
|
|
// Quantifiers emit markers in pairs, so a stretch is never shorter
|
|
// than what replaces it.
|
|
std.mem.copyForwards(u8, self.out[i + 2 .. self.n - (middle.len - 2)], right);
|
|
self.out[i] = head;
|
|
self.out[i + 1] = tail;
|
|
self.n -= middle.len - 2;
|
|
i += 2;
|
|
}
|
|
}
|
|
|
|
fn take(self: *RegexShape) ?u8 {
|
|
if (self.i == self.source.len) return null;
|
|
defer self.i += 1;
|
|
return self.source[self.i];
|
|
}
|
|
|
|
fn emit(self: *RegexShape, c: u8) void {
|
|
self.out[self.n] = c;
|
|
self.n += 1;
|
|
}
|
|
|
|
/// A literal is kept only as far as tokens care: a token character,
|
|
/// lowercased like the URL it is looked up in, or the fact that it is not
|
|
/// one.
|
|
fn emitLiteral(self: *RegexShape, c: u8) void {
|
|
self.emit(if (std.ascii.isAlphanumeric(c)) std.ascii.toLower(c) else not_token);
|
|
}
|
|
|
|
fn isOptional(c: u8) bool {
|
|
return c == not_token_optional or c == maybe_token_optional;
|
|
}
|
|
|
|
fn isTokenish(c: u8) bool {
|
|
return c == maybe_token or c == maybe_token_optional or isTokenChar(c);
|
|
}
|
|
|
|
fn startsTokenish(s: []const u8) bool {
|
|
return s.len != 0 and isTokenish(s[0]);
|
|
}
|
|
|
|
fn endsTokenish(s: []const u8) bool {
|
|
return s.len != 0 and isTokenish(s[s.len - 1]);
|
|
}
|
|
};
|
|
|
|
const testing = @import("../../testing.zig");
|
|
|
|
fn tokensOf(arena: Allocator, line: []const u8, buf: []u32) ![]u32 {
|
|
const filter = try NetworkFilter.parse(arena, line);
|
|
return collectTokens(&filter, buf);
|
|
}
|
|
|
|
fn contains(tokens: []const u32, token: []const u8) bool {
|
|
for (tokens) |t| {
|
|
if (t == hash(token)) return true;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
test "adblock.Engine: regex filters yield the tokens every match carries" {
|
|
var arena_state = std.heap.ArenaAllocator.init(testing.allocator);
|
|
defer arena_state.deinit();
|
|
const arena = arena_state.allocator();
|
|
var buf: [TOKENS_MAX]u32 = undefined;
|
|
|
|
// Literal runs between literal non-token characters; the open end of an
|
|
// unanchored pattern bounds nothing.
|
|
var tokens = try tokensOf(arena, "/\\/[0-9a-f]{32}\\/invoke\\.js/", &buf);
|
|
try testing.expectEqual(1, tokens.len);
|
|
try testing.expect(contains(tokens, "invoke"));
|
|
|
|
// Anchors bound; `https?` may be either, so "http" is no token.
|
|
tokens = try tokensOf(arena, "/^https?:\\/\\/[0-9a-z]{5,}\\.com\\/.*/", &buf);
|
|
try testing.expectEqual(1, tokens.len);
|
|
try testing.expect(contains(tokens, "com"));
|
|
tokens = try tokensOf(arena, "/[a-z]{2,}\\.gif$/", &buf);
|
|
try testing.expectEqual(1, tokens.len);
|
|
try testing.expect(contains(tokens, "gif"));
|
|
tokens = try tokensOf(arena, "/about:blank.*/", &buf);
|
|
try testing.expectEqual(0, tokens.len);
|
|
|
|
// `\b` bounds, as uBO reads it: `/\bads\b/` is "ads", not "bads".
|
|
tokens = try tokensOf(arena, "/\\bads\\b/", &buf);
|
|
try testing.expect(contains(tokens, "ads"));
|
|
|
|
// Hashed lowercased, like the URL it is looked up in.
|
|
tokens = try tokensOf(arena, "/\\/Ads\\//$match-case", &buf);
|
|
try testing.expect(contains(tokens, "ads"));
|
|
|
|
// An optional stretch may vanish and glue its neighbours: "adsbanner".
|
|
tokens = try tokensOf(arena, "/\\/ads\\/?banner\\//", &buf);
|
|
try testing.expectEqual(0, tokens.len);
|
|
// A repeat is not the literal it repeats.
|
|
tokens = try tokensOf(arena, "/\\/ab+c\\//", &buf);
|
|
try testing.expectEqual(0, tokens.len);
|
|
|
|
// Alternation and the text under a quantified group are uncertain.
|
|
tokens = try tokensOf(arena, "/^https?:\\/\\/(35|104)\\.(\\d){1,3}\\//", &buf);
|
|
try testing.expectEqual(0, tokens.len);
|
|
// ... but a group with one branch is transparent.
|
|
tokens = try tokensOf(arena, "/\\/(?:ads)\\//", &buf);
|
|
try testing.expect(contains(tokens, "ads"));
|
|
|
|
// A negative lookaround consumes nothing; a positive one is not
|
|
// trusted to spell anything.
|
|
tokens = try tokensOf(arena, "/\\/(?!ads)banner\\//", &buf);
|
|
try testing.expect(contains(tokens, "banner"));
|
|
tokens = try tokensOf(arena, "/\\/(?=ads)ads\\//", &buf);
|
|
try testing.expectEqual(0, tokens.len);
|
|
|
|
// What is not followed yields nothing rather than something wrong.
|
|
tokens = try tokensOf(arena, "/\\/\\x41ds\\//", &buf);
|
|
try testing.expectEqual(0, tokens.len);
|
|
tokens = try tokensOf(arena, "/\\/(ads\\//", &buf);
|
|
try testing.expectEqual(0, tokens.len);
|
|
}
|
|
|
|
test "adblock.Engine: a request keeps its first tokens, the rest as tail" {
|
|
const max = Request.URL_TOKENS_MAX;
|
|
const kind: NetworkFilter.ResourceTypes = .{ .script = true };
|
|
|
|
// Exactly as many tokens as the buffer holds: nothing is left to walk...
|
|
const full = "x/" ** (max - 1) ++ "x";
|
|
var buf: [512]u8 = undefined;
|
|
var request: Request = Request.init(full, &buf, "", kind).?;
|
|
try testing.expectEqual(max, request.tokens_len);
|
|
try testing.expectEqual(0, request.tail.len);
|
|
|
|
// ...one more, and only that one is in the tail.
|
|
request = Request.init(full ++ "/y", &buf, "", kind).?;
|
|
try testing.expectEqual(max, request.tokens_len);
|
|
try testing.expectString("/y", request.tail);
|
|
var it: Tokens = .{ .text = request.tail };
|
|
try testing.expectEqual(hash("y"), it.next().?);
|
|
try testing.expect(it.next() == null);
|
|
|
|
// Fewer than the buffer holds: the tail is empty.
|
|
request = Request.init("https://example.com/a", &buf, "", kind).?;
|
|
try testing.expectEqual(4, request.tokens_len);
|
|
try testing.expectEqual(0, request.tail.len);
|
|
}
|
|
|
|
test "adblock.Engine: only tokens the URL must reproduce are collected" {
|
|
var arena_state = std.heap.ArenaAllocator.init(testing.allocator);
|
|
defer arena_state.deinit();
|
|
const arena = arena_state.allocator();
|
|
|
|
var buf: [TOKENS_MAX]u32 = undefined;
|
|
|
|
// Every hostname label is bounded, the last one by the matcher's
|
|
// label-boundary requirement — with or without '^'.
|
|
var tokens = try tokensOf(arena, "||ads.example.com^", &buf);
|
|
try testing.expect(contains(tokens, "ads"));
|
|
try testing.expect(contains(tokens, "example"));
|
|
try testing.expect(contains(tokens, "com"));
|
|
|
|
tokens = try tokensOf(arena, "||ads.example.com", &buf);
|
|
try testing.expect(contains(tokens, "example"));
|
|
try testing.expect(contains(tokens, "com"));
|
|
|
|
// A path following the hostname bounds it just as well.
|
|
tokens = try tokensOf(arena, "||youtube.com/pagead/", &buf);
|
|
try testing.expect(contains(tokens, "com"));
|
|
try testing.expect(contains(tokens, "pagead"));
|
|
|
|
// A trailing token is unbounded unless the pattern is right-anchored:
|
|
// "/ads" also matches "/adserver".
|
|
tokens = try tokensOf(arena, "||example.com/ads", &buf);
|
|
try testing.expect(!contains(tokens, "ads"));
|
|
tokens = try tokensOf(arena, "||example.com/ads|", &buf);
|
|
try testing.expect(contains(tokens, "ads"));
|
|
|
|
// A '*' on either side disqualifies the token beside it, so this one is
|
|
// carried entirely by its hostname.
|
|
tokens = try tokensOf(arena, "||example.com/a*banner*c^", &buf);
|
|
try testing.expect(!contains(tokens, "banner"));
|
|
try testing.expect(contains(tokens, "example"));
|
|
|
|
// Nothing to index: option-only filters ride the fallback bucket.
|
|
tokens = try tokensOf(arena, "$script,domain=example.com", &buf);
|
|
try testing.expectEqual(0, tokens.len);
|
|
}
|