From b553139d1ca286a67427b4b107aee97b98b16ecd Mon Sep 17 00:00:00 2001 From: Halil Durak Date: Mon, 3 Aug 2026 12:37:59 +0300 Subject: [PATCH] `HostnameTrie`: add a Zig port of uBO's HNTrie Add a faithful port of `HNTrie`; from the benchmarks made, memory usage is lower than hashmap approach (with wyhash as hasher), roughly ~1.8x. --- src/adblock/HostnameTrie.zig | 406 +++++++++++++++++++++++++++++++++++ 1 file changed, 406 insertions(+) create mode 100644 src/adblock/HostnameTrie.zig diff --git a/src/adblock/HostnameTrie.zig b/src/adblock/HostnameTrie.zig new file mode 100644 index 000000000..e41f33302 --- /dev/null +++ b/src/adblock/HostnameTrie.zig @@ -0,0 +1,406 @@ +// Copyright (C) 2023-2026 Lightpanda (Selecy SAS) +// +// Francis Bouvier +// Pierre Tachoire +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +//! A container of hostname tries: a port of uBlock Origin's +//! HNTrieContainer (src/js/hntrie.js), a character-level radix trie +//! specialized for hostname matching. +//! +//! A stored hostname matches itself and any of its subdomains, so the trie +//! compares suffixes: both the needle and the stored segments are consumed +//! back-to-front, and segment bytes are stored reversed so the matcher +//! reads `chars` forward while walking the needle backwards. A match only +//! counts on a label boundary — the needle must be fully consumed or the +//! next byte toward its start must be a '.' ("abc.com" matches +//! "www.abc.com" but not "xabc.com"). +//! +//! Character-level segments keep every sibling chain shorter than the +//! hostname alphabet (~38), so linear scans stay cheap no matter how many +//! hostnames are stored. Cells live in one flat array and link through +//! indices, which keeps the structure position-independent: growth cannot +//! invalidate links and serialization is a plain byte copy. + +const std = @import("std"); +const Allocator = std.mem.Allocator; + +const lp = @import("lightpanda"); + +const HostnameTrie = @This(); + +/// Cell 0 is reserved as the null sentinel, link value 0 means "none". +cells: std.ArrayList(Cell), +/// Bytes are stored reversed; bump-allocated, never freed +/// individually. Split segments alias sub-ranges of existing bytes. +chars: std.ArrayList(u8), + +/// Segments hold at most 127 bytes (V.len is 7 bits); longer remainders +/// chain through `right`. +const SEG_MAX = std.math.maxInt(u7); + +const Cell = extern struct { + /// Next sibling: an alternative segment at this depth. + down: u32, + /// Child: the continuation once this segment matched. + right: u32, + v: V, +}; + +/// uBO's packed segment descriptor: `offset << 8 | boundary | len`. +const V = packed struct(u32) { + /// Number of segment bytes, 1..SEG_MAX. + len: u7, + /// A stored hostname ends at this segment's end. + boundary: bool, + /// Start of the segment's bytes in `chars`. + offset: u24, +}; + +comptime { + if (@sizeOf(Cell) != 12) unreachable; +} + +pub const Error = error{ + OutOfMemory, + InvalidHostname, + TrieFull, +}; + +pub fn init(allocator: Allocator) Allocator.Error!HostnameTrie { + // TODO: We can mmap memory used here together with memory that's used for + // easylist parsing. + var self: HostnameTrie = .{ + .cells = .empty, + .chars = .empty, + }; + errdefer self.deinit(allocator); + try self.cells.append(allocator, .{ .down = 0, .right = 0, .v = @bitCast(@as(u32, 0)) }); + return self; +} + +pub fn deinit(self: *HostnameTrie, allocator: Allocator) void { + self.cells.deinit(allocator); + self.chars.deinit(allocator); +} + +/// Creates an empty trie and returns its handle. +/// Distinct tries share the container's cell and character storage. +pub fn createTrie(self: *HostnameTrie, allocator: Allocator) Allocator.Error!u32 { + const root: u32 = @intCast(self.cells.items.len); + try self.cells.append(allocator, .{ .down = 0, .right = 0, .v = @bitCast(@as(u32, 0)) }); + return root; +} + +/// Expects a normalized hostname: lowercase, no trailing dot. +pub fn add(self: *HostnameTrie, allocator: Allocator, root: u32, hostname: []const u8) Error!void { + if (hostname.len == 0) return error.InvalidHostname; + // Index just past the next needle byte to consume. + var lhnchar = hostname.len; + + if (self.cells.items[root].down == 0) { + const leaf = try self.addLeafCell(allocator, hostname, lhnchar); + self.cells.items[root].down = leaf; + return; + } + + var icell = self.cells.items[root].down; + while (true) { + const v = self.cells.items[icell].v; + // First byte mismatch; move to the next sibling, or become one. + if (self.chars.items[v.offset] != hostname[lhnchar - 1]) { + if (self.cells.items[icell].down == 0) { + const leaf = try self.addLeafCell(allocator, hostname, lhnchar); + self.cells.items[icell].down = leaf; + return; + } + icell = self.cells.items[icell].down; + continue; + } + // First byte matched; find the first mismatch in the rest. + var isegchar: usize = 1; + lhnchar -= 1; + const lsegchar: usize = v.len; + while (isegchar != lsegchar and lhnchar != 0 and + self.chars.items[v.offset + isegchar] == hostname[lhnchar - 1]) + { + isegchar += 1; + lhnchar -= 1; + } + if (isegchar == lsegchar) { + // Whole segment matched. + if (lhnchar == 0) { + // Duplicate entries just (re)mark the boundary. + self.cells.items[icell].v.boundary = true; + return; + } + // The remainder sits on a label boundary of a stored entry: + // it could only ever lose to that shorter suffix, drop it. + if (v.boundary and hostname[lhnchar - 1] == '.') return; + if (self.cells.items[icell].right == 0) { + const leaf = try self.addLeafCell(allocator, hostname, lhnchar); + self.cells.items[icell].right = leaf; + return; + } + icell = self.cells.items[icell].right; + continue; + } + // Partial match: split the segment at the mismatch. The tail cell + // aliases the existing bytes and inherits the boundary and child. + const cont = try self.addCell(allocator, .{ + .down = 0, + .right = self.cells.items[icell].right, + .v = .{ + .len = @intCast(lsegchar - isegchar), + .boundary = v.boundary, + .offset = v.offset + @as(u24, @intCast(isegchar)), + }, + }); + self.cells.items[icell].right = cont; + self.cells.items[icell].v = .{ + .len = @intCast(isegchar), + .boundary = lhnchar == 0, + .offset = v.offset, + }; + if (lhnchar != 0) { + const leaf = try self.addLeafCell(allocator, hostname, lhnchar); + self.cells.items[cont].down = leaf; + } + return; + } +} + +pub fn matches(self: *const HostnameTrie, root: u32, hostname: []const u8) ?usize { + const cells = self.cells.items; + const chars = self.chars.items; + var icell = cells[root].down; + if (icell == 0) return null; + var ineedle = hostname.len; + while (true) { + if (ineedle == 0) return null; + ineedle -= 1; + const c = hostname[ineedle]; + // Find a sibling whose segment starts with `c`. + var v = cells[icell].v; + while (chars[v.offset] != c) { + icell = cells[icell].down; + if (icell == 0) return null; + v = cells[icell].v; + } + // Every remaining byte of the segment must match. + if (v.len > 1) { + const n: usize = v.len - 1; + if (n > ineedle) return null; + for (chars[v.offset + 1 ..][0..n]) |sc| { + ineedle -= 1; + if (sc != hostname[ineedle]) return null; + } + } + if (v.boundary) { + if (ineedle == 0 or hostname[ineedle - 1] == '.') return ineedle; + } + icell = cells[icell].right; + if (icell == 0) return null; + } +} + +fn addCell(self: *HostnameTrie, allocator: Allocator, cell: Cell) Allocator.Error!u32 { + const icell: u32 = @intCast(self.cells.items.len); + try self.cells.append(allocator, cell); + return icell; +} + +fn addLeafCell(self: *HostnameTrie, allocator: Allocator, hostname: []const u8, lhnchar: usize) Error!u32 { + var remaining = lhnchar; + const first: u32 = @intCast(self.cells.items.len); + while (remaining > SEG_MAX) { + const v = try self.addSegment(allocator, hostname, remaining, remaining - SEG_MAX); + const icell: u32 = @intCast(self.cells.items.len); + try self.cells.append(allocator, .{ .down = 0, .right = icell + 1, .v = v }); + remaining -= SEG_MAX; + } + var v = try self.addSegment(allocator, hostname, remaining, 0); + v.boundary = true; + try self.cells.append(allocator, .{ .down = 0, .right = 0, .v = v }); + return first; +} + +fn addSegment(self: *HostnameTrie, allocator: Allocator, hostname: []const u8, lsegchar: usize, lsegend: usize) Error!V { + const len = lsegchar - lsegend; + lp.assert(len >= 1 and len <= SEG_MAX, "HostnameTrie.addSegment: bad segment length", .{ .len = len }); + const offset = self.chars.items.len; + if (offset + len > std.math.maxInt(u24)) return error.TrieFull; + try self.chars.ensureUnusedCapacity(allocator, len); + var i = lsegchar; + while (i != lsegend) { + i -= 1; + self.chars.appendAssumeCapacity(hostname[i]); + } + return .{ .len = @intCast(len), .boundary = false, .offset = @intCast(offset) }; +} + +const testing = std.testing; + +test "adblock.HostnameTrie: exact and subdomain matching" { + var trie: HostnameTrie = try .init(testing.allocator); + defer trie.deinit(testing.allocator); + const root = try trie.createTrie(testing.allocator); + + try trie.add(testing.allocator, root, "google.com"); + try trie.add(testing.allocator, root, "doubleclick.net"); + + try testing.expectEqual(0, trie.matches(root, "google.com").?); + try testing.expectEqual(4, trie.matches(root, "www.google.com").?); + + const needle = "metrics.ssl.doubleclick.net"; + const offset = trie.matches(root, needle).?; + try testing.expectEqualStrings("doubleclick.net", needle[offset..]); + + try testing.expectEqual(null, trie.matches(root, "google.net")); + try testing.expectEqual(null, trie.matches(root, "evilgoogle.com")); + try testing.expectEqual(null, trie.matches(root, "gle.com")); + try testing.expectEqual(null, trie.matches(root, "google.com.evil.org")); + try testing.expectEqual(null, trie.matches(root, "com")); + try testing.expectEqual(null, trie.matches(root, "")); +} + +test "adblock.HostnameTrie: duplicate and covered entries store nothing" { + var trie: HostnameTrie = try .init(testing.allocator); + defer trie.deinit(testing.allocator); + const root = try trie.createTrie(testing.allocator); + + try trie.add(testing.allocator, root, "google.com"); + const cells_len = trie.cells.items.len; + const chars_len = trie.chars.items.len; + + // An exact duplicate is a no-op. + try trie.add(testing.allocator, root, "google.com"); + try testing.expectEqual(cells_len, trie.cells.items.len); + try testing.expectEqual(chars_len, trie.chars.items.len); + + // A subdomain of a stored entry could never win over the shorter + // suffix, so it is dropped before allocating anything. + try trie.add(testing.allocator, root, "ads.google.com"); + try testing.expectEqual(cells_len, trie.cells.items.len); + try testing.expectEqual(chars_len, trie.chars.items.len); + + // The reverse order: a later, shorter entry splits the stored segment + // and lands its boundary on the split point. + try trie.add(testing.allocator, root, "mail.example.org"); + try trie.add(testing.allocator, root, "example.org"); + // The shorter suffix now wins for its former subdomain entry. + try testing.expectEqual(5, trie.matches(root, "mail.example.org").?); + try testing.expectEqual(0, trie.matches(root, "example.org").?); +} + +test "adblock.HostnameTrie: segment splitting" { + var trie: HostnameTrie = try .init(testing.allocator); + defer trie.deinit(testing.allocator); + const root = try trie.createTrie(testing.allocator); + + // Reversed, "google.com" and "gle.com" share the prefix "moc.elg": + // adding the shorter one splits the segment mid-label. + try trie.add(testing.allocator, root, "google.com"); + try trie.add(testing.allocator, root, "gle.com"); + try testing.expectEqual(0, trie.matches(root, "google.com").?); + try testing.expectEqual(4, trie.matches(root, "www.google.com").?); + try testing.expectEqual(0, trie.matches(root, "gle.com").?); + try testing.expectEqual(2, trie.matches(root, "x.gle.com").?); + // "oogle.com" reaches the "gle.com" boundary mid-hostname ('o' is not + // a label boundary) and dies in the next segment. + try testing.expectEqual(null, trie.matches(root, "oogle.com")); + try testing.expectEqual(null, trie.matches(root, "e.com")); + + // A non-'.' continuation past a stored boundary is its own entry: + // "xgoogle.com" is not a subdomain of "google.com". + try trie.add(testing.allocator, root, "xgoogle.com"); + try testing.expectEqual(0, trie.matches(root, "xgoogle.com").?); + try testing.expectEqual(2, trie.matches(root, "a.xgoogle.com").?); + try testing.expectEqual(4, trie.matches(root, "sub.google.com").?); +} + +test "adblock.HostnameTrie: sibling chains keep earlier entries intact" { + var trie: HostnameTrie = try .init(testing.allocator); + defer trie.deinit(testing.allocator); + const root = try trie.createTrie(testing.allocator); + + try trie.add(testing.allocator, root, "google.com"); + try trie.add(testing.allocator, root, "awesomes.com"); + + try testing.expectEqual(0, trie.matches(root, "google.com").?); + try testing.expectEqual(0, trie.matches(root, "awesomes.com").?); + try testing.expectEqual(4, trie.matches(root, "sub.awesomes.com").?); + try testing.expectEqual(null, trie.matches(root, "awesome.com")); +} + +test "adblock.HostnameTrie: hostnames longer than one segment" { + var trie: HostnameTrie = try .init(testing.allocator); + defer trie.deinit(testing.allocator); + const root = try trie.createTrie(testing.allocator); + + // 191 bytes: the leaf spans two chained segments (127 + 64). + const long = "a" ** 63 ++ "." ++ "b" ** 63 ++ "." ++ "c" ** 63; + try trie.add(testing.allocator, root, long); + const cells_len = trie.cells.items.len; + try trie.add(testing.allocator, root, long); // duplicate multi-segment entry: no-op + try testing.expectEqual(cells_len, trie.cells.items.len); + try testing.expectEqual(0, trie.matches(root, long).?); + try testing.expectEqual(2, trie.matches(root, "x." ++ long).?); + // A suffix of the stored hostname is not an entry. + try testing.expectEqual(null, trie.matches(root, "b" ** 63 ++ "." ++ "c" ** 63)); +} + +test "adblock.HostnameTrie: stored deeper than query" { + var trie: HostnameTrie = try .init(testing.allocator); + defer trie.deinit(testing.allocator); + const root = try trie.createTrie(testing.allocator); + + try trie.add(testing.allocator, root, "ads.tracking.example.com"); + try testing.expectEqual(null, trie.matches(root, "example.com")); + try testing.expectEqual(null, trie.matches(root, "tracking.example.com")); + try testing.expectEqual(0, trie.matches(root, "ads.tracking.example.com").?); + try testing.expectEqual(2, trie.matches(root, "x.ads.tracking.example.com").?); +} + +test "adblock.HostnameTrie: multiple tries share one container" { + var trie: HostnameTrie = try .init(testing.allocator); + defer trie.deinit(testing.allocator); + const blocked = try trie.createTrie(testing.allocator); + const allowed = try trie.createTrie(testing.allocator); + + try trie.add(testing.allocator, blocked, "ads.example.com"); + try trie.add(testing.allocator, allowed, "cdn.example.com"); + + try testing.expectEqual(0, trie.matches(blocked, "ads.example.com").?); + try testing.expectEqual(null, trie.matches(blocked, "cdn.example.com")); + try testing.expectEqual(0, trie.matches(allowed, "cdn.example.com").?); + try testing.expectEqual(null, trie.matches(allowed, "ads.example.com")); +} + +test "adblock.HostnameTrie: degenerate inputs" { + var trie: HostnameTrie = try .init(testing.allocator); + defer trie.deinit(testing.allocator); + const root = try trie.createTrie(testing.allocator); + + try testing.expectError(error.InvalidHostname, trie.add(testing.allocator, root, "")); + try testing.expectEqual(null, trie.matches(root, "example.com")); + + try trie.add(testing.allocator, root, "example.com"); + // Trailing dot: no stored segment starts with '.'. + try testing.expectEqual(null, trie.matches(root, "example.com.")); + // Byte-faithful like uBO: the walk returns at the boundary before + // ever seeing the malformed empty label. + try testing.expectEqual(3, trie.matches(root, "a..example.com").?); +}