From b0a087450deed696e26b38f0488375243cf5efab Mon Sep 17 00:00:00 2001 From: Navid EMAD Date: Tue, 9 Jun 2026 15:29:48 +0200 Subject: [PATCH 1/8] Surface server-side agent-discovery signals (Link header + Content-Signal) Parse the HTTP `Link:` response header (RFC 8288) and the robots.txt `Content-Signal` directive, surfacing both to the driving agent. - structured_data: extract the registered Link relations (service-doc, service-desc, api) from the frame's retained response headers into a new `linkHeaders` field, flowing out via the structuredData MCP tool and LP.getStructuredData. Emitted only when present. - Robots: capture Content-Signal preferences on a new content_signals field, read back via RobotStore.getContentSignals and the new LP.getContentSignal CDP command. Advisory only - never reaches isAllowed. Refs #2680 --- src/browser/structured_data.zig | 226 ++++++++++++++++++++++++++++++++ src/cdp/domains/lp.zig | 43 ++++++ src/network/Robots.zig | 222 +++++++++++++++++++++++++++++-- 3 files changed, 482 insertions(+), 9 deletions(-) diff --git a/src/browser/structured_data.zig b/src/browser/structured_data.zig index 09cf72a2a..21e063b86 100644 --- a/src/browser/structured_data.zig +++ b/src/browser/structured_data.zig @@ -39,6 +39,13 @@ pub const AlternateLink = struct { title: ?[]const u8, }; +/// A relation discovered in the HTTP `Link:` response header (RFC 8288), +/// restricted to the registered relations an agent can act on. +pub const LinkRel = struct { + rel: []const u8, + href: []const u8, +}; + pub const StructuredData = struct { json_ld: []const []const u8, open_graph: []const Property, @@ -46,6 +53,7 @@ pub const StructuredData = struct { meta: []const Property, links: []const Property, alternate: []const AlternateLink, + link_headers: []const LinkRel, pub fn jsonStringify(self: *const StructuredData, jw: anytype) !void { try jw.beginObject(); @@ -89,6 +97,23 @@ pub const StructuredData = struct { try jw.endArray(); } + // Emitted only when present so existing consumers of the CDP/MCP + // structured-data shape see no new key on sites without a `Link:` + // response header. + if (self.link_headers.len > 0) { + try jw.objectField("linkHeaders"); + try jw.beginArray(); + for (self.link_headers) |lh| { + try jw.beginObject(); + try jw.objectField("rel"); + try jw.write(lh.rel); + try jw.objectField("href"); + try jw.write(lh.href); + try jw.endObject(); + } + try jw.endArray(); + } + try jw.endObject(); } }; @@ -144,6 +169,7 @@ pub fn collectStructuredData( var meta: std.ArrayList(Property) = .empty; var links: std.ArrayList(Property) = .empty; var alternate: std.ArrayList(AlternateLink) = .empty; + var link_headers: std.ArrayList(LinkRel) = .empty; // Extract language from the root element. if (root.is(Element)) |root_el| { @@ -182,6 +208,10 @@ pub fn collectStructuredData( } } + // The `Link:` response header lives on the frame, not in the DOM, so it is + // collected separately from the navigation's retained headers. + try collectLinkHeaders(arena, frame, &link_headers); + return .{ .json_ld = json_ld.items, .open_graph = open_graph.items, @@ -189,9 +219,133 @@ pub fn collectStructuredData( .meta = meta.items, .links = links.items, .alternate = alternate.items, + .link_headers = link_headers.items, }; } +/// Registered link relations we surface from the HTTP `Link:` response header +/// (RFC 8288). `service-doc`/`service-desc` come from RFC 8631; `api` is in the +/// IANA Link Relations registry. These let an agent discover a site's developer +/// docs / API description without scraping. +const header_link_rels = [_][]const u8{ "service-doc", "service-desc", "api" }; + +/// Parse every `Link:` response header retained on the frame, surfacing only the +/// registered, agent-useful relations. Relative targets are resolved against the +/// frame's base URL, mirroring `collectLink`. +fn collectLinkHeaders( + arena: Allocator, + frame: *Frame, + out: *std.ArrayList(LinkRel), +) !void { + for (frame._http_headers.items) |header| { + if (!std.ascii.eqlIgnoreCase(header.name, "link")) continue; + + var it: LinkHeaderIterator = .{ .value = header.value }; + while (it.next()) |entry| { + const rel_value = entry.rel orelse continue; + + // The `rel` parameter is a space-separated list of relation types. + var rel_it = std.mem.tokenizeAny(u8, rel_value, " \t"); + while (rel_it.next()) |rel_token| { + const known = for (header_link_rels) |candidate| { + if (std.ascii.eqlIgnoreCase(rel_token, candidate)) break candidate; + } else continue; + + const href = URL.resolve(arena, frame.base(), entry.target, .{ .encoding = frame.charset }) catch entry.target; + + // Drop duplicate (rel, href) pairs — a target can legitimately + // carry the same relation across repeated headers. + const dup = for (out.items) |existing| { + if (std.mem.eql(u8, existing.rel, known) and std.mem.eql(u8, existing.href, href)) break true; + } else false; + if (!dup) try out.append(arena, .{ .rel = known, .href = href }); + } + } + } +} + +/// One `< target >; param=value; ...` entry of an RFC 8288 `Link` header value. +const LinkHeaderEntry = struct { + target: []const u8, + rel: ?[]const u8, +}; + +/// Splits a `Link:` header field value into its comma-separated link-values, +/// skipping commas that fall inside the `<...>` target or a `"..."` quoted +/// parameter value (RFC 8288 §3). +const LinkHeaderIterator = struct { + value: []const u8, + i: usize = 0, + + fn next(self: *LinkHeaderIterator) ?LinkHeaderEntry { + const v = self.value; + + // Loop (rather than recurse) over malformed segments so stack usage is + // independent of the attacker-controlled header length. + while (true) { + // Skip separators and whitespace between link-values. The comma that + // ended the previous link-value is consumed here. + while (self.i < v.len and (std.ascii.isWhitespace(v[self.i]) or v[self.i] == ',')) : (self.i += 1) {} + if (self.i >= v.len) return null; + + // A link-value must begin with the angle-bracketed target. + if (v[self.i] != '<') { + self.skipToNextComma(); + continue; + } + const target_start = self.i + 1; + const gt = std.mem.indexOfScalarPos(u8, v, target_start, '>') orelse { + self.i = v.len; + return null; + }; + const target = v[target_start..gt]; + self.i = gt + 1; + + // Parameters run to the next top-level comma, left at `self.i`. + const params_start = self.i; + self.skipToNextComma(); + return .{ .target = target, .rel = extractRel(v[params_start..self.i]) }; + } + } + + // Advance until `self.i` sits on the next top-level comma (one outside a + // `"..."` quoted-string, honouring RFC 7230 backslash escapes) or the end + // of the value. The comma itself is left for the leading-separator skip. + fn skipToNextComma(self: *LinkHeaderIterator) void { + const v = self.value; + var in_quotes = false; + while (self.i < v.len) : (self.i += 1) { + const c = v[self.i]; + if (c == '\\' and in_quotes) { + // Skip the escaped byte; guard a trailing backslash. + if (self.i + 1 < v.len) self.i += 1; + } else if (c == '"') { + in_quotes = !in_quotes; + } else if (c == ',' and !in_quotes) { + return; + } + } + } +}; + +/// Return the (unquoted) value of the `rel` parameter from a link-value's +/// parameter list, or null when absent. +fn extractRel(params: []const u8) ?[]const u8 { + var it = std.mem.splitScalar(u8, params, ';'); + while (it.next()) |raw_param| { + const param = std.mem.trim(u8, raw_param, &std.ascii.whitespace); + const eq = std.mem.indexOfScalar(u8, param, '=') orelse continue; + if (!std.ascii.eqlIgnoreCase(std.mem.trim(u8, param[0..eq], &std.ascii.whitespace), "rel")) continue; + + var val = std.mem.trim(u8, param[eq + 1 ..], &std.ascii.whitespace); + if (val.len >= 2 and val[0] == '"' and val[val.len - 1] == '"') { + val = val[1 .. val.len - 1]; + } + return val; + } + return null; +} + fn collectJsonLd( el: *Element, arena: Allocator, @@ -484,6 +638,78 @@ test "structured_data: charset and http-equiv" { try testing.expectEqual("text/html; charset=utf-8", findProperty(data.meta, "Content-Type").?); } +test "structured_data: parseLinkHeader - single link with quoted rel" { + var it: LinkHeaderIterator = .{ .value = + \\; rel="service-desc" + }; + const entry = it.next().?; + try testing.expectString("https://api.example.com/openapi.json", entry.target); + try testing.expectString("service-desc", entry.rel.?); + try testing.expectEqual(null, it.next()); +} + +test "structured_data: parseLinkHeader - multiple links and unquoted rel" { + var it: LinkHeaderIterator = .{ .value = + \\; rel=service-doc, ; rel="stylesheet"; type="text/css" + }; + const first = it.next().?; + try testing.expectString("https://docs.example.com/", first.target); + try testing.expectString("service-doc", first.rel.?); + + const second = it.next().?; + try testing.expectString("/style.css", second.target); + // A comma inside the quoted type param must not split the link-value. + try testing.expectString("stylesheet", second.rel.?); + + try testing.expectEqual(null, it.next()); +} + +test "structured_data: parseLinkHeader - escaped quote in param does not split link" { + // A backslash-escaped quote inside an earlier param must not terminate the + // quoted-string, so the comma inside it is not treated as a separator. + var it: LinkHeaderIterator = .{ .value = + \\; title="x\",y"; rel="service-doc" + }; + const entry = it.next().?; + try testing.expectString("https://a/", entry.target); + try testing.expectString("service-doc", entry.rel.?); + try testing.expectEqual(null, it.next()); +} + +test "structured_data: link headers from response" { + const frame = try testing.test_session.createPage(); + defer testing.test_session.removePage(); + + // Stand in for what frameHeaderDoneCallback records from the navigation. + try frame._http_headers.append(frame.arena, .{ .name = "Link", .value = + \\; rel="service-doc" + }); + try frame._http_headers.append(frame.arena, .{ .name = "link", .value = + \\; rel="service-desc", ; rel="stylesheet" + }); + + const doc = frame.window._document; + const div = try doc.createElement("div", null, frame); + try frame.parseHtmlAsChildren(div.asNode(), "Example"); + + const data = try collectStructuredData(div.asNode(), frame.call_arena, frame); + + // service-doc + service-desc are surfaced; stylesheet is filtered out. + try testing.expectEqual(2, data.link_headers.len); + try testing.expectString("service-doc", data.link_headers[0].rel); + try testing.expectString("https://docs.example.com/", data.link_headers[0].href); + try testing.expectString("service-desc", data.link_headers[1].rel); + try testing.expectString("https://api.example.com/spec", data.link_headers[1].href); +} + +test "structured_data: no link header yields empty list" { + const data = try testStructuredData( + \\ + ); + defer testing.test_session.removePage(); + try testing.expectEqual(0, data.link_headers.len); +} + test "structured_data: mixed content" { const data = try testStructuredData( \\My Site diff --git a/src/cdp/domains/lp.zig b/src/cdp/domains/lp.zig index b3b3aec0a..dd8fdd991 100644 --- a/src/cdp/domains/lp.zig +++ b/src/cdp/domains/lp.zig @@ -23,6 +23,7 @@ const CDP = @import("../CDP.zig"); const Node = @import("../Node.zig"); const DOMNode = @import("../../browser/webapi/Node.zig"); +const Robots = @import("../../network/Robots.zig"); const markdown = lp.markdown; const SemanticTree = lp.SemanticTree; @@ -36,6 +37,7 @@ pub fn processMessage(cmd: *CDP.Command) !void { getInteractiveElements, getNodeDetails, getStructuredData, + getContentSignal, detectForms, clickNode, fillNode, @@ -51,6 +53,7 @@ pub fn processMessage(cmd: *CDP.Command) !void { .getInteractiveElements => return getInteractiveElements(cmd), .getNodeDetails => return getNodeDetails(cmd), .getStructuredData => return getStructuredData(cmd), + .getContentSignal => return getContentSignal(cmd), .detectForms => return detectForms(cmd), .clickNode => return clickNode(cmd), .fillNode => return fillNode(cmd), @@ -201,6 +204,27 @@ fn getStructuredData(cmd: anytype) !void { }, .{}); } +// Advisory `Content-Signal` robots.txt preferences for the current document's +// host (https://contentsignals.org). `available` is false when no robots.txt +// has been fetched for the host — note this is the case unless `obey_robots` +// is enabled, since the robots layer is what populates the store. +fn getContentSignal(cmd: anytype) !void { + const bc = cmd.browser_context orelse return error.NoBrowserContext; + const frame = bc.session.currentFrame() orelse return error.FrameNotLoaded; + const network = bc.cdp.browser.http_client.network; + + const empty: []const Robots.ContentSignal = &.{}; + const robots_url = lp.URL.getRobotsUrl(cmd.arena, frame.url) catch { + return cmd.sendResult(.{ .available = false, .contentSignals = empty }, .{}); + }; + + const signals = network.robot_store.getContentSignals(robots_url); + return cmd.sendResult(.{ + .available = signals != null, + .contentSignals = signals orelse empty, + }, .{}); +} + fn detectForms(cmd: anytype) !void { const bc = cmd.browser_context orelse return error.NoBrowserContext; const frame = bc.session.currentFrame() orelse return error.FrameNotLoaded; @@ -410,6 +434,25 @@ test "cdp.lp: getStructuredData" { try testing.expect(result.get("structuredData") != null); } +test "cdp.lp: getContentSignal" { + var ctx = try testing.context(); + defer ctx.deinit(); + + const bc = try ctx.loadBrowserContext(.{}); + _ = try bc.session.createPage(); + + try ctx.processMessage(.{ + .id = 1, + .method = "LP.getContentSignal", + }); + + // No robots.txt fetched for the page, so the directive is reported absent + // rather than erroring — the command is still wired and returns the shape. + const result = (try ctx.getSentMessage(0)).?.object.get("result").?.object; + try testing.expectEqual(false, result.get("available").?.bool); + try testing.expect(result.get("contentSignals") != null); +} + test "cdp.lp: action tools" { var ctx = try testing.context(); defer ctx.deinit(); diff --git a/src/network/Robots.zig b/src/network/Robots.zig index 202e0a5c7..423a308cf 100644 --- a/src/network/Robots.zig +++ b/src/network/Robots.zig @@ -72,11 +72,20 @@ pub const Key = enum { @"user-agent", allow, disallow, + @"content-signal", +}; + +/// A declared content-usage preference from a `Content-Signal:` robots.txt +/// directive (https://contentsignals.org), e.g. name="ai-train" value="no". +/// Advisory metadata surfaced to the agent — it never affects crawl gating. +pub const ContentSignal = struct { + name: []const u8, + value: []const u8, }; /// https://www.rfc-editor.org/rfc/rfc9309.html pub const Robots = @This(); -pub const empty: Robots = .{ .rules = &.{} }; +pub const empty: Robots = .{ .rules = &.{}, .content_signals = &.{} }; pub const RobotStore = struct { const RobotsEntry = union(enum) { @@ -156,6 +165,21 @@ pub const RobotStore = struct { try self.map.put(self.allocator, duped, .{ .present = robots }); } + /// Advisory `Content-Signal` preferences declared for `url`'s host, or null + /// when no robots.txt has been fetched/stored for it. The returned slice is + /// owned by the store; callers must read it synchronously (and dupe to keep + /// it) and must never free it. + pub fn getContentSignals(self: *RobotStore, url: []const u8) ?[]const ContentSignal { + self.mutex.lock(); + defer self.mutex.unlock(); + + const entry = self.map.get(url) orelse return null; + return switch (entry) { + .present => |robots| robots.content_signals, + .absent => null, + }; + } + pub fn putAbsent(self: *RobotStore, url: []const u8) !void { self.mutex.lock(); defer self.mutex.unlock(); @@ -166,6 +190,15 @@ pub const RobotStore = struct { }; rules: []const Rule, +content_signals: []const ContentSignal = &.{}, + +/// Result of parsing a robots.txt for one user-agent: the applicable crawl +/// rules plus any advisory `Content-Signal` preferences. The two are kept +/// distinct so content signals can never reach `isAllowed`. +const ParseResult = struct { + rules: []Rule, + content_signals: []ContentSignal, +}; const State = struct { entry: enum { @@ -186,17 +219,56 @@ fn freeRulesInList(allocator: std.mem.Allocator, rules: []const Rule) void { } } +fn freeContentSignals(allocator: std.mem.Allocator, signals: []const ContentSignal) void { + for (signals) |signal| { + allocator.free(signal.name); + allocator.free(signal.value); + } +} + +/// Parse a `Content-Signal:` value — a comma-separated list of `name=value` +/// pairs (e.g. `ai-train=no, search=yes`) — into the given list. Names and +/// values are lower-cased and owned by `allocator`. +fn appendContentSignals( + allocator: std.mem.Allocator, + list: *std.ArrayList(ContentSignal), + value: []const u8, +) !void { + var iter = std.mem.splitScalar(u8, value, ','); + while (iter.next()) |raw_pair| { + const pair = std.mem.trim(u8, raw_pair, &std.ascii.whitespace); + const eq = std.mem.indexOfScalar(u8, pair, '=') orelse continue; + + const name_raw = std.mem.trim(u8, pair[0..eq], &std.ascii.whitespace); + const value_raw = std.mem.trim(u8, pair[eq + 1 ..], &std.ascii.whitespace); + if (name_raw.len == 0) continue; + + const name = try std.ascii.allocLowerString(allocator, name_raw); + errdefer allocator.free(name); + const val = try std.ascii.allocLowerString(allocator, value_raw); + errdefer allocator.free(val); + + try list.append(allocator, .{ .name = name, .value = val }); + } +} + fn parseRulesWithUserAgent( allocator: std.mem.Allocator, user_agent: []const u8, raw_bytes: []const u8, -) ![]Rule { +) !ParseResult { var rules: std.ArrayList(Rule) = .empty; defer rules.deinit(allocator); var wildcard_rules: std.ArrayList(Rule) = .empty; defer wildcard_rules.deinit(allocator); + var content_signals: std.ArrayList(ContentSignal) = .empty; + defer content_signals.deinit(allocator); + + var wildcard_content_signals: std.ArrayList(ContentSignal) = .empty; + defer wildcard_content_signals.deinit(allocator); + var state: State = .{ .entry = .not_in_entry, .has_rules = false }; // https://en.wikipedia.org/wiki/Byte_order_mark @@ -308,22 +380,51 @@ fn parseRulesWithUserAgent( }, } }, + .@"content-signal" => { + // A group member like allow/disallow, so it sets has_rules + // (a following `User-agent:` opens a new group), but it is + // captured into a separate list and never reaches isAllowed. + defer state.has_rules = true; + + switch (state.entry) { + .in_our_entry => try appendContentSignals(allocator, &content_signals, value), + .in_wildcard_entry => try appendContentSignals(allocator, &wildcard_content_signals, value), + .in_other_entry, .not_in_entry => {}, + } + }, } } // If we have rules for our specific User-Agent, we will use those rules. // If we don't have any rules, we fallback to using the wildcard ("*") rules. - if (rules.items.len > 0) { + const out_rules = if (rules.items.len > 0) blk: { freeRulesInList(allocator, wildcard_rules.items); - return try rules.toOwnedSlice(allocator); - } else { + break :blk try rules.toOwnedSlice(allocator); + } else blk: { freeRulesInList(allocator, rules.items); - return try wildcard_rules.toOwnedSlice(allocator); + break :blk try wildcard_rules.toOwnedSlice(allocator); + }; + errdefer { + freeRulesInList(allocator, out_rules); + allocator.free(out_rules); } + + // Content signals follow the same specific-else-wildcard precedence, + // chosen independently of the rule selection above. + const out_signals = if (content_signals.items.len > 0) blk: { + freeContentSignals(allocator, wildcard_content_signals.items); + break :blk try content_signals.toOwnedSlice(allocator); + } else blk: { + freeContentSignals(allocator, content_signals.items); + break :blk try wildcard_content_signals.toOwnedSlice(allocator); + }; + + return .{ .rules = out_rules, .content_signals = out_signals }; } pub fn fromBytes(allocator: std.mem.Allocator, user_agent: []const u8, bytes: []const u8) !Robots { - const rules = try parseRulesWithUserAgent(allocator, user_agent, bytes); + const parsed = try parseRulesWithUserAgent(allocator, user_agent, bytes); + const rules = parsed.rules; // sort by order once. std.mem.sort(Rule, rules, {}, struct { @@ -357,12 +458,14 @@ pub fn fromBytes(allocator: std.mem.Allocator, user_agent: []const u8, bytes: [] } }.lessThan); - return .{ .rules = rules }; + return .{ .rules = rules, .content_signals = parsed.content_signals }; } pub fn deinit(self: *Robots, allocator: std.mem.Allocator) void { freeRulesInList(allocator, self.rules); allocator.free(self.rules); + freeContentSignals(allocator, self.content_signals); + allocator.free(self.content_signals); } /// There are rules for how the pattern in robots.txt should be matched. @@ -476,10 +579,13 @@ test "Robots: simple robots.txt" { \\ ; - const rules = try parseRulesWithUserAgent(allocator, "GoogleBot", file); + const parsed = try parseRulesWithUserAgent(allocator, "GoogleBot", file); + const rules = parsed.rules; defer { freeRulesInList(allocator, rules); allocator.free(rules); + freeContentSignals(allocator, parsed.content_signals); + allocator.free(parsed.content_signals); } try std.testing.expectEqual(1, rules.len); @@ -1003,3 +1109,101 @@ test "Robots: blank lines don't end entries" { try std.testing.expect(robots.isAllowed("/admin/") == false); try std.testing.expect(robots.isAllowed("/public/") == true); } + +test "Robots: content-signal captured and gating unaffected" { + const allocator = std.testing.allocator; + + var robots = try Robots.fromBytes(allocator, "MyBot", + \\User-agent: * + \\Content-Signal: ai-train=no, search=yes, ai-input=yes + \\Disallow: /private/ + \\ + ); + defer robots.deinit(allocator); + + try std.testing.expectEqual(3, robots.content_signals.len); + try std.testing.expectEqualStrings("ai-train", robots.content_signals[0].name); + try std.testing.expectEqualStrings("no", robots.content_signals[0].value); + try std.testing.expectEqualStrings("search", robots.content_signals[1].name); + try std.testing.expectEqualStrings("yes", robots.content_signals[1].value); + try std.testing.expectEqualStrings("ai-input", robots.content_signals[2].name); + + // The whole point: Content-Signal is advisory metadata, it must not move + // the crawl gate. Disallow/Allow still decide isAllowed. + try std.testing.expect(robots.isAllowed("/private/page") == false); + try std.testing.expect(robots.isAllowed("/public/page") == true); +} + +test "Robots: content-signal mixed casing is normalized" { + const allocator = std.testing.allocator; + + var robots = try Robots.fromBytes(allocator, "MyBot", + \\User-agent: * + \\Content-Signal: AI-Train=NO + \\ + ); + defer robots.deinit(allocator); + + try std.testing.expectEqual(1, robots.content_signals.len); + try std.testing.expectEqualStrings("ai-train", robots.content_signals[0].name); + try std.testing.expectEqualStrings("no", robots.content_signals[0].value); +} + +test "Robots: content-signal absent yields empty list" { + const allocator = std.testing.allocator; + + var robots = try Robots.fromBytes(allocator, "MyBot", + \\User-agent: * + \\Disallow: /admin/ + \\ + ); + defer robots.deinit(allocator); + + try std.testing.expectEqual(0, robots.content_signals.len); + try std.testing.expect(robots.isAllowed("/admin/") == false); +} + +test "Robots: content-signal prefers specific user-agent over wildcard" { + const allocator = std.testing.allocator; + + const file = + \\User-agent: * + \\Content-Signal: ai-train=yes + \\ + \\User-agent: MyBot + \\Content-Signal: ai-train=no + \\ + ; + + var mine = try Robots.fromBytes(allocator, "MyBot", file); + defer mine.deinit(allocator); + try std.testing.expectEqual(1, mine.content_signals.len); + try std.testing.expectEqualStrings("no", mine.content_signals[0].value); + + var other = try Robots.fromBytes(allocator, "OtherBot", file); + defer other.deinit(allocator); + try std.testing.expectEqual(1, other.content_signals.len); + try std.testing.expectEqualStrings("yes", other.content_signals[0].value); +} + +test "Robots: RobotStore.getContentSignals round-trips" { + const allocator = std.testing.allocator; + + var store = RobotStore.init(allocator); + defer store.deinit(); + + const robots = try store.robotsFromBytes("MyBot", + \\User-agent: * + \\Content-Signal: ai-train=no + \\ + ); + try store.put("https://example.com/robots.txt", robots); + + const signals = store.getContentSignals("https://example.com/robots.txt").?; + try std.testing.expectEqual(1, signals.len); + try std.testing.expectEqualStrings("ai-train", signals[0].name); + try std.testing.expectEqualStrings("no", signals[0].value); + + // Unknown host has no stored robots. + try std.testing.expectEqual(null, store.getContentSignals("https://other.com/robots.txt")); +} From 7c1af5762d7c11eca082b349616ff3afdc3ae264 Mon Sep 17 00:00:00 2001 From: Navid EMAD Date: Thu, 11 Jun 2026 13:44:08 +0200 Subject: [PATCH 2/8] webapi: run sequential focus navigation on Tab MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A synthetic Tab delivered via Input.dispatchKeyEvent did not move document.activeElement. Two gaps in Frame.zig caused this: handleKeydown (the keydown default-action handler) had no Tab branch, and triggerKeyboard dropped the keydown entirely when nothing was focused — it read the raw _active_element field, which is null until an element is explicitly focused. The first Tab on a page therefore went nowhere, and keyboard focus traversal was impossible over CDP. Dispatch the keydown to the effective active element (getActiveElement(), which resolves to when nothing is focused) so the event fires and its default action can run, and add a Tab branch to handleKeydown that performs HTML sequential focus navigation: move focus to the next (Shift+Tab: previous) focusable element ordered by tabindex, then document position. The order is computed from tabindex + tree position, so no layout is needed. Closes #2699 --- src/browser/Frame.zig | 99 ++++++++++++++++++++++++++++++++++++++- src/cdp/domains/input.zig | 56 ++++++++++++++++++++++ 2 files changed, 154 insertions(+), 1 deletion(-) diff --git a/src/browser/Frame.zig b/src/browser/Frame.zig index bbe669fc2..b446429ae 100644 --- a/src/browser/Frame.zig +++ b/src/browser/Frame.zig @@ -4167,7 +4167,11 @@ pub fn handleClick(self: *Frame, target: *Node) !void { pub fn triggerKeyboard(self: *Frame, keyboard_event: *KeyboardEvent) !void { const event = keyboard_event.asEvent(); - const element = self.window._document._active_element orelse { + // Dispatch to the effective active element. When nothing is explicitly + // focused this resolves to (matching `document.activeElement`), so + // the keydown still fires and its default action — e.g. sequential focus + // navigation on Tab — can run. + const element = self.window._document.getActiveElement() orelse { event.deinit(self._page); return; }; @@ -4191,6 +4195,12 @@ pub fn handleKeydown(self: *Frame, target: *Node, event: *Event) !void { return; } + if (key == .Tab) { + // Sequential focus navigation is the default action of a Tab keydown. + // Shift+Tab walks backward. + return self.moveFocus(if (keyboard_event.getShiftKey()) .backward else .forward); + } + if (target.is(Element.Html.Input)) |input| { if (key == .Enter) { return self.submitForm(input.asElement(), input.getForm(self), .{}); @@ -4221,6 +4231,93 @@ pub fn handleKeydown(self: *Frame, target: *Node, event: *Event) !void { } } +const FocusDirection = enum { forward, backward }; + +// Sequential focus navigation: move `document.activeElement` to the next (Tab) +// or previous (Shift+Tab) focusable element, firing the usual blur/focus events +// via `Element.focus`. The order is fully determined by tabindex + document +// position, so no layout is needed: +// 1. elements with a positive tabindex, in ascending tabindex order; +// 2. then elements with tabindex 0 (or a natively-focusable default), in +// document order. +// Ties within a group break on document order, and Tab wraps around at the ends. +// https://html.spec.whatwg.org/multipage/interaction.html#sequential-focus-navigation +fn moveFocus(self: *Frame, direction: FocusDirection) !void { + const document = self.document; + const current = document._active_element; + const forward = direction == .forward; + + // Single document-order pass tracking two candidates: + // chosen — the closest focusable element strictly past `current` in the + // travel direction. + // edge — the global first (forward) / last (backward) focusable element, + // used to wrap around when `current` is at an end, or as the + // landing spot when nothing is focused yet. + var chosen: ?*Element = null; + var edge: ?*Element = null; + + var tw = @import("webapi/TreeWalker.zig").Full.Elements.init(document.asNode(), .{}); + while (tw.next()) |el| { + if (!sequentiallyFocusable(el)) continue; + + if (edge == null or focusOrderBefore(el, edge.?) == forward) { + edge = el; + } + + const cur = current orelse continue; + if (el == cur) continue; + const past = if (forward) focusOrderBefore(cur, el) else focusOrderBefore(el, cur); + if (!past) continue; + if (chosen == null or focusOrderBefore(el, chosen.?) == forward) { + chosen = el; + } + } + + const next = chosen orelse edge orelse return; + try next.focus(self); +} + +// True when `el` participates in sequential focus navigation: enabled, with a +// non-negative tabindex, and either an explicit tabindex or a natively +// focusable shape. `getTabIndex` defaults several tags to 0, including +// href-less anchors and hidden inputs, so the structural conditions are +// re-checked here. +fn sequentiallyFocusable(el: *Element) bool { + const html_el = el.is(Element.Html) orelse return false; + if (el.isDisabled()) return false; + if (html_el.getTabIndex() < 0) return false; + + if (el.getAttributeSafe(comptime .wrap("tabindex")) != null) return true; + + return switch (el.getTag()) { + .button, .select, .textarea, .iframe => true, + .input => if (el.is(Element.Html.Input)) |input| input._input_type != .hidden else false, + .anchor, .area => el.getAttributeSafe(comptime .wrap("href")) != null, + else => false, + }; +} + +// Orders two focusable elements by sequential focus navigation order: positive +// tabindex first (ascending), then tabindex 0, ties broken by document order. +fn focusOrderBefore(a: *Element, b: *Element) bool { + const ta = tabOrderIndex(a); + const tb = tabOrderIndex(b); + if (ta != tb) { + const group_a: u8 = if (ta > 0) 0 else 1; + const group_b: u8 = if (tb > 0) 0 else 1; + if (group_a != group_b) return group_a < group_b; + return ta < tb; + } + // Equal tabindex → document order: `a` precedes `b` when `b` follows `a`. + const FOLLOWING: u16 = 0x04; + return (a.asNode().compareDocumentPosition(b.asNode()) & FOLLOWING) != 0; +} + +fn tabOrderIndex(el: *Element) i32 { + const html_el = el.is(Element.Html) orelse return 0; + return html_el.getTabIndex(); +} + const SubmitFormOpts = struct { fire_event: bool = true, }; diff --git a/src/cdp/domains/input.zig b/src/cdp/domains/input.zig index a6772abd3..a865ee344 100644 --- a/src/cdp/domains/input.zig +++ b/src/cdp/domains/input.zig @@ -314,3 +314,59 @@ test "cdp.input: dispatchMouseEvent right button fires contextmenu, double-click const result = try ls.local.compileAndRun("window.downButton === 2 && window.ctxButton === 2 && window.dbl === true", null); try testing.expect(result.isTrue()); } + +test "cdp.input: dispatchKeyEvent Tab runs sequential focus navigation" { + var ctx = try testing.context(); + defer ctx.deinit(); + + const bc = try ctx.loadBrowserContext(.{}); + const frame = try bc.session.createPage(); + const url = "http://localhost:9582/src/browser/tests/mcp_actions.html"; + try frame.navigate(url, .{ .reason = .address_bar, .kind = .{ .push = null } }); + var runner = try bc.session.runner(.{}); + try runner.wait(.{ .ms = 2000 }); + + var ls: lp.js.Local.Scope = undefined; + frame.js.localScope(&ls); + defer ls.deinit(); + + var try_catch: lp.js.TryCatch = undefined; + try_catch.init(&ls.local); + defer try_catch.deinit(); + + // Three controls whose tabindex order (1, 2, 3) differs from document + // order (2, 1, 3): focus order must follow tabindex, not the tree. + _ = try ls.local.compileAndRun( + \\document.body.innerHTML = + \\ '' + + \\ '' + + \\ ''; + , null); + + // Nothing focused yet → activeElement is . + try testing.expect((try ls.local.compileAndRun("document.activeElement === document.body", null)).isTrue()); + + // First Tab → lowest positive tabindex (#b1), regardless of document order. + try ctx.processMessage(.{ + .id = 1, + .method = "Input.dispatchKeyEvent", + .params = .{ .type = "keyDown", .key = "Tab", .code = "Tab" }, + }); + try testing.expect((try ls.local.compileAndRun("document.activeElement.id === 'b1'", null)).isTrue()); + + // Second Tab → next in tabindex order (#i2). + try ctx.processMessage(.{ + .id = 2, + .method = "Input.dispatchKeyEvent", + .params = .{ .type = "keyDown", .key = "Tab", .code = "Tab" }, + }); + try testing.expect((try ls.local.compileAndRun("document.activeElement.id === 'i2'", null)).isTrue()); + + // Shift+Tab (modifiers bit 8) walks backward → back to #b1. + try ctx.processMessage(.{ + .id = 3, + .method = "Input.dispatchKeyEvent", + .params = .{ .type = "keyDown", .key = "Tab", .code = "Tab", .modifiers = 8 }, + }); + try testing.expect((try ls.local.compileAndRun("document.activeElement.id === 'b1'", null)).isTrue()); +} From 79dfdf7465575d5617dfeadce4ebee3d53f43d32 Mon Sep 17 00:00:00 2001 From: Scott Taylor Date: Thu, 11 Jun 2026 12:31:14 -0400 Subject: [PATCH 3/8] fix(cdp): let setExtraHTTPHeaders override built-in headers like User-Agent Network.setExtraHTTPHeaders applied caller headers with a plain curl_slist append. Requests are seeded with a default `User-Agent: Lightpanda/1.0`, so a caller-supplied User-Agent produced two User-Agent entries; libcurl keeps the first and the override was silently dropped (origins always saw `Lightpanda/1.0`). Other headers worked because they had no default to collide with. Add Headers.set, which replaces any existing header with the same (case-insensitive) name before appending, and use it when applying the CDP extra headers. This matches Chrome, where Network.setExtraHTTPHeaders overrides default request headers. Refs #2704 --- src/cdp/domains/network.zig | 6 ++- src/network/http.zig | 75 +++++++++++++++++++++++++++++++++++++ 2 files changed, 79 insertions(+), 2 deletions(-) diff --git a/src/cdp/domains/network.zig b/src/cdp/domains/network.zig index dea679989..cd3e55f28 100644 --- a/src/cdp/domains/network.zig +++ b/src/cdp/domains/network.zig @@ -307,9 +307,11 @@ pub fn httpRequestStart(bc: *CDP.BrowserContext, msg: *const Notification.Reques const frame_id = req.frame_id; const frame = bc.session.findFrameByFrameId(frame_id) orelse return; - // Modify request with extra CDP headers + // Modify request with extra CDP headers. Use set (replace by name) so a + // caller-supplied header overrides a built-in default of the same name + // (e.g. User-Agent) instead of producing a duplicate libcurl drops. for (bc.extra_headers.items) |extra| { - try req.headers.add(extra); + try req.headers.set(extra); } // We're missing a bunch of fields, but, for now, this eems like enough diff --git a/src/network/http.zig b/src/network/http.zig index bec25af53..3664bac48 100644 --- a/src/network/http.zig +++ b/src/network/http.zig @@ -99,6 +99,34 @@ pub const Headers = struct { self.headers = updated_headers; } + // Adds `header` ("Name: Value"), replacing any existing header with the + // same case-insensitive name. Caller-supplied headers (CDP + // Network.setExtraHTTPHeaders) must override built-in defaults like + // User-Agent; a plain append produces a duplicate that libcurl silently + // collapses to the first occurrence, so the override would be dropped. + pub fn set(self: *Headers, header: [*c]const u8) !void { + const new = parseHeader(std.mem.span(@as([*:0]const u8, @ptrCast(header)))) orelse { + // No colon: nothing to match against, fall back to append. + return self.add(header); + }; + + var rebuilt: ?*libcurl.CurlSList = null; + errdefer libcurl.curl_slist_free_all(rebuilt); + + var node = self.headers; + while (node) |n| : (node = n.*.next) { + const data = @as([*:0]const u8, @ptrCast(n.*.data)); + if (parseHeader(std.mem.span(data))) |existing| { + if (std.ascii.eqlIgnoreCase(existing.name, new.name)) continue; + } + rebuilt = libcurl.curl_slist_append(rebuilt, data) orelse return error.OutOfMemory; + } + rebuilt = libcurl.curl_slist_append(rebuilt, header) orelse return error.OutOfMemory; + + libcurl.curl_slist_free_all(self.headers); + self.headers = rebuilt; + } + pub fn parseHeader(header_str: []const u8) ?Header { const colon_pos = std.mem.indexOfScalar(u8, header_str, ':') orelse return null; @@ -689,6 +717,53 @@ fn makeSockAddrV4(ip: [4]u8) libcurl.CurlSockAddr { } const testing = @import("../testing.zig"); + +fn findHeader(headers: Headers, name: []const u8) struct { count: usize, value: []const u8 } { + var count: usize = 0; + var value: []const u8 = ""; + var it = headers.iterator(); + while (it.next()) |h| { + if (std.ascii.eqlIgnoreCase(h.name, name)) { + count += 1; + value = h.value; + } + } + return .{ .count = count, .value = value }; +} + +test "Headers.set replaces an existing header instead of duplicating it" { + var headers = try Headers.init("User-Agent: Lightpanda/1.0"); + defer headers.deinit(); + + try headers.set("User-Agent: Custom/1.0"); + + const ua = findHeader(headers, "User-Agent"); + try testing.expectEqual(@as(usize, 1), ua.count); + try testing.expectString("Custom/1.0", ua.value); +} + +test "Headers.set matches header names case-insensitively" { + var headers = try Headers.init("User-Agent: Lightpanda/1.0"); + defer headers.deinit(); + + try headers.set("user-agent: Custom/1.0"); + + const ua = findHeader(headers, "User-Agent"); + try testing.expectEqual(@as(usize, 1), ua.count); + try testing.expectString("Custom/1.0", ua.value); +} + +test "Headers.set adds a new header and preserves defaults" { + var headers = try Headers.init("User-Agent: Lightpanda/1.0"); + defer headers.deinit(); + + try headers.set("X-Custom: yes"); + + try testing.expectEqual(@as(usize, 1), findHeader(headers, "X-Custom").count); + try testing.expectEqual(@as(usize, 1), findHeader(headers, "User-Agent").count); + try testing.expectEqual(@as(usize, 1), findHeader(headers, "Accept-Language").count); +} + test "opensocketCallback: private IPv4 returns CURL_SOCKET_BAD" { const lf: testing.LogFilter = .init(&.{.http}); defer lf.deinit(); From 723e7eb9d5bfeea3d86054d2da29878441f6e071 Mon Sep 17 00:00:00 2001 From: Scott Taylor Date: Fri, 12 Jun 2026 12:20:55 -0400 Subject: [PATCH 4/8] fix(cdp): validate User-Agent in Network.setExtraHTTPHeaders A caller-supplied User-Agent overrides the built-in default and goes out on the wire, so reject non-printable values the same way Emulation.setUserAgentOverride and the --user-agent flag do. We deliberately do NOT apply validateUserAgent's reserved (Mozilla) check here: unlike the Emulation override (kept Lightpanda-branded to keep go-rod working), setExtraHTTPHeaders is the intended, Chrome-compatible escape hatch for setting a real browser-like User-Agent, which by definition contains Mozilla. Refs #2704 --- src/cdp/domains/network.zig | 57 +++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/src/cdp/domains/network.zig b/src/cdp/domains/network.zig index cd3e55f28..c3874262c 100644 --- a/src/cdp/domains/network.zig +++ b/src/cdp/domains/network.zig @@ -108,6 +108,16 @@ fn setExtraHTTPHeaders(cmd: *CDP.Command) !void { try extra_headers.ensureTotalCapacity(arena, params.headers.map.count()); var it = params.headers.map.iterator(); while (it.next()) |header| { + // Reject a non-printable User-Agent, but (unlike + // Emulation.setUserAgentOverride) allow Mozilla: this is the + // intended escape hatch for a real browser-like UA (#2704). + if (std.ascii.eqlIgnoreCase(header.key_ptr.*, "user-agent")) { + for (header.value_ptr.*) |c| { + if (!std.ascii.isPrint(c)) { + return cmd.sendError(-32602, "User agent contains non-printable characters", .{}); + } + } + } const header_string = try std.fmt.allocPrintSentinel(arena, "{s}: {s}", .{ header.key_ptr.*, header.value_ptr.* }, 0); extra_headers.appendAssumeCapacity(header_string); } @@ -554,6 +564,53 @@ test "cdp.network setExtraHTTPHeaders" { try testing.expectEqual(bc.extra_headers.items.len, 1); } +test "cdp.network setExtraHTTPHeaders rejects non-printable User-Agent" { + var ctx = try testing.context(); + defer ctx.deinit(); + + _ = try ctx.loadBrowserContext(.{ .id = "NID-UA1", .session_id = "NESI-UA1" }); + + try ctx.processMessage(.{ + .id = 3, + .method = "Network.setExtraHTTPHeaders", + .params = .{ .headers = .{ .@"User-Agent" = "Bot/1.0\x01hidden" } }, + }); + + try ctx.expectSentError(-32602, "User agent contains non-printable characters", .{ .id = 3 }); +} + +test "cdp.network setExtraHTTPHeaders allows a Mozilla User-Agent" { + var ctx = try testing.context(); + defer ctx.deinit(); + + _ = try ctx.loadBrowserContext(.{ .id = "NID-UA2", .session_id = "NESI-UA2" }); + + try ctx.processMessage(.{ + .id = 3, + .method = "Network.setExtraHTTPHeaders", + .params = .{ .headers = .{ .@"User-Agent" = "Mozilla/5.0" } }, + }); + + const bc = ctx.cdp().browser_context.?; + try testing.expectEqual(bc.extra_headers.items.len, 1); +} + +test "cdp.network setExtraHTTPHeaders accepts valid User-Agent" { + var ctx = try testing.context(); + defer ctx.deinit(); + + _ = try ctx.loadBrowserContext(.{ .id = "NID-UA3", .session_id = "NESI-UA3" }); + + try ctx.processMessage(.{ + .id = 3, + .method = "Network.setExtraHTTPHeaders", + .params = .{ .headers = .{ .@"User-Agent" = "CustomBot/2.0" } }, + }); + + const bc = ctx.cdp().browser_context.?; + try testing.expectEqual(bc.extra_headers.items.len, 1); +} + test "cdp.Network: cookies" { const ResCookie = CdpStorage.ResCookie; const CdpCookie = CdpStorage.CdpCookie; From b957ad74c465408a1d2c6f633b575c19e4f08951 Mon Sep 17 00:00:00 2001 From: Karl Seguin Date: Sat, 13 Jun 2026 15:43:38 +0800 Subject: [PATCH 5/8] cdp, bot: Apply user-agent validation to CDP's setExtraHTTPHeaders This is a change to https://github.com/lightpanda-io/browser/pull/2706 which applies the Config.validateUserAgent to CDP's network.setExtraHTTPHeaders. This makes the CDP method consistent with all other forms of user-agent setting, i.e not allowing 'mozilla'-container values. --- src/cdp/domains/network.zig | 93 +++++++++++++++++++++++++++++++------ 1 file changed, 79 insertions(+), 14 deletions(-) diff --git a/src/cdp/domains/network.zig b/src/cdp/domains/network.zig index c3874262c..fcaa2c876 100644 --- a/src/cdp/domains/network.zig +++ b/src/cdp/domains/network.zig @@ -22,10 +22,12 @@ const lp = @import("lightpanda"); const id = @import("../id.zig"); const CDP = @import("../CDP.zig"); +const Config = @import("../../Config.zig"); const URL = @import("../../browser/URL.zig"); const Mime = @import("../../browser/Mime.zig"); const Notification = @import("../../Notification.zig"); const timestamp = @import("../../datetime.zig").timestamp; +const Headers = @import("../../browser/HttpClient.zig").Headers; const Transfer = @import("../../browser/HttpClient.zig").Transfer; const Response = @import("../../browser/HttpClient.zig").Response; @@ -108,17 +110,24 @@ fn setExtraHTTPHeaders(cmd: *CDP.Command) !void { try extra_headers.ensureTotalCapacity(arena, params.headers.map.count()); var it = params.headers.map.iterator(); while (it.next()) |header| { - // Reject a non-printable User-Agent, but (unlike - // Emulation.setUserAgentOverride) allow Mozilla: this is the - // intended escape hatch for a real browser-like UA (#2704). - if (std.ascii.eqlIgnoreCase(header.key_ptr.*, "user-agent")) { - for (header.value_ptr.*) |c| { - if (!std.ascii.isPrint(c)) { - return cmd.sendError(-32602, "User agent contains non-printable characters", .{}); - } + const key = header.key_ptr.*; + const value = header.value_ptr.*; + + if (std.mem.indexOfAny(u8, key, "\r\n") != null or std.mem.indexOfAny(u8, value, "\r\n") != null) { + log.warn(.not_implemented, "network.setExtraHTTPHeaders", .{ .param = "header", .value = key, .info = "header name/value must not contain CR or LF" }); + continue; + } + + const header_string = try std.fmt.allocPrintSentinel(arena, "{s}: {s}", .{ key, value }, 0); + + if (Headers.parseHeader(header_string)) |parsed| { + if (std.ascii.eqlIgnoreCase(parsed.name, "user-agent")) { + Config.validateUserAgent(parsed.value) catch |err| { + log.warn(.not_implemented, "network.setExtraHTTPHeaders", .{ .param = "userAgent", .value = parsed.value, .err = err }); + continue; + }; } } - const header_string = try std.fmt.allocPrintSentinel(arena, "{s}: {s}", .{ header.key_ptr.*, header.value_ptr.* }, 0); extra_headers.appendAssumeCapacity(header_string); } @@ -565,21 +574,31 @@ test "cdp.network setExtraHTTPHeaders" { } test "cdp.network setExtraHTTPHeaders rejects non-printable User-Agent" { + const filter: testing.LogFilter = .init(&.{.not_implemented}); + defer filter.deinit(); + var ctx = try testing.context(); defer ctx.deinit(); - _ = try ctx.loadBrowserContext(.{ .id = "NID-UA1", .session_id = "NESI-UA1" }); + const bc = try ctx.loadBrowserContext(.{ .id = "NID-UA1", .session_id = "NESI-UA1" }); try ctx.processMessage(.{ .id = 3, .method = "Network.setExtraHTTPHeaders", - .params = .{ .headers = .{ .@"User-Agent" = "Bot/1.0\x01hidden" } }, + .params = .{ .headers = .{ + .@"User-Agent" = "Bot/1.0\x01hidden", + .@"x-custom" = "hi", + } }, }); - try ctx.expectSentError(-32602, "User agent contains non-printable characters", .{ .id = 3 }); + try testing.expectEqual(bc.extra_headers.items.len, 1); + try testing.expectEqual("x-custom: hi", std.mem.span(bc.extra_headers.items[0])); } -test "cdp.network setExtraHTTPHeaders allows a Mozilla User-Agent" { +test "cdp.network setExtraHTTPHeaders rejects a Mozilla User-Agent" { + const filter: testing.LogFilter = .init(&.{.not_implemented}); + defer filter.deinit(); + var ctx = try testing.context(); defer ctx.deinit(); @@ -592,7 +611,7 @@ test "cdp.network setExtraHTTPHeaders allows a Mozilla User-Agent" { }); const bc = ctx.cdp().browser_context.?; - try testing.expectEqual(bc.extra_headers.items.len, 1); + try testing.expectEqual(bc.extra_headers.items.len, 0); } test "cdp.network setExtraHTTPHeaders accepts valid User-Agent" { @@ -611,6 +630,52 @@ test "cdp.network setExtraHTTPHeaders accepts valid User-Agent" { try testing.expectEqual(bc.extra_headers.items.len, 1); } +test "cdp.network setExtraHTTPHeaders rejects a Mozilla User-Agent smuggled via a colon in the key" { + const filter: testing.LogFilter = .init(&.{.not_implemented}); + defer filter.deinit(); + + var ctx = try testing.context(); + defer ctx.deinit(); + + _ = try ctx.loadBrowserContext(.{ .id = "NID-UA4", .session_id = "NESI-UA4" }); + + // A colon in the key desyncs the raw key from the first-colon parse that + // req.headers.set/libcurl use: "User-Agent:Mozilla/5.0 (X: Y)" parses to + // name="User-Agent", value="Mozilla/5.0 (X: Y)" on the wire. + try ctx.processMessage(.{ + .id = 3, + .method = "Network.setExtraHTTPHeaders", + .params = .{ .headers = .{ .@"User-Agent:Mozilla/5.0 (X" = "Y)" } }, + }); + + const bc = ctx.cdp().browser_context.?; + try testing.expectEqual(bc.extra_headers.items.len, 0); +} + +test "cdp.network setExtraHTTPHeaders rejects a header that smuggles CRLF" { + const filter: testing.LogFilter = .init(&.{.not_implemented}); + defer filter.deinit(); + + var ctx = try testing.context(); + defer ctx.deinit(); + + const bc = try ctx.loadBrowserContext(.{ .id = "NID-UA5", .session_id = "NESI-UA5" }); + + // The CRLF in the value would inject a second User-Agent line that never + // saw validation; the whole header must be dropped. + try ctx.processMessage(.{ + .id = 3, + .method = "Network.setExtraHTTPHeaders", + .params = .{ .headers = .{ + .@"x-custom" = "bar\r\nUser-Agent: Mozilla/5.0", + .@"x-keep" = "ok", + } }, + }); + + try testing.expectEqual(bc.extra_headers.items.len, 1); + try testing.expectEqual("x-keep: ok", std.mem.span(bc.extra_headers.items[0])); +} + test "cdp.Network: cookies" { const ResCookie = CdpStorage.ResCookie; const CdpCookie = CdpStorage.CdpCookie; From eec06238d05e17627f333d67293d18138fefa117 Mon Sep 17 00:00:00 2001 From: Karl Seguin Date: Sun, 14 Jun 2026 10:17:39 +0800 Subject: [PATCH 6/8] nits: just personalizing the code as a means to understand it. --- src/browser/structured_data.zig | 65 +++++++++++++++++++++------------ src/network/Robots.zig | 15 +++++--- 2 files changed, 50 insertions(+), 30 deletions(-) diff --git a/src/browser/structured_data.zig b/src/browser/structured_data.zig index 21e063b86..55043cfb6 100644 --- a/src/browser/structured_data.zig +++ b/src/browser/structured_data.zig @@ -223,12 +223,6 @@ pub fn collectStructuredData( }; } -/// Registered link relations we surface from the HTTP `Link:` response header -/// (RFC 8288). `service-doc`/`service-desc` come from RFC 8631; `api` is in the -/// IANA Link Relations registry. These let an agent discover a site's developer -/// docs / API description without scraping. -const header_link_rels = [_][]const u8{ "service-doc", "service-desc", "api" }; - /// Parse every `Link:` response header retained on the frame, surfacing only the /// registered, agent-useful relations. Relative targets are resolved against the /// frame's base URL, mirroring `collectLink`. @@ -237,8 +231,14 @@ fn collectLinkHeaders( frame: *Frame, out: *std.ArrayList(LinkRel), ) !void { + // Registered link relations we surface from the HTTP `Link:` response header + // (RFC 8288). + const header_link_rels = [_][]const u8{ "service-doc", "service-desc", "api" }; + for (frame._http_headers.items) |header| { - if (!std.ascii.eqlIgnoreCase(header.name, "link")) continue; + if (!std.ascii.eqlIgnoreCase(header.name, "link")) { + continue; + } var it: LinkHeaderIterator = .{ .value = header.value }; while (it.next()) |entry| { @@ -248,17 +248,21 @@ fn collectLinkHeaders( var rel_it = std.mem.tokenizeAny(u8, rel_value, " \t"); while (rel_it.next()) |rel_token| { const known = for (header_link_rels) |candidate| { - if (std.ascii.eqlIgnoreCase(rel_token, candidate)) break candidate; + if (std.ascii.eqlIgnoreCase(rel_token, candidate)) { + break candidate; + } } else continue; const href = URL.resolve(arena, frame.base(), entry.target, .{ .encoding = frame.charset }) catch entry.target; - // Drop duplicate (rel, href) pairs — a target can legitimately - // carry the same relation across repeated headers. - const dup = for (out.items) |existing| { - if (std.mem.eql(u8, existing.rel, known) and std.mem.eql(u8, existing.href, href)) break true; - } else false; - if (!dup) try out.append(arena, .{ .rel = known, .href = href }); + // Drop duplicate (rel, href) pairs (it happens) + for (out.items) |existing| { + if (std.mem.eql(u8, existing.rel, known) and std.mem.eql(u8, existing.href, href)) { + break; + } + } else { + try out.append(arena, .{ .rel = known, .href = href }); + } } } } @@ -274,25 +278,30 @@ const LinkHeaderEntry = struct { /// skipping commas that fall inside the `<...>` target or a `"..."` quoted /// parameter value (RFC 8288 §3). const LinkHeaderIterator = struct { - value: []const u8, i: usize = 0, + value: []const u8, fn next(self: *LinkHeaderIterator) ?LinkHeaderEntry { const v = self.value; - // Loop (rather than recurse) over malformed segments so stack usage is - // independent of the attacker-controlled header length. while (true) { // Skip separators and whitespace between link-values. The comma that // ended the previous link-value is consumed here. - while (self.i < v.len and (std.ascii.isWhitespace(v[self.i]) or v[self.i] == ',')) : (self.i += 1) {} - if (self.i >= v.len) return null; + for (v[self.i..]) |c| { + if (std.ascii.isWhitespace(c) == false and c != ',') { + break; + } + self.i += 1; + } else { + return null; + } // A link-value must begin with the angle-bracketed target. if (v[self.i] != '<') { self.skipToNextComma(); continue; } + const target_start = self.i + 1; const gt = std.mem.indexOfScalarPos(u8, v, target_start, '>') orelse { self.i = v.len; @@ -316,13 +325,18 @@ const LinkHeaderIterator = struct { var in_quotes = false; while (self.i < v.len) : (self.i += 1) { const c = v[self.i]; + + if (c == ',' and in_quotes == false) { + return; + } + if (c == '\\' and in_quotes) { // Skip the escaped byte; guard a trailing backslash. - if (self.i + 1 < v.len) self.i += 1; + if (self.i + 1 < v.len) { + self.i += 1; + } } else if (c == '"') { in_quotes = !in_quotes; - } else if (c == ',' and !in_quotes) { - return; } } } @@ -334,8 +348,11 @@ fn extractRel(params: []const u8) ?[]const u8 { var it = std.mem.splitScalar(u8, params, ';'); while (it.next()) |raw_param| { const param = std.mem.trim(u8, raw_param, &std.ascii.whitespace); - const eq = std.mem.indexOfScalar(u8, param, '=') orelse continue; - if (!std.ascii.eqlIgnoreCase(std.mem.trim(u8, param[0..eq], &std.ascii.whitespace), "rel")) continue; + const eq = std.mem.indexOfScalarPos(u8, param, 0, '=') orelse continue; + const name = std.mem.trim(u8, param[0..eq], &std.ascii.whitespace); + if (std.ascii.eqlIgnoreCase(name, "rel") == false) { + continue; + } var val = std.mem.trim(u8, param[eq + 1 ..], &std.ascii.whitespace); if (val.len >= 2 and val[0] == '"' and val[val.len - 1] == '"') { diff --git a/src/network/Robots.zig b/src/network/Robots.zig index 423a308cf..73038a460 100644 --- a/src/network/Robots.zig +++ b/src/network/Robots.zig @@ -87,6 +87,9 @@ pub const ContentSignal = struct { pub const Robots = @This(); pub const empty: Robots = .{ .rules = &.{}, .content_signals = &.{} }; +// Think twice before deleting/freeing any entries from the map. Readers, e.g. +// get and getContentSignals, receive values from the map, and if another thread +// was to delete / free those values while in use, UAF. pub const RobotStore = struct { const RobotsEntry = union(enum) { present: Robots, @@ -165,10 +168,7 @@ pub const RobotStore = struct { try self.map.put(self.allocator, duped, .{ .present = robots }); } - /// Advisory `Content-Signal` preferences declared for `url`'s host, or null - /// when no robots.txt has been fetched/stored for it. The returned slice is - /// owned by the store; callers must read it synchronously (and dupe to keep - /// it) and must never free it. + // The returned slice is owned by the store pub fn getContentSignals(self: *RobotStore, url: []const u8) ?[]const ContentSignal { self.mutex.lock(); defer self.mutex.unlock(); @@ -237,14 +237,17 @@ fn appendContentSignals( var iter = std.mem.splitScalar(u8, value, ','); while (iter.next()) |raw_pair| { const pair = std.mem.trim(u8, raw_pair, &std.ascii.whitespace); - const eq = std.mem.indexOfScalar(u8, pair, '=') orelse continue; + const eq = std.mem.indexOfScalarPos(u8, pair, 0, '=') orelse continue; const name_raw = std.mem.trim(u8, pair[0..eq], &std.ascii.whitespace); const value_raw = std.mem.trim(u8, pair[eq + 1 ..], &std.ascii.whitespace); - if (name_raw.len == 0) continue; + if (name_raw.len == 0) { + continue; + } const name = try std.ascii.allocLowerString(allocator, name_raw); errdefer allocator.free(name); + const val = try std.ascii.allocLowerString(allocator, value_raw); errdefer allocator.free(val); From 9ace686d1e9dcdd0bf354350d8f50e74d51fb3f7 Mon Sep 17 00:00:00 2001 From: Karl Seguin Date: Sun, 14 Jun 2026 21:57:03 +0800 Subject: [PATCH 7/8] perf: avoid excessive tabIndex lookup/parse Store tab index for current, edge, chosen in a local, and only get/parse it once per candidate (vs twice). --- src/browser/Frame.zig | 128 ++++++++++++++++------------ src/browser/webapi/element/Html.zig | 10 ++- 2 files changed, 81 insertions(+), 57 deletions(-) diff --git a/src/browser/Frame.zig b/src/browser/Frame.zig index b446429ae..e38b71514 100644 --- a/src/browser/Frame.zig +++ b/src/browser/Frame.zig @@ -4196,9 +4196,8 @@ pub fn handleKeydown(self: *Frame, target: *Node, event: *Event) !void { } if (key == .Tab) { - // Sequential focus navigation is the default action of a Tab keydown. - // Shift+Tab walks backward. - return self.moveFocus(if (keyboard_event.getShiftKey()) .backward else .forward); + // tab -> forward, shift+tab -> backwards + return self.moveFocus(keyboard_event.getShiftKey() == false); } if (target.is(Element.Html.Input)) |input| { @@ -4231,8 +4230,6 @@ pub fn handleKeydown(self: *Frame, target: *Node, event: *Event) !void { } } -const FocusDirection = enum { forward, backward }; - // Sequential focus navigation: move `document.activeElement` to the next (Tab) // or previous (Shift+Tab) focusable element, firing the usual blur/focus events // via `Element.focus`. The order is fully determined by tabindex + document @@ -4242,34 +4239,80 @@ const FocusDirection = enum { forward, backward }; // document order. // Ties within a group break on document order, and Tab wraps around at the ends. // https://html.spec.whatwg.org/multipage/interaction.html#sequential-focus-navigation -fn moveFocus(self: *Frame, direction: FocusDirection) !void { +fn moveFocus(self: *Frame, forward: bool) !void { const document = self.document; const current = document._active_element; - const forward = direction == .forward; + + const current_tab_index = blk: { + const cur = current orelse break :blk 0; + const current_html = cur.is(Element.Html) orelse break :blk 0; + break :blk current_html.getTabIndex(); + }; // Single document-order pass tracking two candidates: - // chosen — the closest focusable element strictly past `current` in the - // travel direction. // edge — the global first (forward) / last (backward) focusable element, // used to wrap around when `current` is at an end, or as the // landing spot when nothing is focused yet. - var chosen: ?*Element = null; + // chosen — the closest focusable element strictly past `current` in the + // travel direction. var edge: ?*Element = null; + var edge_tab_index: i32 = 0; + + var chosen: ?*Element = null; + var chosen_tab_index: i32 = 0; var tw = @import("webapi/TreeWalker.zig").Full.Elements.init(document.asNode(), .{}); - while (tw.next()) |el| { - if (!sequentiallyFocusable(el)) continue; + while (tw.next()) |candidate| { + if (candidate.isDisabled()) { + continue; + } + if (candidate.is(Element.Html) == null) { + continue; + } - if (edge == null or focusOrderBefore(el, edge.?) == forward) { - edge = el; + const candidate_tab_index = blk: { + if (candidate.getAttributeSafe(comptime .wrap("tabindex"))) |attr| { + if (Element.Html.parseInteger(attr)) |tab_index| { + if (tab_index < 0) { + continue; + } + break :blk tab_index; + } + break :blk 0; + } + + // no tab index, maybe this item isn't focusable.. + const focusable = switch (candidate.getTag()) { + .button, .select, .textarea, .iframe => true, + .input => candidate.as(Element.Html.Input)._input_type != .hidden, + .anchor, .area => candidate.getAttributeSafe(comptime .wrap("href")) != null, + else => false, + }; + if (focusable == false) { + continue; + } + + break :blk 0; + }; + + if (edge == null or focusOrderBefore(candidate, candidate_tab_index, edge.?, edge_tab_index) == forward) { + edge = candidate; + edge_tab_index = candidate_tab_index; } const cur = current orelse continue; - if (el == cur) continue; - const past = if (forward) focusOrderBefore(cur, el) else focusOrderBefore(el, cur); - if (!past) continue; - if (chosen == null or focusOrderBefore(el, chosen.?) == forward) { - chosen = el; + + if (candidate == cur) { + continue; + } + + const past = if (forward) focusOrderBefore(cur, current_tab_index, candidate, candidate_tab_index) else focusOrderBefore(candidate, candidate_tab_index, cur, current_tab_index); + if (!past) { + continue; + } + if (chosen == null or focusOrderBefore(candidate, candidate_tab_index, chosen.?, chosen_tab_index) == forward) { + chosen = candidate; + chosen_tab_index = candidate_tab_index; } } @@ -4277,45 +4320,22 @@ fn moveFocus(self: *Frame, direction: FocusDirection) !void { try next.focus(self); } -// True when `el` participates in sequential focus navigation: enabled, with a -// non-negative tabindex, and either an explicit tabindex or a natively -// focusable shape. `getTabIndex` defaults several tags to 0, including -// href-less anchors and hidden inputs, so the structural conditions are -// re-checked here. -fn sequentiallyFocusable(el: *Element) bool { - const html_el = el.is(Element.Html) orelse return false; - if (el.isDisabled()) return false; - if (html_el.getTabIndex() < 0) return false; - - if (el.getAttributeSafe(comptime .wrap("tabindex")) != null) return true; - - return switch (el.getTag()) { - .button, .select, .textarea, .iframe => true, - .input => if (el.is(Element.Html.Input)) |input| input._input_type != .hidden else false, - .anchor, .area => el.getAttributeSafe(comptime .wrap("href")) != null, - else => false, - }; -} - // Orders two focusable elements by sequential focus navigation order: positive // tabindex first (ascending), then tabindex 0, ties broken by document order. -fn focusOrderBefore(a: *Element, b: *Element) bool { - const ta = tabOrderIndex(a); - const tb = tabOrderIndex(b); - if (ta != tb) { - const group_a: u8 = if (ta > 0) 0 else 1; - const group_b: u8 = if (tb > 0) 0 else 1; - if (group_a != group_b) return group_a < group_b; - return ta < tb; +fn focusOrderBefore(a: *Element, a_tab_index: i32, b: *Element, b_tab_index: i32) bool { + if (a_tab_index == b_tab_index) { + // Equal tabindex → document order: `a` precedes `b` when `b` follows `a`. + const FOLLOWING: u16 = 0x04; + return (a.asNode().compareDocumentPosition(b.asNode()) & FOLLOWING) != 0; } - // Equal tabindex → document order: `a` precedes `b` when `b` follows `a`. - const FOLLOWING: u16 = 0x04; - return (a.asNode().compareDocumentPosition(b.asNode()) & FOLLOWING) != 0; -} -fn tabOrderIndex(el: *Element) i32 { - const html_el = el.is(Element.Html) orelse return 0; - return html_el.getTabIndex(); + const group_a: u8 = if (a_tab_index > 0) 0 else 1; + const group_b: u8 = if (b_tab_index > 0) 0 else 1; + if (group_a != group_b) { + return group_a < group_b; + } + + return a_tab_index < b_tab_index; } const SubmitFormOpts = struct { diff --git a/src/browser/webapi/element/Html.zig b/src/browser/webapi/element/Html.zig index e124c156d..871a0882d 100644 --- a/src/browser/webapi/element/Html.zig +++ b/src/browser/webapi/element/Html.zig @@ -387,12 +387,16 @@ pub fn togglePopover(self: *HtmlElement, force: ?bool, frame: *Frame) !bool { } pub fn getTabIndex(self: *HtmlElement) i32 { - const default: i32 = switch (self._type) { + if (self.asElement().getAttributeSafe(comptime .wrap("tabindex"))) |attr| { + if (parseInteger(attr)) |tab_index| { + return tab_index; + } + } + + return switch (self._type) { .anchor, .area, .button, .input, .select, .textarea, .iframe => 0, else => -1, }; - const attr = self.asElement().getAttributeSafe(comptime .wrap("tabindex")) orelse return default; - return parseInteger(attr) orelse default; } pub fn setTabIndex(self: *HtmlElement, value: i32, frame: *Frame) !void { From 78d779332780b06661f884d152ec0b1ea18e8784 Mon Sep 17 00:00:00 2001 From: Karl Seguin Date: Mon, 15 Jun 2026 16:59:04 +0800 Subject: [PATCH 8/8] dev: silence a test that uses/needs console --- src/cdp/domains/runtime.zig | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/cdp/domains/runtime.zig b/src/cdp/domains/runtime.zig index 6228ee738..f2758cd92 100644 --- a/src/cdp/domains/runtime.zig +++ b/src/cdp/domains/runtime.zig @@ -167,6 +167,9 @@ pub fn consoleMessage(arena: Allocator, bc: *CDP.BrowserContext, event: *const N const testing = @import("../testing.zig"); test "cdp.runtime: consoleAPICalled type matches the console method" { + const filter: testing.LogFilter = .init(&.{.js}); + defer filter.deinit(); + // Wire types per the CDP protocol: console.log -> "log", // console.warn -> "warning" (not "warn"), console.info -> "info", // console.error -> "error", console.debug -> "debug".