diff --git a/src/browser/structured_data.zig b/src/browser/structured_data.zig index 09cf72a2a..21e063b86 100644 --- a/src/browser/structured_data.zig +++ b/src/browser/structured_data.zig @@ -39,6 +39,13 @@ pub const AlternateLink = struct { title: ?[]const u8, }; +/// A relation discovered in the HTTP `Link:` response header (RFC 8288), +/// restricted to the registered relations an agent can act on. +pub const LinkRel = struct { + rel: []const u8, + href: []const u8, +}; + pub const StructuredData = struct { json_ld: []const []const u8, open_graph: []const Property, @@ -46,6 +53,7 @@ pub const StructuredData = struct { meta: []const Property, links: []const Property, alternate: []const AlternateLink, + link_headers: []const LinkRel, pub fn jsonStringify(self: *const StructuredData, jw: anytype) !void { try jw.beginObject(); @@ -89,6 +97,23 @@ pub const StructuredData = struct { try jw.endArray(); } + // Emitted only when present so existing consumers of the CDP/MCP + // structured-data shape see no new key on sites without a `Link:` + // response header. + if (self.link_headers.len > 0) { + try jw.objectField("linkHeaders"); + try jw.beginArray(); + for (self.link_headers) |lh| { + try jw.beginObject(); + try jw.objectField("rel"); + try jw.write(lh.rel); + try jw.objectField("href"); + try jw.write(lh.href); + try jw.endObject(); + } + try jw.endArray(); + } + try jw.endObject(); } }; @@ -144,6 +169,7 @@ pub fn collectStructuredData( var meta: std.ArrayList(Property) = .empty; var links: std.ArrayList(Property) = .empty; var alternate: std.ArrayList(AlternateLink) = .empty; + var link_headers: std.ArrayList(LinkRel) = .empty; // Extract language from the root element. if (root.is(Element)) |root_el| { @@ -182,6 +208,10 @@ pub fn collectStructuredData( } } + // The `Link:` response header lives on the frame, not in the DOM, so it is + // collected separately from the navigation's retained headers. + try collectLinkHeaders(arena, frame, &link_headers); + return .{ .json_ld = json_ld.items, .open_graph = open_graph.items, @@ -189,9 +219,133 @@ pub fn collectStructuredData( .meta = meta.items, .links = links.items, .alternate = alternate.items, + .link_headers = link_headers.items, }; } +/// Registered link relations we surface from the HTTP `Link:` response header +/// (RFC 8288). `service-doc`/`service-desc` come from RFC 8631; `api` is in the +/// IANA Link Relations registry. These let an agent discover a site's developer +/// docs / API description without scraping. +const header_link_rels = [_][]const u8{ "service-doc", "service-desc", "api" }; + +/// Parse every `Link:` response header retained on the frame, surfacing only the +/// registered, agent-useful relations. Relative targets are resolved against the +/// frame's base URL, mirroring `collectLink`. +fn collectLinkHeaders( + arena: Allocator, + frame: *Frame, + out: *std.ArrayList(LinkRel), +) !void { + for (frame._http_headers.items) |header| { + if (!std.ascii.eqlIgnoreCase(header.name, "link")) continue; + + var it: LinkHeaderIterator = .{ .value = header.value }; + while (it.next()) |entry| { + const rel_value = entry.rel orelse continue; + + // The `rel` parameter is a space-separated list of relation types. + var rel_it = std.mem.tokenizeAny(u8, rel_value, " \t"); + while (rel_it.next()) |rel_token| { + const known = for (header_link_rels) |candidate| { + if (std.ascii.eqlIgnoreCase(rel_token, candidate)) break candidate; + } else continue; + + const href = URL.resolve(arena, frame.base(), entry.target, .{ .encoding = frame.charset }) catch entry.target; + + // Drop duplicate (rel, href) pairs — a target can legitimately + // carry the same relation across repeated headers. + const dup = for (out.items) |existing| { + if (std.mem.eql(u8, existing.rel, known) and std.mem.eql(u8, existing.href, href)) break true; + } else false; + if (!dup) try out.append(arena, .{ .rel = known, .href = href }); + } + } + } +} + +/// One `< target >; param=value; ...` entry of an RFC 8288 `Link` header value. +const LinkHeaderEntry = struct { + target: []const u8, + rel: ?[]const u8, +}; + +/// Splits a `Link:` header field value into its comma-separated link-values, +/// skipping commas that fall inside the `<...>` target or a `"..."` quoted +/// parameter value (RFC 8288 §3). +const LinkHeaderIterator = struct { + value: []const u8, + i: usize = 0, + + fn next(self: *LinkHeaderIterator) ?LinkHeaderEntry { + const v = self.value; + + // Loop (rather than recurse) over malformed segments so stack usage is + // independent of the attacker-controlled header length. + while (true) { + // Skip separators and whitespace between link-values. The comma that + // ended the previous link-value is consumed here. + while (self.i < v.len and (std.ascii.isWhitespace(v[self.i]) or v[self.i] == ',')) : (self.i += 1) {} + if (self.i >= v.len) return null; + + // A link-value must begin with the angle-bracketed target. + if (v[self.i] != '<') { + self.skipToNextComma(); + continue; + } + const target_start = self.i + 1; + const gt = std.mem.indexOfScalarPos(u8, v, target_start, '>') orelse { + self.i = v.len; + return null; + }; + const target = v[target_start..gt]; + self.i = gt + 1; + + // Parameters run to the next top-level comma, left at `self.i`. + const params_start = self.i; + self.skipToNextComma(); + return .{ .target = target, .rel = extractRel(v[params_start..self.i]) }; + } + } + + // Advance until `self.i` sits on the next top-level comma (one outside a + // `"..."` quoted-string, honouring RFC 7230 backslash escapes) or the end + // of the value. The comma itself is left for the leading-separator skip. + fn skipToNextComma(self: *LinkHeaderIterator) void { + const v = self.value; + var in_quotes = false; + while (self.i < v.len) : (self.i += 1) { + const c = v[self.i]; + if (c == '\\' and in_quotes) { + // Skip the escaped byte; guard a trailing backslash. + if (self.i + 1 < v.len) self.i += 1; + } else if (c == '"') { + in_quotes = !in_quotes; + } else if (c == ',' and !in_quotes) { + return; + } + } + } +}; + +/// Return the (unquoted) value of the `rel` parameter from a link-value's +/// parameter list, or null when absent. +fn extractRel(params: []const u8) ?[]const u8 { + var it = std.mem.splitScalar(u8, params, ';'); + while (it.next()) |raw_param| { + const param = std.mem.trim(u8, raw_param, &std.ascii.whitespace); + const eq = std.mem.indexOfScalar(u8, param, '=') orelse continue; + if (!std.ascii.eqlIgnoreCase(std.mem.trim(u8, param[0..eq], &std.ascii.whitespace), "rel")) continue; + + var val = std.mem.trim(u8, param[eq + 1 ..], &std.ascii.whitespace); + if (val.len >= 2 and val[0] == '"' and val[val.len - 1] == '"') { + val = val[1 .. val.len - 1]; + } + return val; + } + return null; +} + fn collectJsonLd( el: *Element, arena: Allocator, @@ -484,6 +638,78 @@ test "structured_data: charset and http-equiv" { try testing.expectEqual("text/html; charset=utf-8", findProperty(data.meta, "Content-Type").?); } +test "structured_data: parseLinkHeader - single link with quoted rel" { + var it: LinkHeaderIterator = .{ .value = + \\; rel="service-desc" + }; + const entry = it.next().?; + try testing.expectString("https://api.example.com/openapi.json", entry.target); + try testing.expectString("service-desc", entry.rel.?); + try testing.expectEqual(null, it.next()); +} + +test "structured_data: parseLinkHeader - multiple links and unquoted rel" { + var it: LinkHeaderIterator = .{ .value = + \\; rel=service-doc, ; rel="stylesheet"; type="text/css" + }; + const first = it.next().?; + try testing.expectString("https://docs.example.com/", first.target); + try testing.expectString("service-doc", first.rel.?); + + const second = it.next().?; + try testing.expectString("/style.css", second.target); + // A comma inside the quoted type param must not split the link-value. + try testing.expectString("stylesheet", second.rel.?); + + try testing.expectEqual(null, it.next()); +} + +test "structured_data: parseLinkHeader - escaped quote in param does not split link" { + // A backslash-escaped quote inside an earlier param must not terminate the + // quoted-string, so the comma inside it is not treated as a separator. + var it: LinkHeaderIterator = .{ .value = + \\; title="x\",y"; rel="service-doc" + }; + const entry = it.next().?; + try testing.expectString("https://a/", entry.target); + try testing.expectString("service-doc", entry.rel.?); + try testing.expectEqual(null, it.next()); +} + +test "structured_data: link headers from response" { + const frame = try testing.test_session.createPage(); + defer testing.test_session.removePage(); + + // Stand in for what frameHeaderDoneCallback records from the navigation. + try frame._http_headers.append(frame.arena, .{ .name = "Link", .value = + \\; rel="service-doc" + }); + try frame._http_headers.append(frame.arena, .{ .name = "link", .value = + \\; rel="service-desc", ; rel="stylesheet" + }); + + const doc = frame.window._document; + const div = try doc.createElement("div", null, frame); + try frame.parseHtmlAsChildren(div.asNode(), "Example"); + + const data = try collectStructuredData(div.asNode(), frame.call_arena, frame); + + // service-doc + service-desc are surfaced; stylesheet is filtered out. + try testing.expectEqual(2, data.link_headers.len); + try testing.expectString("service-doc", data.link_headers[0].rel); + try testing.expectString("https://docs.example.com/", data.link_headers[0].href); + try testing.expectString("service-desc", data.link_headers[1].rel); + try testing.expectString("https://api.example.com/spec", data.link_headers[1].href); +} + +test "structured_data: no link header yields empty list" { + const data = try testStructuredData( + \\ + ); + defer testing.test_session.removePage(); + try testing.expectEqual(0, data.link_headers.len); +} + test "structured_data: mixed content" { const data = try testStructuredData( \\My Site diff --git a/src/cdp/domains/lp.zig b/src/cdp/domains/lp.zig index b3b3aec0a..dd8fdd991 100644 --- a/src/cdp/domains/lp.zig +++ b/src/cdp/domains/lp.zig @@ -23,6 +23,7 @@ const CDP = @import("../CDP.zig"); const Node = @import("../Node.zig"); const DOMNode = @import("../../browser/webapi/Node.zig"); +const Robots = @import("../../network/Robots.zig"); const markdown = lp.markdown; const SemanticTree = lp.SemanticTree; @@ -36,6 +37,7 @@ pub fn processMessage(cmd: *CDP.Command) !void { getInteractiveElements, getNodeDetails, getStructuredData, + getContentSignal, detectForms, clickNode, fillNode, @@ -51,6 +53,7 @@ pub fn processMessage(cmd: *CDP.Command) !void { .getInteractiveElements => return getInteractiveElements(cmd), .getNodeDetails => return getNodeDetails(cmd), .getStructuredData => return getStructuredData(cmd), + .getContentSignal => return getContentSignal(cmd), .detectForms => return detectForms(cmd), .clickNode => return clickNode(cmd), .fillNode => return fillNode(cmd), @@ -201,6 +204,27 @@ fn getStructuredData(cmd: anytype) !void { }, .{}); } +// Advisory `Content-Signal` robots.txt preferences for the current document's +// host (https://contentsignals.org). `available` is false when no robots.txt +// has been fetched for the host — note this is the case unless `obey_robots` +// is enabled, since the robots layer is what populates the store. +fn getContentSignal(cmd: anytype) !void { + const bc = cmd.browser_context orelse return error.NoBrowserContext; + const frame = bc.session.currentFrame() orelse return error.FrameNotLoaded; + const network = bc.cdp.browser.http_client.network; + + const empty: []const Robots.ContentSignal = &.{}; + const robots_url = lp.URL.getRobotsUrl(cmd.arena, frame.url) catch { + return cmd.sendResult(.{ .available = false, .contentSignals = empty }, .{}); + }; + + const signals = network.robot_store.getContentSignals(robots_url); + return cmd.sendResult(.{ + .available = signals != null, + .contentSignals = signals orelse empty, + }, .{}); +} + fn detectForms(cmd: anytype) !void { const bc = cmd.browser_context orelse return error.NoBrowserContext; const frame = bc.session.currentFrame() orelse return error.FrameNotLoaded; @@ -410,6 +434,25 @@ test "cdp.lp: getStructuredData" { try testing.expect(result.get("structuredData") != null); } +test "cdp.lp: getContentSignal" { + var ctx = try testing.context(); + defer ctx.deinit(); + + const bc = try ctx.loadBrowserContext(.{}); + _ = try bc.session.createPage(); + + try ctx.processMessage(.{ + .id = 1, + .method = "LP.getContentSignal", + }); + + // No robots.txt fetched for the page, so the directive is reported absent + // rather than erroring — the command is still wired and returns the shape. + const result = (try ctx.getSentMessage(0)).?.object.get("result").?.object; + try testing.expectEqual(false, result.get("available").?.bool); + try testing.expect(result.get("contentSignals") != null); +} + test "cdp.lp: action tools" { var ctx = try testing.context(); defer ctx.deinit(); diff --git a/src/network/Robots.zig b/src/network/Robots.zig index 202e0a5c7..423a308cf 100644 --- a/src/network/Robots.zig +++ b/src/network/Robots.zig @@ -72,11 +72,20 @@ pub const Key = enum { @"user-agent", allow, disallow, + @"content-signal", +}; + +/// A declared content-usage preference from a `Content-Signal:` robots.txt +/// directive (https://contentsignals.org), e.g. name="ai-train" value="no". +/// Advisory metadata surfaced to the agent — it never affects crawl gating. +pub const ContentSignal = struct { + name: []const u8, + value: []const u8, }; /// https://www.rfc-editor.org/rfc/rfc9309.html pub const Robots = @This(); -pub const empty: Robots = .{ .rules = &.{} }; +pub const empty: Robots = .{ .rules = &.{}, .content_signals = &.{} }; pub const RobotStore = struct { const RobotsEntry = union(enum) { @@ -156,6 +165,21 @@ pub const RobotStore = struct { try self.map.put(self.allocator, duped, .{ .present = robots }); } + /// Advisory `Content-Signal` preferences declared for `url`'s host, or null + /// when no robots.txt has been fetched/stored for it. The returned slice is + /// owned by the store; callers must read it synchronously (and dupe to keep + /// it) and must never free it. + pub fn getContentSignals(self: *RobotStore, url: []const u8) ?[]const ContentSignal { + self.mutex.lock(); + defer self.mutex.unlock(); + + const entry = self.map.get(url) orelse return null; + return switch (entry) { + .present => |robots| robots.content_signals, + .absent => null, + }; + } + pub fn putAbsent(self: *RobotStore, url: []const u8) !void { self.mutex.lock(); defer self.mutex.unlock(); @@ -166,6 +190,15 @@ pub const RobotStore = struct { }; rules: []const Rule, +content_signals: []const ContentSignal = &.{}, + +/// Result of parsing a robots.txt for one user-agent: the applicable crawl +/// rules plus any advisory `Content-Signal` preferences. The two are kept +/// distinct so content signals can never reach `isAllowed`. +const ParseResult = struct { + rules: []Rule, + content_signals: []ContentSignal, +}; const State = struct { entry: enum { @@ -186,17 +219,56 @@ fn freeRulesInList(allocator: std.mem.Allocator, rules: []const Rule) void { } } +fn freeContentSignals(allocator: std.mem.Allocator, signals: []const ContentSignal) void { + for (signals) |signal| { + allocator.free(signal.name); + allocator.free(signal.value); + } +} + +/// Parse a `Content-Signal:` value — a comma-separated list of `name=value` +/// pairs (e.g. `ai-train=no, search=yes`) — into the given list. Names and +/// values are lower-cased and owned by `allocator`. +fn appendContentSignals( + allocator: std.mem.Allocator, + list: *std.ArrayList(ContentSignal), + value: []const u8, +) !void { + var iter = std.mem.splitScalar(u8, value, ','); + while (iter.next()) |raw_pair| { + const pair = std.mem.trim(u8, raw_pair, &std.ascii.whitespace); + const eq = std.mem.indexOfScalar(u8, pair, '=') orelse continue; + + const name_raw = std.mem.trim(u8, pair[0..eq], &std.ascii.whitespace); + const value_raw = std.mem.trim(u8, pair[eq + 1 ..], &std.ascii.whitespace); + if (name_raw.len == 0) continue; + + const name = try std.ascii.allocLowerString(allocator, name_raw); + errdefer allocator.free(name); + const val = try std.ascii.allocLowerString(allocator, value_raw); + errdefer allocator.free(val); + + try list.append(allocator, .{ .name = name, .value = val }); + } +} + fn parseRulesWithUserAgent( allocator: std.mem.Allocator, user_agent: []const u8, raw_bytes: []const u8, -) ![]Rule { +) !ParseResult { var rules: std.ArrayList(Rule) = .empty; defer rules.deinit(allocator); var wildcard_rules: std.ArrayList(Rule) = .empty; defer wildcard_rules.deinit(allocator); + var content_signals: std.ArrayList(ContentSignal) = .empty; + defer content_signals.deinit(allocator); + + var wildcard_content_signals: std.ArrayList(ContentSignal) = .empty; + defer wildcard_content_signals.deinit(allocator); + var state: State = .{ .entry = .not_in_entry, .has_rules = false }; // https://en.wikipedia.org/wiki/Byte_order_mark @@ -308,22 +380,51 @@ fn parseRulesWithUserAgent( }, } }, + .@"content-signal" => { + // A group member like allow/disallow, so it sets has_rules + // (a following `User-agent:` opens a new group), but it is + // captured into a separate list and never reaches isAllowed. + defer state.has_rules = true; + + switch (state.entry) { + .in_our_entry => try appendContentSignals(allocator, &content_signals, value), + .in_wildcard_entry => try appendContentSignals(allocator, &wildcard_content_signals, value), + .in_other_entry, .not_in_entry => {}, + } + }, } } // If we have rules for our specific User-Agent, we will use those rules. // If we don't have any rules, we fallback to using the wildcard ("*") rules. - if (rules.items.len > 0) { + const out_rules = if (rules.items.len > 0) blk: { freeRulesInList(allocator, wildcard_rules.items); - return try rules.toOwnedSlice(allocator); - } else { + break :blk try rules.toOwnedSlice(allocator); + } else blk: { freeRulesInList(allocator, rules.items); - return try wildcard_rules.toOwnedSlice(allocator); + break :blk try wildcard_rules.toOwnedSlice(allocator); + }; + errdefer { + freeRulesInList(allocator, out_rules); + allocator.free(out_rules); } + + // Content signals follow the same specific-else-wildcard precedence, + // chosen independently of the rule selection above. + const out_signals = if (content_signals.items.len > 0) blk: { + freeContentSignals(allocator, wildcard_content_signals.items); + break :blk try content_signals.toOwnedSlice(allocator); + } else blk: { + freeContentSignals(allocator, content_signals.items); + break :blk try wildcard_content_signals.toOwnedSlice(allocator); + }; + + return .{ .rules = out_rules, .content_signals = out_signals }; } pub fn fromBytes(allocator: std.mem.Allocator, user_agent: []const u8, bytes: []const u8) !Robots { - const rules = try parseRulesWithUserAgent(allocator, user_agent, bytes); + const parsed = try parseRulesWithUserAgent(allocator, user_agent, bytes); + const rules = parsed.rules; // sort by order once. std.mem.sort(Rule, rules, {}, struct { @@ -357,12 +458,14 @@ pub fn fromBytes(allocator: std.mem.Allocator, user_agent: []const u8, bytes: [] } }.lessThan); - return .{ .rules = rules }; + return .{ .rules = rules, .content_signals = parsed.content_signals }; } pub fn deinit(self: *Robots, allocator: std.mem.Allocator) void { freeRulesInList(allocator, self.rules); allocator.free(self.rules); + freeContentSignals(allocator, self.content_signals); + allocator.free(self.content_signals); } /// There are rules for how the pattern in robots.txt should be matched. @@ -476,10 +579,13 @@ test "Robots: simple robots.txt" { \\ ; - const rules = try parseRulesWithUserAgent(allocator, "GoogleBot", file); + const parsed = try parseRulesWithUserAgent(allocator, "GoogleBot", file); + const rules = parsed.rules; defer { freeRulesInList(allocator, rules); allocator.free(rules); + freeContentSignals(allocator, parsed.content_signals); + allocator.free(parsed.content_signals); } try std.testing.expectEqual(1, rules.len); @@ -1003,3 +1109,101 @@ test "Robots: blank lines don't end entries" { try std.testing.expect(robots.isAllowed("/admin/") == false); try std.testing.expect(robots.isAllowed("/public/") == true); } + +test "Robots: content-signal captured and gating unaffected" { + const allocator = std.testing.allocator; + + var robots = try Robots.fromBytes(allocator, "MyBot", + \\User-agent: * + \\Content-Signal: ai-train=no, search=yes, ai-input=yes + \\Disallow: /private/ + \\ + ); + defer robots.deinit(allocator); + + try std.testing.expectEqual(3, robots.content_signals.len); + try std.testing.expectEqualStrings("ai-train", robots.content_signals[0].name); + try std.testing.expectEqualStrings("no", robots.content_signals[0].value); + try std.testing.expectEqualStrings("search", robots.content_signals[1].name); + try std.testing.expectEqualStrings("yes", robots.content_signals[1].value); + try std.testing.expectEqualStrings("ai-input", robots.content_signals[2].name); + + // The whole point: Content-Signal is advisory metadata, it must not move + // the crawl gate. Disallow/Allow still decide isAllowed. + try std.testing.expect(robots.isAllowed("/private/page") == false); + try std.testing.expect(robots.isAllowed("/public/page") == true); +} + +test "Robots: content-signal mixed casing is normalized" { + const allocator = std.testing.allocator; + + var robots = try Robots.fromBytes(allocator, "MyBot", + \\User-agent: * + \\Content-Signal: AI-Train=NO + \\ + ); + defer robots.deinit(allocator); + + try std.testing.expectEqual(1, robots.content_signals.len); + try std.testing.expectEqualStrings("ai-train", robots.content_signals[0].name); + try std.testing.expectEqualStrings("no", robots.content_signals[0].value); +} + +test "Robots: content-signal absent yields empty list" { + const allocator = std.testing.allocator; + + var robots = try Robots.fromBytes(allocator, "MyBot", + \\User-agent: * + \\Disallow: /admin/ + \\ + ); + defer robots.deinit(allocator); + + try std.testing.expectEqual(0, robots.content_signals.len); + try std.testing.expect(robots.isAllowed("/admin/") == false); +} + +test "Robots: content-signal prefers specific user-agent over wildcard" { + const allocator = std.testing.allocator; + + const file = + \\User-agent: * + \\Content-Signal: ai-train=yes + \\ + \\User-agent: MyBot + \\Content-Signal: ai-train=no + \\ + ; + + var mine = try Robots.fromBytes(allocator, "MyBot", file); + defer mine.deinit(allocator); + try std.testing.expectEqual(1, mine.content_signals.len); + try std.testing.expectEqualStrings("no", mine.content_signals[0].value); + + var other = try Robots.fromBytes(allocator, "OtherBot", file); + defer other.deinit(allocator); + try std.testing.expectEqual(1, other.content_signals.len); + try std.testing.expectEqualStrings("yes", other.content_signals[0].value); +} + +test "Robots: RobotStore.getContentSignals round-trips" { + const allocator = std.testing.allocator; + + var store = RobotStore.init(allocator); + defer store.deinit(); + + const robots = try store.robotsFromBytes("MyBot", + \\User-agent: * + \\Content-Signal: ai-train=no + \\ + ); + try store.put("https://example.com/robots.txt", robots); + + const signals = store.getContentSignals("https://example.com/robots.txt").?; + try std.testing.expectEqual(1, signals.len); + try std.testing.expectEqualStrings("ai-train", signals[0].name); + try std.testing.expectEqualStrings("no", signals[0].value); + + // Unknown host has no stored robots. + try std.testing.expectEqual(null, store.getContentSignals("https://other.com/robots.txt")); +}