Surface server-side agent-discovery signals (Link header + Content-Signal)

Parse the HTTP `Link:` response header (RFC 8288) and the robots.txt `Content-Signal` directive, surfacing both to the driving agent. - structured_data: extract the registered Link relations (service-doc, service-desc, api) from the frame's retained response headers into a new `linkHeaders` field, flowing out via the structuredData MCP tool and LP.getStructuredData. Emitted only when present. - Robots: capture Content-Signal preferences on a new content_signals field, read back via RobotStore.getContentSignals and the new LP.getContentSignal CDP command. Advisory only - never reaches isAllowed. Refs #2680
2026-07-30 17:25:58 -04:00 · 2026-06-09 15:29:48 +02:00
parent 02d6c45dad
commit b0a087450d
3 changed files with 482 additions and 9 deletions
--- a/src/browser/structured_data.zig
+++ b/src/browser/structured_data.zig
@@ -39,6 +39,13 @@ pub const AlternateLink = struct {
    title: ?[]const u8,
 };

+/// A relation discovered in the HTTP `Link:` response header (RFC 8288),
+/// restricted to the registered relations an agent can act on.
+pub const LinkRel = struct {
+    rel: []const u8,
+    href: []const u8,
+};
+
 pub const StructuredData = struct {
    json_ld: []const []const u8,
    open_graph: []const Property,
@@ -46,6 +53,7 @@ pub const StructuredData = struct {
    meta: []const Property,
    links: []const Property,
    alternate: []const AlternateLink,
+    link_headers: []const LinkRel,

    pub fn jsonStringify(self: *const StructuredData, jw: anytype) !void {
        try jw.beginObject();
@@ -89,6 +97,23 @@ pub const StructuredData = struct {
            try jw.endArray();
        }

+        // Emitted only when present so existing consumers of the CDP/MCP
+        // structured-data shape see no new key on sites without a `Link:`
+        // response header.
+        if (self.link_headers.len > 0) {
+            try jw.objectField("linkHeaders");
+            try jw.beginArray();
+            for (self.link_headers) |lh| {
+                try jw.beginObject();
+                try jw.objectField("rel");
+                try jw.write(lh.rel);
+                try jw.objectField("href");
+                try jw.write(lh.href);
+                try jw.endObject();
+            }
+            try jw.endArray();
+        }
+
        try jw.endObject();
    }
 };
@@ -144,6 +169,7 @@ pub fn collectStructuredData(
    var meta: std.ArrayList(Property) = .empty;
    var links: std.ArrayList(Property) = .empty;
    var alternate: std.ArrayList(AlternateLink) = .empty;
+    var link_headers: std.ArrayList(LinkRel) = .empty;

    // Extract language from the root <html> element.
    if (root.is(Element)) |root_el| {
@@ -182,6 +208,10 @@ pub fn collectStructuredData(
        }
    }

+    // The `Link:` response header lives on the frame, not in the DOM, so it is
+    // collected separately from the navigation's retained headers.
+    try collectLinkHeaders(arena, frame, &link_headers);
+
    return .{
        .json_ld = json_ld.items,
        .open_graph = open_graph.items,
@@ -189,9 +219,133 @@ pub fn collectStructuredData(
        .meta = meta.items,
        .links = links.items,
        .alternate = alternate.items,
+        .link_headers = link_headers.items,
    };
 }

+/// Registered link relations we surface from the HTTP `Link:` response header
+/// (RFC 8288). `service-doc`/`service-desc` come from RFC 8631; `api` is in the
+/// IANA Link Relations registry. These let an agent discover a site's developer
+/// docs / API description without scraping.
+const header_link_rels = [_][]const u8{ "service-doc", "service-desc", "api" };
+
+/// Parse every `Link:` response header retained on the frame, surfacing only the
+/// registered, agent-useful relations. Relative targets are resolved against the
+/// frame's base URL, mirroring `collectLink`.
+fn collectLinkHeaders(
+    arena: Allocator,
+    frame: *Frame,
+    out: *std.ArrayList(LinkRel),
+) !void {
+    for (frame._http_headers.items) |header| {
+        if (!std.ascii.eqlIgnoreCase(header.name, "link")) continue;
+
+        var it: LinkHeaderIterator = .{ .value = header.value };
+        while (it.next()) |entry| {
+            const rel_value = entry.rel orelse continue;
+
+            // The `rel` parameter is a space-separated list of relation types.
+            var rel_it = std.mem.tokenizeAny(u8, rel_value, " \t");
+            while (rel_it.next()) |rel_token| {
+                const known = for (header_link_rels) |candidate| {
+                    if (std.ascii.eqlIgnoreCase(rel_token, candidate)) break candidate;
+                } else continue;
+
+                const href = URL.resolve(arena, frame.base(), entry.target, .{ .encoding = frame.charset }) catch entry.target;
+
+                // Drop duplicate (rel, href) pairs — a target can legitimately
+                // carry the same relation across repeated headers.
+                const dup = for (out.items) |existing| {
+                    if (std.mem.eql(u8, existing.rel, known) and std.mem.eql(u8, existing.href, href)) break true;
+                } else false;
+                if (!dup) try out.append(arena, .{ .rel = known, .href = href });
+            }
+        }
+    }
+}
+
+/// One `< target >; param=value; ...` entry of an RFC 8288 `Link` header value.
+const LinkHeaderEntry = struct {
+    target: []const u8,
+    rel: ?[]const u8,
+};
+
+/// Splits a `Link:` header field value into its comma-separated link-values,
+/// skipping commas that fall inside the `<...>` target or a `"..."` quoted
+/// parameter value (RFC 8288 §3).
+const LinkHeaderIterator = struct {
+    value: []const u8,
+    i: usize = 0,
+
+    fn next(self: *LinkHeaderIterator) ?LinkHeaderEntry {
+        const v = self.value;
+
+        // Loop (rather than recurse) over malformed segments so stack usage is
+        // independent of the attacker-controlled header length.
+        while (true) {
+            // Skip separators and whitespace between link-values. The comma that
+            // ended the previous link-value is consumed here.
+            while (self.i < v.len and (std.ascii.isWhitespace(v[self.i]) or v[self.i] == ',')) : (self.i += 1) {}
+            if (self.i >= v.len) return null;
+
+            // A link-value must begin with the angle-bracketed target.
+            if (v[self.i] != '<') {
+                self.skipToNextComma();
+                continue;
+            }
+            const target_start = self.i + 1;
+            const gt = std.mem.indexOfScalarPos(u8, v, target_start, '>') orelse {
+                self.i = v.len;
+                return null;
+            };
+            const target = v[target_start..gt];
+            self.i = gt + 1;
+
+            // Parameters run to the next top-level comma, left at `self.i`.
+            const params_start = self.i;
+            self.skipToNextComma();
+            return .{ .target = target, .rel = extractRel(v[params_start..self.i]) };
+        }
+    }
+
+    // Advance until `self.i` sits on the next top-level comma (one outside a
+    // `"..."` quoted-string, honouring RFC 7230 backslash escapes) or the end
+    // of the value. The comma itself is left for the leading-separator skip.
+    fn skipToNextComma(self: *LinkHeaderIterator) void {
+        const v = self.value;
+        var in_quotes = false;
+        while (self.i < v.len) : (self.i += 1) {
+            const c = v[self.i];
+            if (c == '\\' and in_quotes) {
+                // Skip the escaped byte; guard a trailing backslash.
+                if (self.i + 1 < v.len) self.i += 1;
+            } else if (c == '"') {
+                in_quotes = !in_quotes;
+            } else if (c == ',' and !in_quotes) {
+                return;
+            }
+        }
+    }
+};
+
+/// Return the (unquoted) value of the `rel` parameter from a link-value's
+/// parameter list, or null when absent.
+fn extractRel(params: []const u8) ?[]const u8 {
+    var it = std.mem.splitScalar(u8, params, ';');
+    while (it.next()) |raw_param| {
+        const param = std.mem.trim(u8, raw_param, &std.ascii.whitespace);
+        const eq = std.mem.indexOfScalar(u8, param, '=') orelse continue;
+        if (!std.ascii.eqlIgnoreCase(std.mem.trim(u8, param[0..eq], &std.ascii.whitespace), "rel")) continue;
+
+        var val = std.mem.trim(u8, param[eq + 1 ..], &std.ascii.whitespace);
+        if (val.len >= 2 and val[0] == '"' and val[val.len - 1] == '"') {
+            val = val[1 .. val.len - 1];
+        }
+        return val;
+    }
+    return null;
+}
+
 fn collectJsonLd(
    el: *Element,
    arena: Allocator,
@@ -484,6 +638,78 @@ test "structured_data: charset and http-equiv" {
    try testing.expectEqual("text/html; charset=utf-8", findProperty(data.meta, "Content-Type").?);
 }

+test "structured_data: parseLinkHeader - single link with quoted rel" {
+    var it: LinkHeaderIterator = .{ .value = 
+        \\<https://api.example.com/openapi.json>; rel="service-desc"
+    };
+    const entry = it.next().?;
+    try testing.expectString("https://api.example.com/openapi.json", entry.target);
+    try testing.expectString("service-desc", entry.rel.?);
+    try testing.expectEqual(null, it.next());
+}
+
+test "structured_data: parseLinkHeader - multiple links and unquoted rel" {
+    var it: LinkHeaderIterator = .{ .value = 
+        \\<https://docs.example.com/>; rel=service-doc, </style.css>; rel="stylesheet"; type="text/css"
+    };
+    const first = it.next().?;
+    try testing.expectString("https://docs.example.com/", first.target);
+    try testing.expectString("service-doc", first.rel.?);
+
+    const second = it.next().?;
+    try testing.expectString("/style.css", second.target);
+    // A comma inside the quoted type param must not split the link-value.
+    try testing.expectString("stylesheet", second.rel.?);
+
+    try testing.expectEqual(null, it.next());
+}
+
+test "structured_data: parseLinkHeader - escaped quote in param does not split link" {
+    // A backslash-escaped quote inside an earlier param must not terminate the
+    // quoted-string, so the comma inside it is not treated as a separator.
+    var it: LinkHeaderIterator = .{ .value = 
+        \\<https://a/>; title="x\",y"; rel="service-doc"
+    };
+    const entry = it.next().?;
+    try testing.expectString("https://a/", entry.target);
+    try testing.expectString("service-doc", entry.rel.?);
+    try testing.expectEqual(null, it.next());
+}
+
+test "structured_data: link headers from response" {
+    const frame = try testing.test_session.createPage();
+    defer testing.test_session.removePage();
+
+    // Stand in for what frameHeaderDoneCallback records from the navigation.
+    try frame._http_headers.append(frame.arena, .{ .name = "Link", .value = 
+        \\<https://docs.example.com/>; rel="service-doc"
+    });
+    try frame._http_headers.append(frame.arena, .{ .name = "link", .value = 
+        \\<https://api.example.com/spec>; rel="service-desc", </css/site.css>; rel="stylesheet"
+    });
+
+    const doc = frame.window._document;
+    const div = try doc.createElement("div", null, frame);
+    try frame.parseHtmlAsChildren(div.asNode(), "<title>Example</title>");
+
+    const data = try collectStructuredData(div.asNode(), frame.call_arena, frame);
+
+    // service-doc + service-desc are surfaced; stylesheet is filtered out.
+    try testing.expectEqual(2, data.link_headers.len);
+    try testing.expectString("service-doc", data.link_headers[0].rel);
+    try testing.expectString("https://docs.example.com/", data.link_headers[0].href);
+    try testing.expectString("service-desc", data.link_headers[1].rel);
+    try testing.expectString("https://api.example.com/spec", data.link_headers[1].href);
+}
+
+test "structured_data: no link header yields empty list" {
+    const data = try testStructuredData(
+        \\<link rel="canonical" href="https://example.com">
+    );
+    defer testing.test_session.removePage();
+    try testing.expectEqual(0, data.link_headers.len);
+}
+
 test "structured_data: mixed content" {
    const data = try testStructuredData(
        \\<title>My Site</title>
--- a/src/cdp/domains/lp.zig
+++ b/src/cdp/domains/lp.zig
@@ -23,6 +23,7 @@ const CDP = @import("../CDP.zig");

 const Node = @import("../Node.zig");
 const DOMNode = @import("../../browser/webapi/Node.zig");
+const Robots = @import("../../network/Robots.zig");

 const markdown = lp.markdown;
 const SemanticTree = lp.SemanticTree;
@@ -36,6 +37,7 @@ pub fn processMessage(cmd: *CDP.Command) !void {
        getInteractiveElements,
        getNodeDetails,
        getStructuredData,
+        getContentSignal,
        detectForms,
        clickNode,
        fillNode,
@@ -51,6 +53,7 @@ pub fn processMessage(cmd: *CDP.Command) !void {
        .getInteractiveElements => return getInteractiveElements(cmd),
        .getNodeDetails => return getNodeDetails(cmd),
        .getStructuredData => return getStructuredData(cmd),
+        .getContentSignal => return getContentSignal(cmd),
        .detectForms => return detectForms(cmd),
        .clickNode => return clickNode(cmd),
        .fillNode => return fillNode(cmd),
@@ -201,6 +204,27 @@ fn getStructuredData(cmd: anytype) !void {
    }, .{});
 }

+// Advisory `Content-Signal` robots.txt preferences for the current document's
+// host (https://contentsignals.org). `available` is false when no robots.txt
+// has been fetched for the host — note this is the case unless `obey_robots`
+// is enabled, since the robots layer is what populates the store.
+fn getContentSignal(cmd: anytype) !void {
+    const bc = cmd.browser_context orelse return error.NoBrowserContext;
+    const frame = bc.session.currentFrame() orelse return error.FrameNotLoaded;
+    const network = bc.cdp.browser.http_client.network;
+
+    const empty: []const Robots.ContentSignal = &.{};
+    const robots_url = lp.URL.getRobotsUrl(cmd.arena, frame.url) catch {
+        return cmd.sendResult(.{ .available = false, .contentSignals = empty }, .{});
+    };
+
+    const signals = network.robot_store.getContentSignals(robots_url);
+    return cmd.sendResult(.{
+        .available = signals != null,
+        .contentSignals = signals orelse empty,
+    }, .{});
+}
+
 fn detectForms(cmd: anytype) !void {
    const bc = cmd.browser_context orelse return error.NoBrowserContext;
    const frame = bc.session.currentFrame() orelse return error.FrameNotLoaded;
@@ -410,6 +434,25 @@ test "cdp.lp: getStructuredData" {
    try testing.expect(result.get("structuredData") != null);
 }

+test "cdp.lp: getContentSignal" {
+    var ctx = try testing.context();
+    defer ctx.deinit();
+
+    const bc = try ctx.loadBrowserContext(.{});
+    _ = try bc.session.createPage();
+
+    try ctx.processMessage(.{
+        .id = 1,
+        .method = "LP.getContentSignal",
+    });
+
+    // No robots.txt fetched for the page, so the directive is reported absent
+    // rather than erroring — the command is still wired and returns the shape.
+    const result = (try ctx.getSentMessage(0)).?.object.get("result").?.object;
+    try testing.expectEqual(false, result.get("available").?.bool);
+    try testing.expect(result.get("contentSignals") != null);
+}
+
 test "cdp.lp: action tools" {
    var ctx = try testing.context();
    defer ctx.deinit();
--- a/src/network/Robots.zig
+++ b/src/network/Robots.zig
@@ -72,11 +72,20 @@ pub const Key = enum {
    @"user-agent",
    allow,
    disallow,
+    @"content-signal",
+};
+
+/// A declared content-usage preference from a `Content-Signal:` robots.txt
+/// directive (https://contentsignals.org), e.g. name="ai-train" value="no".
+/// Advisory metadata surfaced to the agent — it never affects crawl gating.
+pub const ContentSignal = struct {
+    name: []const u8,
+    value: []const u8,
 };

 /// https://www.rfc-editor.org/rfc/rfc9309.html
 pub const Robots = @This();
-pub const empty: Robots = .{ .rules = &.{} };
+pub const empty: Robots = .{ .rules = &.{}, .content_signals = &.{} };

 pub const RobotStore = struct {
    const RobotsEntry = union(enum) {
@@ -156,6 +165,21 @@ pub const RobotStore = struct {
        try self.map.put(self.allocator, duped, .{ .present = robots });
    }

+    /// Advisory `Content-Signal` preferences declared for `url`'s host, or null
+    /// when no robots.txt has been fetched/stored for it. The returned slice is
+    /// owned by the store; callers must read it synchronously (and dupe to keep
+    /// it) and must never free it.
+    pub fn getContentSignals(self: *RobotStore, url: []const u8) ?[]const ContentSignal {
+        self.mutex.lock();
+        defer self.mutex.unlock();
+
+        const entry = self.map.get(url) orelse return null;
+        return switch (entry) {
+            .present => |robots| robots.content_signals,
+            .absent => null,
+        };
+    }
+
    pub fn putAbsent(self: *RobotStore, url: []const u8) !void {
        self.mutex.lock();
        defer self.mutex.unlock();
@@ -166,6 +190,15 @@ pub const RobotStore = struct {
 };

 rules: []const Rule,
+content_signals: []const ContentSignal = &.{},
+
+/// Result of parsing a robots.txt for one user-agent: the applicable crawl
+/// rules plus any advisory `Content-Signal` preferences. The two are kept
+/// distinct so content signals can never reach `isAllowed`.
+const ParseResult = struct {
+    rules: []Rule,
+    content_signals: []ContentSignal,
+};

 const State = struct {
    entry: enum {
@@ -186,17 +219,56 @@ fn freeRulesInList(allocator: std.mem.Allocator, rules: []const Rule) void {
    }
 }

+fn freeContentSignals(allocator: std.mem.Allocator, signals: []const ContentSignal) void {
+    for (signals) |signal| {
+        allocator.free(signal.name);
+        allocator.free(signal.value);
+    }
+}
+
+/// Parse a `Content-Signal:` value — a comma-separated list of `name=value`
+/// pairs (e.g. `ai-train=no, search=yes`) — into the given list. Names and
+/// values are lower-cased and owned by `allocator`.
+fn appendContentSignals(
+    allocator: std.mem.Allocator,
+    list: *std.ArrayList(ContentSignal),
+    value: []const u8,
+) !void {
+    var iter = std.mem.splitScalar(u8, value, ',');
+    while (iter.next()) |raw_pair| {
+        const pair = std.mem.trim(u8, raw_pair, &std.ascii.whitespace);
+        const eq = std.mem.indexOfScalar(u8, pair, '=') orelse continue;
+
+        const name_raw = std.mem.trim(u8, pair[0..eq], &std.ascii.whitespace);
+        const value_raw = std.mem.trim(u8, pair[eq + 1 ..], &std.ascii.whitespace);
+        if (name_raw.len == 0) continue;
+
+        const name = try std.ascii.allocLowerString(allocator, name_raw);
+        errdefer allocator.free(name);
+        const val = try std.ascii.allocLowerString(allocator, value_raw);
+        errdefer allocator.free(val);
+
+        try list.append(allocator, .{ .name = name, .value = val });
+    }
+}
+
 fn parseRulesWithUserAgent(
    allocator: std.mem.Allocator,
    user_agent: []const u8,
    raw_bytes: []const u8,
-) ![]Rule {
+) !ParseResult {
    var rules: std.ArrayList(Rule) = .empty;
    defer rules.deinit(allocator);

    var wildcard_rules: std.ArrayList(Rule) = .empty;
    defer wildcard_rules.deinit(allocator);

+    var content_signals: std.ArrayList(ContentSignal) = .empty;
+    defer content_signals.deinit(allocator);
+
+    var wildcard_content_signals: std.ArrayList(ContentSignal) = .empty;
+    defer wildcard_content_signals.deinit(allocator);
+
    var state: State = .{ .entry = .not_in_entry, .has_rules = false };

    // https://en.wikipedia.org/wiki/Byte_order_mark
@@ -308,22 +380,51 @@ fn parseRulesWithUserAgent(
                    },
                }
            },
+            .@"content-signal" => {
+                // A group member like allow/disallow, so it sets has_rules
+                // (a following `User-agent:` opens a new group), but it is
+                // captured into a separate list and never reaches isAllowed.
+                defer state.has_rules = true;
+
+                switch (state.entry) {
+                    .in_our_entry => try appendContentSignals(allocator, &content_signals, value),
+                    .in_wildcard_entry => try appendContentSignals(allocator, &wildcard_content_signals, value),
+                    .in_other_entry, .not_in_entry => {},
+                }
+            },
        }
    }

    // If we have rules for our specific User-Agent, we will use those rules.
    // If we don't have any rules, we fallback to using the wildcard ("*") rules.
-    if (rules.items.len > 0) {
+    const out_rules = if (rules.items.len > 0) blk: {
        freeRulesInList(allocator, wildcard_rules.items);
-        return try rules.toOwnedSlice(allocator);
-    } else {
+        break :blk try rules.toOwnedSlice(allocator);
+    } else blk: {
        freeRulesInList(allocator, rules.items);
-        return try wildcard_rules.toOwnedSlice(allocator);
+        break :blk try wildcard_rules.toOwnedSlice(allocator);
+    };
+    errdefer {
+        freeRulesInList(allocator, out_rules);
+        allocator.free(out_rules);
    }
+
+    // Content signals follow the same specific-else-wildcard precedence,
+    // chosen independently of the rule selection above.
+    const out_signals = if (content_signals.items.len > 0) blk: {
+        freeContentSignals(allocator, wildcard_content_signals.items);
+        break :blk try content_signals.toOwnedSlice(allocator);
+    } else blk: {
+        freeContentSignals(allocator, content_signals.items);
+        break :blk try wildcard_content_signals.toOwnedSlice(allocator);
+    };
+
+    return .{ .rules = out_rules, .content_signals = out_signals };
 }

 pub fn fromBytes(allocator: std.mem.Allocator, user_agent: []const u8, bytes: []const u8) !Robots {
-    const rules = try parseRulesWithUserAgent(allocator, user_agent, bytes);
+    const parsed = try parseRulesWithUserAgent(allocator, user_agent, bytes);
+    const rules = parsed.rules;

    // sort by order once.
    std.mem.sort(Rule, rules, {}, struct {
@@ -357,12 +458,14 @@ pub fn fromBytes(allocator: std.mem.Allocator, user_agent: []const u8, bytes: []
        }
    }.lessThan);

-    return .{ .rules = rules };
+    return .{ .rules = rules, .content_signals = parsed.content_signals };
 }

 pub fn deinit(self: *Robots, allocator: std.mem.Allocator) void {
    freeRulesInList(allocator, self.rules);
    allocator.free(self.rules);
+    freeContentSignals(allocator, self.content_signals);
+    allocator.free(self.content_signals);
 }

 /// There are rules for how the pattern in robots.txt should be matched.
@@ -476,10 +579,13 @@ test "Robots: simple robots.txt" {
        \\
    ;

-    const rules = try parseRulesWithUserAgent(allocator, "GoogleBot", file);
+    const parsed = try parseRulesWithUserAgent(allocator, "GoogleBot", file);
+    const rules = parsed.rules;
    defer {
        freeRulesInList(allocator, rules);
        allocator.free(rules);
+        freeContentSignals(allocator, parsed.content_signals);
+        allocator.free(parsed.content_signals);
    }

    try std.testing.expectEqual(1, rules.len);
@@ -1003,3 +1109,101 @@ test "Robots: blank lines don't end entries" {
    try std.testing.expect(robots.isAllowed("/admin/") == false);
    try std.testing.expect(robots.isAllowed("/public/") == true);
 }
+
+test "Robots: content-signal captured and gating unaffected" {
+    const allocator = std.testing.allocator;
+
+    var robots = try Robots.fromBytes(allocator, "MyBot",
+        \\User-agent: *
+        \\Content-Signal: ai-train=no, search=yes, ai-input=yes
+        \\Disallow: /private/
+        \\
+    );
+    defer robots.deinit(allocator);
+
+    try std.testing.expectEqual(3, robots.content_signals.len);
+    try std.testing.expectEqualStrings("ai-train", robots.content_signals[0].name);
+    try std.testing.expectEqualStrings("no", robots.content_signals[0].value);
+    try std.testing.expectEqualStrings("search", robots.content_signals[1].name);
+    try std.testing.expectEqualStrings("yes", robots.content_signals[1].value);
+    try std.testing.expectEqualStrings("ai-input", robots.content_signals[2].name);
+
+    // The whole point: Content-Signal is advisory metadata, it must not move
+    // the crawl gate. Disallow/Allow still decide isAllowed.
+    try std.testing.expect(robots.isAllowed("/private/page") == false);
+    try std.testing.expect(robots.isAllowed("/public/page") == true);
+}
+
+test "Robots: content-signal mixed casing is normalized" {
+    const allocator = std.testing.allocator;
+
+    var robots = try Robots.fromBytes(allocator, "MyBot",
+        \\User-agent: *
+        \\Content-Signal: AI-Train=NO
+        \\
+    );
+    defer robots.deinit(allocator);
+
+    try std.testing.expectEqual(1, robots.content_signals.len);
+    try std.testing.expectEqualStrings("ai-train", robots.content_signals[0].name);
+    try std.testing.expectEqualStrings("no", robots.content_signals[0].value);
+}
+
+test "Robots: content-signal absent yields empty list" {
+    const allocator = std.testing.allocator;
+
+    var robots = try Robots.fromBytes(allocator, "MyBot",
+        \\User-agent: *
+        \\Disallow: /admin/
+        \\
+    );
+    defer robots.deinit(allocator);
+
+    try std.testing.expectEqual(0, robots.content_signals.len);
+    try std.testing.expect(robots.isAllowed("/admin/") == false);
+}
+
+test "Robots: content-signal prefers specific user-agent over wildcard" {
+    const allocator = std.testing.allocator;
+
+    const file =
+        \\User-agent: *
+        \\Content-Signal: ai-train=yes
+        \\
+        \\User-agent: MyBot
+        \\Content-Signal: ai-train=no
+        \\
+    ;
+
+    var mine = try Robots.fromBytes(allocator, "MyBot", file);
+    defer mine.deinit(allocator);
+    try std.testing.expectEqual(1, mine.content_signals.len);
+    try std.testing.expectEqualStrings("no", mine.content_signals[0].value);
+
+    var other = try Robots.fromBytes(allocator, "OtherBot", file);
+    defer other.deinit(allocator);
+    try std.testing.expectEqual(1, other.content_signals.len);
+    try std.testing.expectEqualStrings("yes", other.content_signals[0].value);
+}
+
+test "Robots: RobotStore.getContentSignals round-trips" {
+    const allocator = std.testing.allocator;
+
+    var store = RobotStore.init(allocator);
+    defer store.deinit();
+
+    const robots = try store.robotsFromBytes("MyBot",
+        \\User-agent: *
+        \\Content-Signal: ai-train=no
+        \\
+    );
+    try store.put("https://example.com/robots.txt", robots);
+
+    const signals = store.getContentSignals("https://example.com/robots.txt").?;
+    try std.testing.expectEqual(1, signals.len);
+    try std.testing.expectEqualStrings("ai-train", signals[0].name);
+    try std.testing.expectEqualStrings("no", signals[0].value);
+
+    // Unknown host has no stored robots.
+    try std.testing.expectEqual(null, store.getContentSignals("https://other.com/robots.txt"));
+}