From 3e801fb83bfd25926f3002058d45ed93f669e79d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A0=20Arrufat?= Date: Mon, 13 Jul 2026 10:54:13 +0200 Subject: [PATCH] script: generate the PandaScript skill from the tool schemas The /save script documentation lived as a hand-maintained string in Agent.zig whose primitives table drifted whenever a tool changed. Move it to src/script/skill.zig and render the reference (signatures, option lists, enums, defaults, per-parameter descriptions) from Schema.all() at first use, keeping the curated per-tool notes behind an exhaustive switch on Tool so a new or renamed tool is a compile error until its doc entry exists. The rendered skill is shared by three consumers: - the /save and revision system prompts (built lazily in Agent.zig) - a new mcp://skill/pandascript resource - `zig build skills`, which writes zig-out/skills//SKILL.md with Claude Code frontmatter via a registry-based generator exe, so future Lightpanda skills are one registry entry each Schema.FieldEntry now retains per-parameter schema descriptions, which previously existed only in the raw JSON. --- build.zig | 32 ++++ src/agent/Agent.zig | 163 +++++--------------- src/lightpanda.zig | 1 + src/main_skills.zig | 61 ++++++++ src/mcp/resources.zig | 20 +++ src/script/Schema.zig | 12 ++ src/script/skill.zig | 341 ++++++++++++++++++++++++++++++++++++++++++ 7 files changed, 501 insertions(+), 129 deletions(-) create mode 100644 src/main_skills.zig create mode 100644 src/script/skill.zig diff --git a/build.zig b/build.zig index c0dd98445..26f678679 100644 --- a/build.zig +++ b/build.zig @@ -176,6 +176,38 @@ pub fn build(b: *Build) !void { run_step.dependOn(&run_cmd.step); } + { + // skills generator + const exe = b.addExecutable(.{ + .name = "lightpanda-skills", + .use_llvm = true, + .root_module = b.createModule(.{ + .root_source_file = b.path("src/main_skills.zig"), + .target = target, + .optimize = optimize, + .imports = &.{ + .{ .name = "lightpanda", .module = lightpanda_module }, + }, + }), + }); + + const exe_check = b.addLibrary(.{ + .name = "skills_check", + .root_module = exe.root_module, + }); + check.dependOn(&exe_check.step); + + const run_cmd = b.addRunArtifact(exe); + const out_dir = run_cmd.addOutputDirectoryArg("skills"); + const install = b.addInstallDirectory(.{ + .source_dir = out_dir, + .install_dir = .prefix, + .install_subdir = "skills", + }); + const skills_step = b.step("skills", "Generate LLM skill docs (zig-out/skills//SKILL.md)"); + skills_step.dependOn(&install.step); + } + { // test const tests = b.addTest(.{ diff --git a/src/agent/Agent.zig b/src/agent/Agent.zig index dc02488d1..e9f243879 100644 --- a/src/agent/Agent.zig +++ b/src/agent/Agent.zig @@ -77,133 +77,12 @@ const default_system_prompt = browser_tools.driver_guidance ++ \\ the Credentials section above) before reporting unavailable. ; -/// Skill-like documentation for writing Lightpanda agent script -/// Used in the system prompt of the `/save` command. -const script_skill = - \\# Writing Lightpanda agent scripts - \\ - \\Run with: - \\ - \\```console - \\./lightpanda agent script.js - \\``` - \\ - \\## Mental model (get this right first) - \\ - \\The script runs in its **own V8 context** — neither the page nor Node.js: - \\ - \\- `Page` is the only global. `new Page()` makes a page and `await page.goto(url)` navigates it; every other primitive is a **method on that page**: `const page = new Page(); await page.goto(url); page.extract({...}); page.click(sel);`. - \\- No `window`, `document`, DOM, `localStorage` — read pages with `page.extract(...)`, run page-side JS only via `page.evaluate("...")`. - \\- No `require`, `process`, `fs`, npm. Standard ECMAScript built-ins only (`JSON`, `Map`, template literals, …). - \\- `page.goto(...)` is **async — always `await` it**. Page methods are **synchronous**: `const data = page.extract({...})`, never `await page.extract(...)`. The script body runs as an async function, so top-level `await` is allowed. - \\- **Re-navigating reuses the same page**: `await page.goto(url2)` keeps `page` valid and points it at the new URL, discarding the old page — read it before navigating away. Independent URLs don't share a page: make a `new Page()` for each and load them in parallel (fan-out, best practice 2). - \\- Page `evaluate("...")` cannot see script variables — interpolate values into the string. Script code cannot see page variables. - \\- Variables persist across navigations within one run, so cross-page aggregation is plain JS. - \\- **`return ` is the script's output**, printed automatically (objects/arrays as JSON). End with `return page.extract({...});` or `return results;`. A bare trailing expression is NOT printed; neither is `console.log(JSON.stringify(...))`. - \\ - \\## Primitives - \\ - \\`Page` is the only global; `new Page()` makes a page and everything else is a method on it. - \\ - \\| Call | Notes | - \\|------|-------| - \\| `new Page()` | Makes a page object. No navigation yet — call `page.goto(url)` before any other method. Make several to navigate in parallel (fan-out, best practice 2). | - \\| `await page.goto(url[, { timeout }])` | **Async — must be `await`ed.** Navigates the page (re-navigating reuses the same object). Waits for `load`. Default timeout 10000 ms. Rejects on navigation failure; a **timeout does NOT reject** (the page may still be usable). | - \\| `page.close()` | Marks the page done; later method calls on it error. The page is otherwise reclaimed at script end. | - \\| `page.extract(schema)` | The only primitive returning a real JS value (object/array). See schema below. | - \\| `page.evaluate(script[, { url, timeout, save }])` | Page-side JS escape hatch; returns text (JSON for objects/arrays). | - \\| `page.click(sel)` / `page.hover(sel)` | | - \\| `page.fill(sel, value)` / `page.selectOption(sel, value)` | | - \\| `page.setChecked(sel[, checked])` | `checked` defaults to `true`. | - \\| `page.press(sel, key)` / `page.press(null, key)` / `page.press({ key })` | Selector first! `page.press("Enter")` binds "Enter" to `selector` and fails. | - \\| `page.scroll()` / `page.scroll({ x, y })` | | - \\| `page.waitForSelector(sel[, { timeout }])` | `waitFor*` default timeout 5000 ms. | - \\| `page.waitForScript(js[, { timeout }])` | Re-evaluates page JS until truthy. | - \\| `page.waitForState(state[, { timeout }])` | `"load"`, `"domcontentloaded"`, `"networkalmostidle"`, `"networkidle"`, `"done"`. | - \\ - \\Calling convention: leading positionals + optional trailing options object, or one object with everything (`page.waitForSelector("#row", { timeout: 2000 })` ≡ `page.waitForSelector({ selector: "#row", timeout: 2000 })`). A bare option positional (`page.waitForSelector("#row", 2000)`) and a field passed both ways are `invalid arguments`. `null` skips a positional. Arguments must be JSON-serializable. - \\ - \\CSS selectors only — `backendNodeId`s don't exist here. Standard CSS only: no jQuery `:contains()` or Playwright `:has-text()`. - \\ - \\## extract schema - \\ - \\Keys = output field names; values pick what to lift (not a JSON Schema): - \\ - \\```js - \\const { stories } = page.extract({ - \\ stories: [{ - \\ selector: "tr.athing", // one record per match - \\ limit: 5, - \\ fields: { // resolved relative to each match - \\ title: ".titleline > a", // first match's text (null if missing) - \\ url: { selector: ".titleline > a", attr: "href" }, - \\ text: "" // "" = the matched element's own text - \\ } - \\ }] - \\}); - \\``` - \\ - \\- `"sel"` → first match's text; `["sel"]` → all matches' text; `{ selector, attr }` / `[{ selector, attr }]` → attribute(s); `limit: N` caps any array form. - \\- Every value is a string or null — parse numbers in script logic. - \\- Empty arrays are valid results; if **every** field misses, extract throws ("no schema selector matched any element") → your selectors are wrong, not the page empty. - \\- An object schema always returns an object (destructure it); a bare array schema returns the array directly. - \\- No `save:` option in scripts — keep results in variables. - \\ - \\## Best practices - \\ - \\1. **Navigate, settle, read.** After `await page.goto` on a dynamic page (feeds, search results, comment threads), call `page.waitForState("networkidle")` or `page.waitForSelector(...)` before extracting. Most static pages are complete at `load` — don't wait blindly. - \\2. **List-to-detail — fan out independent pages.** Extract the list, then open one page per item and start every navigation together so the detail pages load in parallel instead of one-after-another: - \\ ```js - \\ const list = new Page(); - \\ await list.goto(listUrl); - \\ const { items } = list.extract({ items: [{ selector: "a.row", fields: { url: { attr: "href" } } }] }); - \\ - \\ const pages = items.map(() => new Page()); - \\ await Promise.all(pages.map((p, i) => p.goto(items[i].url))); // all in flight at once - \\ return pages.map((p, i) => ({ ...items[i], ...p.extract({ /* schema */ }) })); - \\ ``` - \\ - Concurrency is bounded by the HTTP connection pool (40 by default, `--http-max-concurrent`); extra navigations queue rather than fail. For long lists, fan out in batches and `page.close()` each page once read so its memory is reclaimed. - \\ - `Promise.all` rejects the whole batch if any `goto` *fails* (a timeout does not reject); use `Promise.allSettled` when partial results are fine. - \\ - Walk **serially** on one page (`for (const it of items) { await page.goto(it.url); … }`) only when the steps depend on each other — each page decides the next URL, or they share login/session state. - \\3. **`evaluate` is a last resort, not a reading tool.** A `querySelectorAll`-and-parse `page.evaluate` block is always wrong: lift the raw strings with `page.extract`, then trim/split/parse them in top-level JS. Reserve `page.evaluate` for behavior that must run inside the page and no builtin covers — and remember its state dies on every navigation/reload, while script variables persist. - \\4. **Credentials via `$LP_*` placeholders** in any string argument (`page.fill("#pw", "$LP_HN_PASSWORD")`). Never inline a real secret; placeholders resolve inside the Lightpanda process. - \\5. **Unique selectors.** Disambiguate with attributes/position: `input[type="submit"][value="login"]`, not `input[type="submit"]`. - \\6. **Let failures fail.** Primitives throw on error and stop the script — only `try/catch` where you have a real fallback (e.g. optional cookie banner: `try { page.click("#accept") } catch {}`). - \\7. **End with `return `.** `console.log` is for debug output only and doesn't JSON-format objects. - \\8. Modern, readable JS: `const`/`let`, `for (const x of xs)`, template literals, destructuring, 2-space indent. - \\9. **Comment the intent of each block.** Put a one-line `//` comment above each logical step describing what it accomplishes toward the goal (not restating the call). One comment per block, not per line — skip self-evident lines: - \\ ```js - \\ // Load the Hacker News front page - \\ const page = new Page(); - \\ await page.goto("https://news.ycombinator.com"); - \\ - \\ // Pull the top 5 stories (title + link) - \\ const { stories } = page.extract({ stories: [{ selector: "tr.athing", limit: 5, fields: { title: ".titleline > a", url: { selector: ".titleline > a", attr: "href" } } }] }); - \\ - \\ // Open each story page in parallel and read its text - \\ const pages = stories.map(() => new Page()); - \\ await Promise.all(pages.map((p, i) => p.goto(stories[i].url))); - \\ ``` - \\ - \\## Common errors - \\ - \\| Error | Cause / fix | - \\|-------|-------------| - \\| `extract is not defined` (or click/fill/…) | These are methods on the page object, not globals → `const page = new Page(); await page.goto(url); page.extract(...)` | - \\| `Page must be called with new` | `Page(...)` called without `new` → `const page = new Page();` | - \\| `page is not navigated or has been closed` | A method on a fresh `new Page()` (or a closed page) → `await page.goto(url)` first | - \\| `page handle is no longer valid` | Used a page after a later `goto` on the **same** page replaced it → read it before navigating away. Sibling pages from other `new Page()` calls stay valid. | - \\| `document is not defined` | DOM API in script context → use `page.extract` or `page.evaluate` | - \\| `require is not defined` | Not Node.js | - \\| `no page loaded - run page.goto(url) first` | Page method before navigation | - \\| `invalid arguments` | Wrong arity/shape, non-JSON value, or a field set both positionally and in options | - \\| `extract: no schema selector matched any element` | All schema fields missed → fix selectors | - \\| `press` fails with one string arg | Selector-first: use `page.press(null, "Enter")` or `page.press({ key: "Enter" })` | -; - -// Sytem prompt of the `/save` command -// With the save instructions and the skill-like agent script documentation. -const save_system_prompt = browser_tools.save_synthesis_prompt ++ "\n\n" ++ script_skill; +// System prompt of the `/save` command: the save instructions plus the +// script skill (`lp.skill`), whose primitives reference is rendered from +// the tool schemas at first use — hence lazy rather than comptime. +var save_prompts_once = std.once(initSavePrompts); +var save_system_prompt: []const u8 = undefined; +var save_revision_system_prompt: []const u8 = undefined; // Swapped in instead of `save_system_prompt` when the user message carries a // previously saved script: the "this session" framing above would otherwise @@ -216,7 +95,23 @@ const save_revision_note = \\a change, and add what this session contributes. Output the complete \\updated script — never a fragment, diff, or continuation. ; -const save_revision_system_prompt = browser_tools.save_synthesis_prompt ++ "\n" ++ save_revision_note ++ "\n" ++ script_skill; + +/// Panics on OOM like `Schema.all()` — the inputs are static, process-lifetime +/// strings, so failure is a build bug, not a runtime condition. +fn initSavePrompts() void { + const a = std.heap.page_allocator; + save_system_prompt = std.mem.concat(a, u8, &.{ + browser_tools.save_synthesis_prompt, "\n\n", lp.skill.text(), + }) catch @panic("OOM building save prompt"); + save_revision_system_prompt = std.mem.concat(a, u8, &.{ + browser_tools.save_synthesis_prompt, "\n", save_revision_note, "\n", lp.skill.text(), + }) catch @panic("OOM building save prompt"); +} + +fn savePrompt(revision: bool) []const u8 { + save_prompts_once.call(); + return if (revision) save_revision_system_prompt else save_system_prompt; +} const synthesis_prompt = \\You have used your tool budget or cannot finish the exploration. @@ -1212,7 +1107,7 @@ fn synthesizeSaveTo(self: *Agent, arena: std.mem.Allocator, path: []const u8, mo // regular turns keep the driver prompt. (`messages[0]` is the system // message — rollback and prune never touch it.) const plain_system = self.conversation.messages.items[0].content; - self.conversation.messages.items[0].content = if (previous_script != null) save_revision_system_prompt else save_system_prompt; + self.conversation.messages.items[0].content = savePrompt(previous_script != null); defer self.conversation.messages.items[0].content = plain_system; const ma = self.conversation.arena.allocator(); @@ -1934,6 +1829,16 @@ test { _ = settings; } +test "savePrompt: save instructions followed by the rendered script skill" { + const prompt = savePrompt(false); + try std.testing.expect(std.mem.startsWith(u8, prompt, browser_tools.save_synthesis_prompt)); + try std.testing.expect(std.mem.endsWith(u8, prompt, lp.skill.text())); + + const revision = savePrompt(true); + try std.testing.expect(std.mem.indexOf(u8, revision, save_revision_note) != null); + try std.testing.expect(std.mem.endsWith(u8, revision, lp.skill.text())); +} + test "capToolOutput: passes through when under cap" { const ta = std.testing.allocator; const out = capToolOutput(ta, "short"); diff --git a/src/lightpanda.zig b/src/lightpanda.zig index b9e9d9cc6..39e9dc2e9 100644 --- a/src/lightpanda.zig +++ b/src/lightpanda.zig @@ -51,6 +51,7 @@ pub const Command = @import("script/command.zig").Command; pub const Recorder = @import("script/Recorder.zig"); pub const Runtime = @import("script/Runtime.zig"); pub const Schema = @import("script/Schema.zig"); +pub const skill = @import("script/skill.zig"); pub const cookies = @import("cookies.zig"); pub const build_config = @import("build_config"); pub const crash_handler = @import("crash_handler.zig"); diff --git a/src/main_skills.zig b/src/main_skills.zig new file mode 100644 index 000000000..ad504ef16 --- /dev/null +++ b/src/main_skills.zig @@ -0,0 +1,61 @@ +// Copyright (C) 2023-2026 Lightpanda (Selecy SAS) +// +// Francis Bouvier +// Pierre Tachoire +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +//! Generates the LLM skill tree (`//SKILL.md`), invoked by +//! `zig build skills`. The documents are rendered from the same code the +//! runtime uses, so they can't go stale. + +const std = @import("std"); +const lp = @import("lightpanda"); + +const Skill = struct { + name: []const u8, + write: *const fn (writer: *std.Io.Writer) std.Io.Writer.Error!void, +}; + +const skills = [_]Skill{ + .{ .name = lp.skill.name, .write = lp.skill.write }, +}; + +pub fn main() !void { + const allocator = std.heap.c_allocator; + + // usage: skills + var args = try std.process.argsWithAllocator(allocator); + _ = args.next(); // executable name + const out_path = args.next() orelse { + std.debug.print("usage: lightpanda-skills \n", .{}); + return error.MissingArgument; + }; + + var out_dir = try std.fs.cwd().makeOpenPath(out_path, .{}); + defer out_dir.close(); + + for (skills) |s| { + var skill_dir = try out_dir.makeOpenPath(s.name, .{}); + defer skill_dir.close(); + + const file = try skill_dir.createFile("SKILL.md", .{}); + defer file.close(); + + var buffer: [4096]u8 = undefined; + var writer = file.writer(&buffer); + try s.write(&writer.interface); + try writer.end(); + } +} diff --git a/src/mcp/resources.zig b/src/mcp/resources.zig index ed413c339..d3a5aafc4 100644 --- a/src/mcp/resources.zig +++ b/src/mcp/resources.zig @@ -19,6 +19,12 @@ pub const resource_list = [_]protocol.Resource{ .description = "The token-efficient markdown representation of the current page", .mimeType = "text/markdown", }, + .{ + .uri = "mcp://skill/pandascript", + .name = "PandaScript skill", + .description = lp.skill.description, + .mimeType = "text/markdown", + }, }; pub fn handleList(server: *Server, req: protocol.Request) !void { @@ -65,11 +71,13 @@ const ResourceStreamingResult = struct { const ResourceUri = enum { @"mcp://page/html", @"mcp://page/markdown", + @"mcp://skill/pandascript", }; const resource_map = std.StaticStringMap(ResourceUri).initComptime(.{ .{ "mcp://page/html", .@"mcp://page/html" }, .{ "mcp://page/markdown", .@"mcp://page/markdown" }, + .{ "mcp://skill/pandascript", .@"mcp://skill/pandascript" }, }); pub fn handleRead(server: *Server, arena: std.mem.Allocator, req: protocol.Request) !void { @@ -86,6 +94,16 @@ pub fn handleRead(server: *Server, arena: std.mem.Allocator, req: protocol.Reque return server.sendError(req_id, .InvalidRequest, "Resource not found"); }; + if (uri == .@"mcp://skill/pandascript") { + return server.sendResult(req_id, .{ + .contents = &.{.{ + .uri = params.uri, + .mimeType = "text/markdown", + .text = lp.skill.text(), + }}, + }); + } + const frame = server.session.currentFrame() orelse { return server.sendError(req_id, .FrameNotLoaded, "Page not loaded"); }; @@ -93,10 +111,12 @@ pub fn handleRead(server: *Server, arena: std.mem.Allocator, req: protocol.Reque const format: Format = switch (uri) { .@"mcp://page/html" => .html, .@"mcp://page/markdown" => .markdown, + .@"mcp://skill/pandascript" => unreachable, }; const mime_type: []const u8 = switch (uri) { .@"mcp://page/html" => "text/html", .@"mcp://page/markdown" => "text/markdown", + .@"mcp://skill/pandascript" => unreachable, }; const result: ResourceStreamingResult = .{ diff --git a/src/script/Schema.zig b/src/script/Schema.zig index 0fdc22a15..d2cf955de 100644 --- a/src/script/Schema.zig +++ b/src/script/Schema.zig @@ -50,6 +50,8 @@ pub const FieldEntry = struct { /// Allowed values when the schema constrains the field with `enum`; empty /// otherwise. The REPL completer/hinter offers these as value suggestions. enum_values: []const []const u8 = &.{}, + /// The schema's per-parameter `description` string; empty when absent. + description: []const u8 = "", /// `backendNodeId` is ephemeral, never replayable. Boolean fields /// matching the schema default are cosmetic noise. @@ -408,6 +410,7 @@ fn buildOne(arena: std.mem.Allocator, tool: BrowserTool, td: BrowserTool.Definit .field_type = fieldTypeOf(entry.value_ptr.*), .default_true = booleanDefaultTrue(entry.value_ptr.*), .enum_values = try enumValuesOf(arena, entry.value_ptr.*), + .description = descriptionOf(entry.value_ptr.*), }; } info.fields = fields; @@ -462,6 +465,13 @@ fn booleanDefaultTrue(value: std.json.Value) bool { return d == .bool and d.bool; } +fn descriptionOf(value: std.json.Value) []const u8 { + if (value != .object) return ""; + const d = value.object.get("description") orelse return ""; + if (d != .string) return ""; + return d.string; +} + fn enumValuesOf(arena: std.mem.Allocator, value: std.json.Value) ![]const []const u8 { if (value != .object) return &.{}; return jsonStringArray(arena, value.object.get("enum") orelse return &.{}); @@ -611,6 +621,8 @@ test "all: comptime tool defs reduce cleanly" { if (std.mem.eql(u8, f.name, "checked")) checked_default_true = f.default_true; } try testing.expect(checked_default_true); + const timeout = goto.findField("timeout").?; + try testing.expect(timeout.description.len > 0); } test "parseValue: single-required positional binds" { diff --git a/src/script/skill.zig b/src/script/skill.zig new file mode 100644 index 000000000..7427fc297 --- /dev/null +++ b/src/script/skill.zig @@ -0,0 +1,341 @@ +// Copyright (C) 2023-2026 Lightpanda (Selecy SAS) +// +// Francis Bouvier +// Pierre Tachoire +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +//! Skill documentation for writing PandaScript agent scripts, assembled +//! from hand-written prose and a primitives reference generated from the +//! tool schemas (`Schema.all()`), so signatures, enums, and defaults +//! can't drift from the code. Consumed by the `/save` system prompt and +//! exported as `SKILL.md` by `zig build skills`. + +const std = @import("std"); +const lp = @import("lightpanda"); +const browser_tools = lp.tools; +const Schema = @import("Schema.zig"); + +pub const name = "pandascript"; +pub const description = "Write PandaScript agent scripts (.js) — Lightpanda's replayable browser-automation format, run token-free with `lightpanda agent script.js`."; + +/// The skill body (no frontmatter). Lazily rendered once, process lifetime. +pub fn text() []const u8 { + text_once.call(); + return text_buffer.written(); +} + +/// Frontmatter + body, the on-disk `SKILL.md` shape. +pub fn write(writer: *std.Io.Writer) std.Io.Writer.Error!void { + try writer.print("---\nname: {s}\ndescription: {s}\n---\n\n", .{ name, description }); + try writer.writeAll(text()); +} + +var text_buffer: std.Io.Writer.Allocating = undefined; +var text_once = std.once(initText); + +/// Panics on failure — the inputs are comptime tool defs, so any render +/// error is a build-time bug. +fn initText() void { + text_buffer = .init(std.heap.page_allocator); + render(&text_buffer.writer) catch |err| { + std.debug.panic("failed to render the {s} skill: {s}", .{ name, @errorName(err) }); + }; +} + +fn render(w: *std.Io.Writer) std.Io.Writer.Error!void { + try w.writeAll(prose_head ++ "\n\n"); + try writeReference(w); + try w.writeAll(prose_tail ++ "\n"); +} + +/// The generated primitives reference: one table row per recorded tool +/// plus an options list surfacing the per-parameter schema descriptions. +fn writeReference(w: *std.Io.Writer) std.Io.Writer.Error!void { + try w.writeAll(table_head ++ "\n"); + for (Schema.all()) |*s| { + if (!s.tool.isRecorded()) continue; + try writeRow(w, s); + } + + try w.writeAll("\nOptions (the trailing `{ … }` object; every option may be omitted):\n\n"); + for (Schema.all()) |*s| { + if (!s.tool.isRecorded() or s.tool == .extract) continue; + if (!hasOptions(s)) continue; + try w.print("- `{s}`:", .{s.tool_name}); + for (s.fields) |f| { + if (!isOption(s, f.name)) continue; + try w.print(" `{s}`", .{f.name}); + if (f.description.len > 0) try w.print(" — {s}", .{f.description}); + } + try w.writeAll("\n"); + } + try w.writeAll("\n"); +} + +fn writeRow(w: *std.Io.Writer, s: *const Schema) std.Io.Writer.Error!void { + try w.writeAll("| `"); + if (s.tool.isAsync()) try w.writeAll("await "); + try w.print("page.{s}(", .{s.tool_name}); + for (s.positional, 0..) |p, i| { + const optional = positionalOptional(s, p); + // A trailing optional positional is bracketed; a leading one + // (press's selector) renders plain and its note explains `null`. + if (i > 0) try w.writeAll(if (optional) "[, " else ", "); + try w.writeAll(p); + if (i > 0 and optional) try w.writeAll("]"); + } + // The script form of extract takes the schema as its only argument + // (`Runtime.extractArgs`); every other tool accepts the options object. + if (s.tool != .extract and hasOptions(s)) { + try w.writeAll(if (s.positional.len > 0) "[, { " else "[{ "); + var i: usize = 0; + for (s.fields) |f| { + if (!isOption(s, f.name)) continue; + if (i > 0) try w.writeAll(", "); + try w.writeAll(f.name); + i += 1; + } + try w.writeAll(" }]"); + } + try w.writeAll(")` | "); + + var wrote = false; + if (s.tool.isAsync()) { + try w.writeAll("**Async — must be `await`ed.**"); + wrote = true; + } + const n = note(s.tool); + if (n.len > 0) { + if (wrote) try w.writeAll(" "); + try w.writeAll(n); + wrote = true; + } + for (s.fields) |f| { + if (f.enum_values.len == 0 or std.mem.eql(u8, f.name, "backendNodeId")) continue; + if (wrote) try w.writeAll(" "); + try w.print("`{s}`: one of", .{f.name}); + for (f.enum_values, 0..) |v, i| { + try w.print("{s} `\"{s}\"`", .{ if (i == 0) "" else ",", v }); + } + try w.writeAll("."); + wrote = true; + } + for (s.positional) |p| { + const f = s.findField(p) orelse continue; + if (!f.default_true) continue; + if (wrote) try w.writeAll(" "); + try w.print("`{s}` defaults to `true`.", .{p}); + wrote = true; + } + try w.writeAll(" |\n"); +} + +/// Non-positional schema fields surfacing in the script signature. +/// `backendNodeId` is ephemeral and has no script form. +fn isOption(s: *const Schema, field_name: []const u8) bool { + if (std.mem.eql(u8, field_name, "backendNodeId")) return false; + for (s.positional) |p| { + if (std.mem.eql(u8, p, field_name)) return false; + } + return true; +} + +fn hasOptions(s: *const Schema) bool { + for (s.fields) |f| { + if (isOption(s, f.name)) return true; + } + return false; +} + +/// Whether a positional may be omitted in the script form. The schema marks +/// `selector` optional (backendNodeId alternative), but locator tools have +/// no such alternative in scripts, so their selector is effectively required. +fn positionalOptional(s: *const Schema, p: []const u8) bool { + if (s.findField(p)) |f| { + if (f.default_true) return true; + } + if (std.mem.eql(u8, p, "selector") and s.tool.needsLocator()) return false; + for (s.required) |r| { + if (std.mem.eql(u8, r, p)) return false; + } + return true; +} + +/// Curated per-tool script semantics the JSON schema can't express. +/// Exhaustive so adding or renaming a tool is a compile error until it +/// makes an explicit choice; non-recorded tools have no script form. +fn note(tool: browser_tools.Tool) []const u8 { + return switch (tool) { + .goto => "Navigates the page (re-navigating reuses the same object). Waits for `load`. Rejects on navigation failure; a **timeout does NOT reject** (the page may still be usable). Default timeout 10000 ms.", + .evaluate => "Page-side JS escape hatch; returns text (JSON for objects/arrays).", + .extract => "The only primitive returning a real JS value (object/array). The schema is its only argument. See schema below.", + .waitForSelector => "`waitFor*` default timeout 5000 ms.", + .waitForScript => "Re-evaluates page JS until truthy.", + .waitForState => "", + .press => "Selector first! `page.press(\"Enter\")` binds \"Enter\" to `selector` and fails — use `page.press(null, \"Enter\")` or `page.press({ key: \"Enter\" })`.", + .click, .fill, .scroll, .hover, .selectOption, .setChecked => "", + .search, .markdown, .html, .links, .tree, .nodeDetails, .interactiveElements, .structuredData, .detectForms, .findElement, .consoleLogs, .getUrl, .getCookies, .getEnv => "", + }; +} + +const prose_head = + \\# Writing Lightpanda agent scripts + \\ + \\Run with: + \\ + \\```console + \\./lightpanda agent script.js + \\``` + \\ + \\## Mental model (get this right first) + \\ + \\The script runs in its **own V8 context** — neither the page nor Node.js: + \\ + \\- `Page` is the only global. `new Page()` makes a page and `await page.goto(url)` navigates it; every other primitive is a **method on that page**: `const page = new Page(); await page.goto(url); page.extract({...}); page.click(sel);`. + \\- No `window`, `document`, DOM, `localStorage` — read pages with `page.extract(...)`, run page-side JS only via `page.evaluate("...")`. + \\- No `require`, `process`, `fs`, npm. Standard ECMAScript built-ins only (`JSON`, `Map`, template literals, …). + \\- `page.goto(...)` is **async — always `await` it**. Page methods are **synchronous**: `const data = page.extract({...})`, never `await page.extract(...)`. The script body runs as an async function, so top-level `await` is allowed. + \\- **Re-navigating reuses the same page**: `await page.goto(url2)` keeps `page` valid and points it at the new URL, discarding the old page — read it before navigating away. Independent URLs don't share a page: make a `new Page()` for each and load them in parallel (fan-out, best practice 2). + \\- Page `evaluate("...")` cannot see script variables — interpolate values into the string. Script code cannot see page variables. + \\- Variables persist across navigations within one run, so cross-page aggregation is plain JS. + \\- **`return ` is the script's output**, printed automatically (objects/arrays as JSON). End with `return page.extract({...});` or `return results;`. A bare trailing expression is NOT printed; neither is `console.log(JSON.stringify(...))`. + \\ + \\## Primitives + \\ + \\`Page` is the only global; `new Page()` makes a page and everything else is a method on it. +; + +const table_head = + \\| Call | Notes | + \\|------|-------| + \\| `new Page()` | Makes a page object. No navigation yet — call `page.goto(url)` before any other method. Make several to navigate in parallel (fan-out, best practice 2). | + \\| `page.close()` | Marks the page done; later method calls on it error. The page is otherwise reclaimed at script end. | +; + +const prose_tail = + \\Calling convention: leading positionals + optional trailing options object, or one object with everything (`page.waitForSelector("#row", { timeout: 2000 })` ≡ `page.waitForSelector({ selector: "#row", timeout: 2000 })`). A bare option positional (`page.waitForSelector("#row", 2000)`) and a field passed both ways are `invalid arguments`. `null` skips a positional. Arguments must be JSON-serializable. + \\ + \\CSS selectors only — `backendNodeId`s don't exist here. Standard CSS only: no jQuery `:contains()` or Playwright `:has-text()`. + \\ + \\## extract schema + \\ + \\Keys = output field names; values pick what to lift (not a JSON Schema): + \\ + \\```js + \\const { stories } = page.extract({ + \\ stories: [{ + \\ selector: "tr.athing", // one record per match + \\ limit: 5, + \\ fields: { // resolved relative to each match + \\ title: ".titleline > a", // first match's text (null if missing) + \\ url: { selector: ".titleline > a", attr: "href" }, + \\ text: "" // "" = the matched element's own text + \\ } + \\ }] + \\}); + \\``` + \\ + \\- `"sel"` → first match's text; `["sel"]` → all matches' text; `{ selector, attr }` / `[{ selector, attr }]` → attribute(s); `limit: N` caps any array form. + \\- Every value is a string or null — parse numbers in script logic. + \\- Empty arrays are valid results; if **every** field misses, extract throws ("no schema selector matched any element") → your selectors are wrong, not the page empty. + \\- An object schema always returns an object (destructure it); a bare array schema returns the array directly. + \\- No `save:` option in scripts — keep results in variables. + \\ + \\## Best practices + \\ + \\1. **Navigate, settle, read.** After `await page.goto` on a dynamic page (feeds, search results, comment threads), call `page.waitForState("networkidle")` or `page.waitForSelector(...)` before extracting. Most static pages are complete at `load` — don't wait blindly. + \\2. **List-to-detail — fan out independent pages.** Extract the list, then open one page per item and start every navigation together so the detail pages load in parallel instead of one-after-another: + \\ ```js + \\ const list = new Page(); + \\ await list.goto(listUrl); + \\ const { items } = list.extract({ items: [{ selector: "a.row", fields: { url: { attr: "href" } } }] }); + \\ + \\ const pages = items.map(() => new Page()); + \\ await Promise.all(pages.map((p, i) => p.goto(items[i].url))); // all in flight at once + \\ return pages.map((p, i) => ({ ...items[i], ...p.extract({ /* schema */ }) })); + \\ ``` + \\ - Concurrency is bounded by the HTTP connection pool (40 by default, `--http-max-concurrent`); extra navigations queue rather than fail. For long lists, fan out in batches and `page.close()` each page once read so its memory is reclaimed. + \\ - `Promise.all` rejects the whole batch if any `goto` *fails* (a timeout does not reject); use `Promise.allSettled` when partial results are fine. + \\ - Walk **serially** on one page (`for (const it of items) { await page.goto(it.url); … }`) only when the steps depend on each other — each page decides the next URL, or they share login/session state. + \\3. **`evaluate` is a last resort, not a reading tool.** A `querySelectorAll`-and-parse `page.evaluate` block is always wrong: lift the raw strings with `page.extract`, then trim/split/parse them in top-level JS. Reserve `page.evaluate` for behavior that must run inside the page and no builtin covers — and remember its state dies on every navigation/reload, while script variables persist. + \\4. **Credentials via `$LP_*` placeholders** in any string argument (`page.fill("#pw", "$LP_HN_PASSWORD")`). Never inline a real secret; placeholders resolve inside the Lightpanda process. + \\5. **Unique selectors.** Disambiguate with attributes/position: `input[type="submit"][value="login"]`, not `input[type="submit"]`. + \\6. **Let failures fail.** Primitives throw on error and stop the script — only `try/catch` where you have a real fallback (e.g. optional cookie banner: `try { page.click("#accept") } catch {}`). + \\7. **End with `return `.** `console.log` is for debug output only and doesn't JSON-format objects. + \\8. Modern, readable JS: `const`/`let`, `for (const x of xs)`, template literals, destructuring, 2-space indent. + \\9. **Comment the intent of each block.** Put a one-line `//` comment above each logical step describing what it accomplishes toward the goal (not restating the call). One comment per block, not per line — skip self-evident lines: + \\ ```js + \\ // Load the Hacker News front page + \\ const page = new Page(); + \\ await page.goto("https://news.ycombinator.com"); + \\ + \\ // Pull the top 5 stories (title + link) + \\ const { stories } = page.extract({ stories: [{ selector: "tr.athing", limit: 5, fields: { title: ".titleline > a", url: { selector: ".titleline > a", attr: "href" } } }] }); + \\ + \\ // Open each story page in parallel and read its text + \\ const pages = stories.map(() => new Page()); + \\ await Promise.all(pages.map((p, i) => p.goto(stories[i].url))); + \\ ``` + \\ + \\## Common errors + \\ + \\| Error | Cause / fix | + \\|-------|-------------| + \\| `extract is not defined` (or click/fill/…) | These are methods on the page object, not globals → `const page = new Page(); await page.goto(url); page.extract(...)` | + \\| `Page must be called with new` | `Page(...)` called without `new` → `const page = new Page();` | + \\| `page is not navigated or has been closed` | A method on a fresh `new Page()` (or a closed page) → `await page.goto(url)` first | + \\| `page handle is no longer valid` | Used a page after a later `goto` on the **same** page replaced it → read it before navigating away. Sibling pages from other `new Page()` calls stay valid. | + \\| `document is not defined` | DOM API in script context → use `page.extract` or `page.evaluate` | + \\| `require is not defined` | Not Node.js | + \\| `no page loaded - run page.goto(url) first` | Page method before navigation | + \\| `invalid arguments` | Wrong arity/shape, non-JSON value, or a field set both positionally and in options | + \\| `extract: no schema selector matched any element` | All schema fields missed → fix selectors | + \\| `press` fails with one string arg | Selector-first: use `page.press(null, "Enter")` or `page.press({ key: "Enter" })` | +; + +const testing = @import("../testing.zig"); + +test "skill: every recorded tool is documented, no non-recorded one is" { + const body = text(); + inline for (comptime std.meta.tags(browser_tools.Tool)) |tool| { + const call = "page." ++ @tagName(tool) ++ "("; + const documented = std.mem.indexOf(u8, body, call) != null; + try testing.expect(documented == tool.isRecorded()); + } + try testing.expect(std.mem.indexOf(u8, body, "await page.goto(") != null); +} + +test "skill: golden fragments track the schemas" { + const body = text(); + // waitForState's enum list tracks Config.WaitUntil. + inline for (comptime std.meta.tags(lp.Config.WaitUntil)) |state| { + try testing.expect(std.mem.indexOf(u8, body, "\"" ++ @tagName(state) ++ "\"") != null); + } + try testing.expect(std.mem.indexOf(u8, body, "`checked` defaults to `true`.") != null); + // extract's script form takes the schema as its only argument. + try testing.expect(std.mem.indexOf(u8, body, "page.extract(schema)") != null); + try testing.expect(std.mem.indexOf(u8, body, "page.extract(schema[, ") == null); + try testing.expect(std.mem.indexOf(u8, body, "{ url, timeout, save }") != null); + // backendNodeId has no script form. + try testing.expect(std.mem.indexOf(u8, body, "backendNodeId }") == null); +} + +test "skill: write emits frontmatter" { + var aw: std.Io.Writer.Allocating = .init(testing.allocator); + defer aw.deinit(); + try write(&aw.writer); + try testing.expect(std.mem.startsWith(u8, aw.written(), "---\nname: pandascript\ndescription: ")); + try testing.expect(std.mem.indexOf(u8, aw.written(), "\n---\n\n# Writing Lightpanda agent scripts\n") != null); +}