agent: verify synthesized scripts during /save

Introduces a multi-step synthesis process for `/save` that derives a
logical JSON output schema and uses a dry-run runtime to verify
candidate scripts. The LLM can now run and self-correct its scripts
using a new `run_script` tool before finalizing the save.
This commit is contained in:
Adrià Arrufat committed 2026-06-09 16:53:39 +02:00
1 parent b23c6c27bc
commit b141da30ca
6 files changed
+574 -147

No files matched your search

+130 -29
View File
@@ -95,6 +95,13 @@ pub const RunError = error{
OutOfMemory,
};
/// A captured script run: `err` is the formatted failure (null on success);
/// `output` is the completion value's display string (empty when void).
pub const RunOutcome = struct {
err: ?[]const u8 = null,
output: []const u8 = "",
};
pub fn init(
allocator: std.mem.Allocator,
app: *lp.App,
@@ -238,6 +245,23 @@ fn setObjectProperty(
/// compile/runtime exception returns a formatted error allocated in this
/// runtime's call arena and valid until deinit or the next run.
pub fn runSource(self: *Runtime, source: []const u8, name: []const u8) RunError!?[]const u8 {
return (try self.runInner(source, name, false)).err;
}
/// Like `runSource`, but capture the completion value's display string instead
/// of printing it (used by `/save` verification to feed output back to the LLM).
/// Both fields live in the call arena — valid until the next run or deinit.
///
/// Each call gets a fresh global context so a prior candidate's top-level
/// `const`/`let` doesn't collide ("Identifier 'x' has already been declared")
/// — verification candidates are independent runs, unlike a `/load` script.
pub fn runSourceCapture(self: *Runtime, source: []const u8, name: []const u8) RunError!RunOutcome {
self.resetContext();
self.createContext() catch return .{ .err = try self.dupeError("script context reset failed") };
return self.runInner(source, name, true);
}
fn runInner(self: *Runtime, source: []const u8, name: []const u8, capture: bool) RunError!RunOutcome {
_ = self.call_arena.reset(.retain_capacity);
var hs: lp.js.HandleScope = undefined;
@@ -245,7 +269,7 @@ pub fn runSource(self: *Runtime, source: []const u8, name: []const u8) RunError!
defer hs.deinit();
const context: *const v8.Context = @ptrCast(v8.v8__Global__Get(&self.context, self.env.isolate.handle) orelse
return try self.dupeError("agent script context is not available"));
return .{ .err = try self.dupeError("agent script context is not available") });
v8.v8__Context__Enter(context);
defer v8.v8__Context__Exit(context);
@@ -268,19 +292,27 @@ pub fn runSource(self: *Runtime, source: []const u8, name: []const u8) RunError!
&compiler_source,
v8.kNoCompileOptions,
v8.kNoCacheNoReason,
) orelse return try self.formatCaught(context, &try_catch, "compile failed");
) orelse return .{ .err = try self.formatCaught(context, &try_catch, "compile failed") };
const completion = v8.v8__Script__Run(script, context) orelse
return try self.formatCaught(context, &try_catch, "script failed");
return .{ .err = try self.formatCaught(context, &try_catch, "script failed") };
// Explicit microtask policy: promise continuations only run once drained.
self.env.performIsolateMicrotasks();
if (v8.v8__TryCatch__HasCaught(&try_catch)) {
return try self.formatCaught(context, &try_catch, "script failed");
return .{ .err = try self.formatCaught(context, &try_catch, "script failed") };
}
if (capture) {
if (v8.v8__Value__IsUndefined(completion)) return .{};
const output = self.displayString(self.call_arena.allocator(), context, completion) catch |err| switch (err) {
error.OutOfMemory => return error.OutOfMemory,
error.JsException => return .{ .output = "<completion value could not be serialized>" },
};
return .{ .output = output };
}
self.printCompletion(context, completion);
return null;
return .{};
}
/// Echo a script's completion value (its last-evaluated expression) so a script
@@ -677,10 +709,10 @@ test "agent script runtime: goto and evaluate dispatch through browser tools" {
defer testing.reset();
defer if (testing.test_session.hasPage()) testing.test_session.removePage();
var registry = CDPNode.Registry.init(testing.allocator);
var registry: CDPNode.Registry = .init(testing.allocator);
defer registry.deinit();
const runtime = try Runtime.init(testing.allocator, testing.test_app, testing.test_session, &registry);
const runtime: *Runtime = try .init(testing.allocator, testing.test_app, testing.test_session, &registry);
defer runtime.deinit();
try runTestScript(runtime,
@@ -698,10 +730,10 @@ test "agent script runtime: extract returns a JavaScript object" {
defer testing.reset();
defer if (testing.test_session.hasPage()) testing.test_session.removePage();
var registry = CDPNode.Registry.init(testing.allocator);
var registry: CDPNode.Registry = .init(testing.allocator);
defer registry.deinit();
const runtime = try Runtime.init(testing.allocator, testing.test_app, testing.test_session, &registry);
const runtime: *Runtime = try .init(testing.allocator, testing.test_app, testing.test_session, &registry);
defer runtime.deinit();
try runTestScript(runtime,
@@ -752,10 +784,10 @@ test "agent script runtime: extract tolerates list selectors that match nothing"
defer testing.reset();
defer if (testing.test_session.hasPage()) testing.test_session.removePage();
var registry = CDPNode.Registry.init(testing.allocator);
var registry: CDPNode.Registry = .init(testing.allocator);
defer registry.deinit();
const runtime = try Runtime.init(testing.allocator, testing.test_app, testing.test_session, &registry);
const runtime: *Runtime = try .init(testing.allocator, testing.test_app, testing.test_session, &registry);
defer runtime.deinit();
try runTestScript(runtime,
@@ -780,10 +812,10 @@ test "agent script runtime: strict-mode scripts can call primitives" {
defer testing.reset();
defer if (testing.test_session.hasPage()) testing.test_session.removePage();
var registry = CDPNode.Registry.init(testing.allocator);
var registry: CDPNode.Registry = .init(testing.allocator);
defer registry.deinit();
const runtime = try Runtime.init(testing.allocator, testing.test_app, testing.test_session, &registry);
const runtime: *Runtime = try .init(testing.allocator, testing.test_app, testing.test_session, &registry);
defer runtime.deinit();
try runTestScript(runtime,
@@ -798,10 +830,10 @@ test "agent script runtime: strict-mode scripts can call primitives" {
test "agent script runtime: promise microtasks run to completion" {
defer testing.reset();
var registry = CDPNode.Registry.init(testing.allocator);
var registry: CDPNode.Registry = .init(testing.allocator);
defer registry.deinit();
const runtime = try Runtime.init(testing.allocator, testing.test_app, testing.test_session, &registry);
const runtime: *Runtime = try .init(testing.allocator, testing.test_app, testing.test_session, &registry);
defer runtime.deinit();
try runTestScript(runtime,
@@ -819,10 +851,10 @@ test "agent script runtime: primitives re-entered from argument callbacks stay i
defer testing.reset();
defer if (testing.test_session.hasPage()) testing.test_session.removePage();
var registry = CDPNode.Registry.init(testing.allocator);
var registry: CDPNode.Registry = .init(testing.allocator);
defer registry.deinit();
const runtime = try Runtime.init(testing.allocator, testing.test_app, testing.test_session, &registry);
const runtime: *Runtime = try .init(testing.allocator, testing.test_app, testing.test_session, &registry);
defer runtime.deinit();
try runTestScript(runtime,
@@ -841,10 +873,10 @@ test "agent script runtime: terminate interrupts local JavaScript" {
defer testing.reset();
defer if (testing.test_session.hasPage()) testing.test_session.removePage();
var registry = CDPNode.Registry.init(testing.allocator);
var registry: CDPNode.Registry = .init(testing.allocator);
defer registry.deinit();
const runtime = try Runtime.init(testing.allocator, testing.test_app, testing.test_session, &registry);
const runtime: *Runtime = try .init(testing.allocator, testing.test_app, testing.test_session, &registry);
defer runtime.deinit();
const thread = try std.Thread.spawn(.{}, terminateRuntimeSoon, .{runtime});
@@ -859,10 +891,10 @@ test "agent script runtime: agent variables persist and page globals are isolate
defer testing.reset();
defer if (testing.test_session.hasPage()) testing.test_session.removePage();
var registry = CDPNode.Registry.init(testing.allocator);
var registry: CDPNode.Registry = .init(testing.allocator);
defer registry.deinit();
const runtime = try Runtime.init(testing.allocator, testing.test_app, testing.test_session, &registry);
const runtime: *Runtime = try .init(testing.allocator, testing.test_app, testing.test_session, &registry);
defer runtime.deinit();
try runTestScript(runtime,
@@ -884,10 +916,10 @@ test "agent script runtime: page evaluate cannot see agent primitives or binding
defer testing.reset();
defer if (testing.test_session.hasPage()) testing.test_session.removePage();
var registry = CDPNode.Registry.init(testing.allocator);
var registry: CDPNode.Registry = .init(testing.allocator);
defer registry.deinit();
const runtime = try Runtime.init(testing.allocator, testing.test_app, testing.test_session, &registry);
const runtime: *Runtime = try .init(testing.allocator, testing.test_app, testing.test_session, &registry);
defer runtime.deinit();
try runTestScript(runtime,
@@ -902,10 +934,10 @@ test "agent script runtime: page evaluate cannot see agent primitives or binding
test "agent script runtime: console is available in agent context" {
defer testing.reset();
var registry = CDPNode.Registry.init(testing.allocator);
var registry: CDPNode.Registry = .init(testing.allocator);
defer registry.deinit();
const runtime = try Runtime.init(testing.allocator, testing.test_app, testing.test_session, &registry);
const runtime: *Runtime = try .init(testing.allocator, testing.test_app, testing.test_session, &registry);
defer runtime.deinit();
try runTestScript(runtime,
@@ -919,10 +951,10 @@ test "agent script runtime: tool errors throw and stop execution" {
defer testing.reset();
defer if (testing.test_session.hasPage()) testing.test_session.removePage();
var registry = CDPNode.Registry.init(testing.allocator);
var registry: CDPNode.Registry = .init(testing.allocator);
defer registry.deinit();
const runtime = try Runtime.init(testing.allocator, testing.test_app, testing.test_session, &registry);
const runtime: *Runtime = try .init(testing.allocator, testing.test_app, testing.test_session, &registry);
defer runtime.deinit();
const message = (try runtime.runSource(
@@ -945,10 +977,10 @@ test "agent script runtime: builtin argument marshalling (positional + options)"
defer testing.reset();
defer if (testing.test_session.hasPage()) testing.test_session.removePage();
var registry = CDPNode.Registry.init(testing.allocator);
var registry: CDPNode.Registry = .init(testing.allocator);
defer registry.deinit();
const runtime = try Runtime.init(testing.allocator, testing.test_app, testing.test_session, &registry);
const runtime: *Runtime = try .init(testing.allocator, testing.test_app, testing.test_session, &registry);
defer runtime.deinit();
try runTestScript(runtime,
@@ -994,3 +1026,72 @@ test "agent script runtime: builtin argument marshalling (positional + options)"
try testing.expect(std.mem.indexOf(u8, message, "invalid arguments") != null);
}
}
test "agent script runtime: runSourceCapture runs the full script live and captures completion" {
defer testing.reset();
defer if (testing.test_session.hasPage()) testing.test_session.removePage();
var registry: CDPNode.Registry = .init(testing.allocator);
defer registry.deinit();
const runtime: *Runtime = try .init(testing.allocator, testing.test_app, testing.test_session, &registry);
defer runtime.deinit();
// Verification runs the candidate exactly as a standalone replay does — it
// must navigate itself; extract then runs against the page it loaded.
const outcome = try runtime.runSourceCapture(
\\goto("http://localhost:9582/src/browser/tests/mcp_actions.html");
\\click("#btn");
\\const data = extract({ label: "#btn" });
\\data.label;
, "candidate.js");
try testing.expect(outcome.err == null);
try std.testing.expectEqualStrings("Click Me", outcome.output);
}
test "agent script runtime: runSourceCapture surfaces a candidate's error" {
defer testing.reset();
defer if (testing.test_session.hasPage()) testing.test_session.removePage();
var registry: CDPNode.Registry = .init(testing.allocator);
defer registry.deinit();
const runtime: *Runtime = try .init(testing.allocator, testing.test_app, testing.test_session, &registry);
defer runtime.deinit();
// extract returns an object; treating it as an array throws — the real
// `raw.map is not a function` failure the model must see and fix.
const bad = try runtime.runSourceCapture(
\\goto("http://localhost:9582/src/browser/tests/mcp_actions.html");
\\const raw = extract({ items: [{ selector: "li" }] });
\\raw.map(x => x);
, "candidate.js");
try testing.expect(bad.err != null);
}
test "agent script runtime: each capture run gets a fresh global scope" {
defer testing.reset();
defer if (testing.test_session.hasPage()) testing.test_session.removePage();
var registry: CDPNode.Registry = .init(testing.allocator);
defer registry.deinit();
const runtime: *Runtime = try .init(testing.allocator, testing.test_app, testing.test_session, &registry);
defer runtime.deinit();
// The same top-level `const` in two consecutive candidates must not collide —
// each run starts from a clean context, not the previous run's globals.
const src =
\\goto("http://localhost:9582/src/browser/tests/mcp_actions.html");
\\const data = extract({ label: "#btn" });
\\const out = data.label;
\\out;
;
const first = try runtime.runSourceCapture(src, "candidate.js");
try testing.expect(first.err == null);
try std.testing.expectEqualStrings("Click Me", first.output);
const second = try runtime.runSourceCapture(src, "candidate.js");
try testing.expect(second.err == null);
try std.testing.expectEqualStrings("Click Me", second.output);
}