mirror of
https://github.com/lightpanda-io/browser.git
synced 2026-09-17 08:27:11 -04:00
findElement: match accessible names by regex
`name` is a substring, which cannot say "starts with", "exactly this" or "either of these". A name written as `/.../`, the spelling adblock lists already use, is a JavaScript-syntax pattern compiled through the App's PCRE2 context; an invalid one comes back as a tool error carrying PCRE2's message and offset so the model can fix it rather than retry blind.
This commit is contained in:
1 parent
d2f72a9fa9
commit
158de60d64
4 files changed
+82
-8
No files matched your search
@@ -20,6 +20,7 @@ const std = @import("std");
|
||||
|
||||
const Frame = @import("Frame.zig");
|
||||
const URL = @import("URL.zig");
|
||||
const Regex = @import("../Regex.zig");
|
||||
const TreeWalker = @import("webapi/TreeWalker.zig");
|
||||
const Label = @import("webapi/element/html/Label.zig");
|
||||
const AXNode = @import("../server/cdp/AXNode.zig");
|
||||
@@ -154,6 +155,8 @@ const FindFilter = struct {
|
||||
role: ?[]const u8 = null,
|
||||
/// Accessible-name substring match (case-insensitive). When null, name is not filtered.
|
||||
name: ?[]const u8 = null,
|
||||
/// Compiled pattern searched in the accessible name.
|
||||
name_regex: ?Regex = null,
|
||||
/// Stop walking once this many matches accumulate. When null, walks the full subtree.
|
||||
max: ?usize = null,
|
||||
};
|
||||
@@ -229,6 +232,10 @@ fn walkInteractive(
|
||||
const n = name orelse continue;
|
||||
if (std.ascii.indexOfIgnoreCase(n, nf) == null) continue;
|
||||
}
|
||||
if (filter.name_regex) |re| {
|
||||
const n = name orelse continue;
|
||||
if (!re.matches(n)) continue;
|
||||
}
|
||||
|
||||
const listener_types = getListenerTypes(el.asEventTarget(), listener_targets);
|
||||
|
||||
@@ -490,6 +497,30 @@ fn testInteractiveInBody(html: []const u8) ![]InteractiveElement {
|
||||
return collectInteractiveElements(div.asNode(), frame.call_arena, frame);
|
||||
}
|
||||
|
||||
test "browser.interactive: a name regex filters the walk" {
|
||||
const frame = try testing.createFrame();
|
||||
defer testing.test_session.closeAllPages();
|
||||
const doc = frame.window._document;
|
||||
const div = try doc.createElement("div", null, frame);
|
||||
try Frame.parse.htmlAsChildren(frame, div.asNode(), "<button>Add to cart</button><button>Cart</button><a href=\"#\">Add item</a>");
|
||||
|
||||
const context = testing.test_app.regex_context;
|
||||
const options: Regex.Options = .{ .case_insensitive = true, .unicode = true };
|
||||
|
||||
const starts_add = try Regex.compile(context, "^add", options, null);
|
||||
defer starts_add.deinit();
|
||||
const found_add = try findInteractiveElements(div.asNode(), frame.call_arena, frame, .{ .name_regex = starts_add });
|
||||
try testing.expectEqual(2, found_add.len);
|
||||
try testing.expectEqual("Add to cart", found_add[0].name.?);
|
||||
try testing.expectEqual("Add item", found_add[1].name.?);
|
||||
|
||||
const only_cart = try Regex.compile(context, "^cart$", options, null);
|
||||
defer only_cart.deinit();
|
||||
const found_cart = try findInteractiveElements(div.asNode(), frame.call_arena, frame, .{ .name_regex = only_cart });
|
||||
try testing.expectEqual(1, found_cart.len);
|
||||
try testing.expectEqual("Cart", found_cart[0].name.?);
|
||||
}
|
||||
|
||||
test "browser.interactive: names come from labels, like the tree" {
|
||||
const elements = try testInteractiveInBody(
|
||||
\\<label for="email">Email address</label><input id="email" type="text">
|
||||
|
||||
+29
-7
@@ -50,8 +50,8 @@ pub const driver_guidance =
|
||||
\\ values are already in the tree — don't re-fetch via `nodeDetails`.
|
||||
\\- `nodeDetails(backendNodeId)` → a ready-to-use CSS `selector` that
|
||||
\\ resolves to one node, plus its id/class/attrs.
|
||||
\\- `findElement(role, name)` → locate a candidate by role/name without
|
||||
\\ parsing the whole tree.
|
||||
\\- `findElement(role, name)` → locate a candidate by role and name (a
|
||||
\\ substring, or `/regex/`) without parsing the whole tree.
|
||||
\\- `markdown(selector | backendNodeId)` → readable text for one
|
||||
\\ subtree. Use after `tree` has shown you where the interesting
|
||||
\\ region is.
|
||||
@@ -683,7 +683,7 @@ pub const Tool = enum {
|
||||
\\ "type": "object",
|
||||
\\ "properties": {
|
||||
\\ "role": { "type": "string", "description": "Optional ARIA role to match (e.g. 'button', 'link', 'textbox', 'checkbox')." },
|
||||
\\ "name": { "type": "string", "description": "Optional accessible name substring to match (case-insensitive)." }
|
||||
\\ "name": { "type": "string", "description": "Optional accessible name to match, case-insensitive: a substring, or a JavaScript-syntax regex written as /.../ (unanchored: use ^...$ for the whole name; prefix (?-i) for case-sensitive)." }
|
||||
\\ }
|
||||
\\}
|
||||
),
|
||||
@@ -924,7 +924,7 @@ fn dispatch(
|
||||
.press => .{ .text = try execPress(arena, session, registry, substituted) },
|
||||
.selectOption => .{ .text = try execSelectOption(arena, session, registry, substituted) },
|
||||
.setChecked => .{ .text = try execSetChecked(arena, session, registry, substituted) },
|
||||
.findElement => .{ .text = try execFindElement(arena, session, registry, substituted) },
|
||||
.findElement => execFindElement(arena, session, registry, substituted),
|
||||
.evaluate => execEvaluate(arena, session, registry, substituted),
|
||||
.extract => execExtract(arena, session, registry, substituted),
|
||||
.getEnv => .{ .text = try execGetEnv(arena, substituted) },
|
||||
@@ -2062,7 +2062,7 @@ fn execSetChecked(arena: std.mem.Allocator, session: *lp.Session, registry: *Nod
|
||||
return finalizeAction(arena, session, registry, scope, body);
|
||||
}
|
||||
|
||||
fn execFindElement(arena: std.mem.Allocator, session: *lp.Session, registry: *NodeRegistry, arguments: ?std.json.Value) ToolError![]const u8 {
|
||||
fn execFindElement(arena: std.mem.Allocator, session: *lp.Session, registry: *NodeRegistry, arguments: ?std.json.Value) ToolError!ToolResult {
|
||||
const Params = struct {
|
||||
role: ?[]const u8 = null,
|
||||
name: ?[]const u8 = null,
|
||||
@@ -2073,14 +2073,36 @@ fn execFindElement(arena: std.mem.Allocator, session: *lp.Session, registry: *No
|
||||
|
||||
const page = try requireFrame(session);
|
||||
|
||||
const pattern: ?[]const u8 = if (args.name) |name| regexBody(name) else null;
|
||||
var diag: lp.Regex.Diagnostic = .{};
|
||||
const name_regex: ?lp.Regex = if (pattern) |p|
|
||||
lp.Regex.compile(session.browser.app.regex_context, p, .{ .case_insensitive = true, .unicode = true }, &diag) catch |err| switch (err) {
|
||||
error.OutOfMemory => return error.OutOfMemory,
|
||||
error.InvalidRegex => return .{
|
||||
.text = try std.fmt.allocPrint(arena, "findElement: invalid name regex '{s}': {s} at offset {d}", .{ p, diag.message(), diag.offset }),
|
||||
.is_error = true,
|
||||
},
|
||||
}
|
||||
else
|
||||
null;
|
||||
defer if (name_regex) |re| re.deinit();
|
||||
|
||||
const matched = lp.interactive.findInteractiveElements(page.document.asNode(), arena, page, .{
|
||||
.role = args.role,
|
||||
.name = args.name,
|
||||
.name = if (pattern == null) args.name else null,
|
||||
.name_regex = name_regex,
|
||||
}) catch return ToolError.InternalError;
|
||||
|
||||
lp.interactive.registerNodes(matched, registry) catch
|
||||
return ToolError.InternalError;
|
||||
return renderJson(arena, matched);
|
||||
return .{ .text = try renderJson(arena, matched) };
|
||||
}
|
||||
|
||||
/// The body of a `/.../` literal, the spelling adblock lists use for a regex
|
||||
/// too. Unanchored, so a name that really is written that way still matches.
|
||||
fn regexBody(text: []const u8) ?[]const u8 {
|
||||
if (text.len > 2 and text[0] == '/' and text[text.len - 1] == '/') return text[1 .. text.len - 1];
|
||||
return null;
|
||||
}
|
||||
|
||||
fn execGetEnv(arena: std.mem.Allocator, arguments: ?std.json.Value) ToolError![]const u8 {
|
||||
|
||||
@@ -1411,6 +1411,26 @@ test "MCP - findElement" {
|
||||
try testing.expect(std.mem.indexOf(u8, out.written(), "error") != null);
|
||||
out.clearRetainingCapacity();
|
||||
}
|
||||
|
||||
{
|
||||
const msg =
|
||||
\\{"jsonrpc":"2.0","id":5,"method":"tools/call","params":{"name":"findElement","arguments":{"name":"/^prevent.*default$/"}}}
|
||||
;
|
||||
try router.handleMessage(server, aa, msg);
|
||||
try testing.expect(std.mem.indexOf(u8, out.written(), "Prevent Default") != null);
|
||||
try testing.expect(std.mem.indexOf(u8, out.written(), "Click Me") == null);
|
||||
out.clearRetainingCapacity();
|
||||
}
|
||||
|
||||
{
|
||||
const msg =
|
||||
\\{"jsonrpc":"2.0","id":6,"method":"tools/call","params":{"name":"findElement","arguments":{"name":"/(/"}}}
|
||||
;
|
||||
try router.handleMessage(server, aa, msg);
|
||||
try testing.expect(std.mem.indexOf(u8, out.written(), "\"isError\":true") != null);
|
||||
try testing.expect(std.mem.indexOf(u8, out.written(), "missing closing parenthesis at offset 1") != null);
|
||||
out.clearRetainingCapacity();
|
||||
}
|
||||
}
|
||||
|
||||
test "MCP - waitForSelector: existing element" {
|
||||
|
||||
@@ -203,7 +203,8 @@ fn note(tool: browser_tools.Tool) []const u8 {
|
||||
.screenshot => "`path` is required: writes a PNG of the text layout.",
|
||||
.press => "Selector first! `page.press(\"Enter\")` binds \"Enter\" to `selector` and fails — use `page.press(null, \"Enter\")` or `page.press({ key: \"Enter\" })`.",
|
||||
.click, .fill, .scroll, .hover, .selectOption, .setChecked => "",
|
||||
.search, .markdown, .html, .links, .tree, .nodeDetails, .interactiveElements, .structuredData, .detectForms, .findElement, .consoleLogs, .getUrl, .getCookies, .getEnv => "",
|
||||
.findElement => "`name` is a case-insensitive substring, or a JS-syntax regex written as `/.../`.",
|
||||
.search, .markdown, .html, .links, .tree, .nodeDetails, .interactiveElements, .structuredData, .detectForms, .consoleLogs, .getUrl, .getCookies, .getEnv => "",
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user