WebDriver: add navigate

This feature is significant because it adds support for processing an HTTP
request via the worker. It requires parking the connection and then having the
worker notify the loop when the response is ready. A lot of this was already
in-place (e.g. worker -> loop notification) but not quite do this extent.

src/server/http.zig

@@ -27,6 +27,7 @@ const statusCategory = @import("../network/http.zig").statusCategory;
 const Server = @import("Server.zig");
 const Driver = @import("Driver.zig");
 const bidi_session = @import("bidi/session.zig");
+const http_command = @import("bidi/http_command.zig");
 const uuidv4 = @import("../id.zig").uuidv4;
 
 const log = lp.log;
@@ -59,11 +60,20 @@ pub const Connection = struct {
 
             // lives as long as the server; referenced, never freed
             static: []const u8,
+
+            // built by a worker in a pooled arena; released once written
+            pooled: Pooled,
+        };
+
+        pub const Pooled = struct {
+            arena: *lp.Arena,
+            bytes: []const u8,
         };
 
         pub fn remaining(self: *const Writing) []const u8 {
             return switch (self.data) {
-                inline else => |d| d[self.pos..],
+                .owned, .static => |d| d[self.pos..],
+                .pooled => |p| p.bytes[self.pos..],
             };
         }
 
@@ -71,6 +81,7 @@ pub const Connection = struct {
             switch (self.data) {
                 .static => {},
                 .owned => |owned| allocator.free(owned),
+                .pooled => |p| p.arena.release(),
             }
         }
     };
@@ -400,7 +411,8 @@ fn processHTTP(server: *Server, conn: *Connection, now: u64) !bool {
             },
             .request => |*req| {
                 defer _ = server.request_arena.reset(.{ .retain_with_limit = REQUEST_ARENA_RETAIN });
-                if (try serveHTTP(server, conn, req) == .upgraded) {
+                const served = try serveHTTP(server, conn, req);
+                if (served == .upgraded) {
                     // The fd moved to a WebSocket (and out of server.http); all
                     // that's left of this Connection is to recycle it.
                     // upgradeConnection already took it out of http_connections.
@@ -413,6 +425,11 @@ fn processHTTP(server: *Server, conn: *Connection, now: u64) !bool {
                 http.* = .header;
                 conn.buffer.len = 0;
 
+                if (served == .parked) {
+                    // off the loop until its worker answers (resumeParked)
+                    return true;
+                }
+
                 if (conn.pending != null) {
                     // We got a WouldBlock and now have a pending write. The
                     // connection stays alive until we flush it. After the write
@@ -450,10 +467,18 @@ const empty_json_list_response = staticResponse(.{ .status = "200 OK", .body = "
 const status_response = staticResponse(.{ .status = "200 OK", .body = "{\"value\":{\"ready\":true,\"message\":\"\"}}", .content_type = "application/json; charset=UTF-8" });
 const delete_session_response = staticResponse(.{ .status = "200 OK", .body = "{\"value\":null}", .content_type = "application/json; charset=UTF-8" });
 const protocol_response = staticResponse(.{ .status = "200 OK", .body = @embedFile("../data/protocol.json"), .content_type = "application/json; charset=UTF-8" });
+// A parked command whose worker went away before answering.
+pub const session_ended_response = staticResponse(.{
+    .status = "404 Not Found",
+    .body = "{\"value\":{\"error\":\"invalid session id\",\"message\":\"session ended\",\"stacktrace\":\"\"}}",
+    .content_type = "application/json; charset=UTF-8",
+    .close = true,
+});
 
 const Served = enum {
     responded,
     upgraded,
+    parked,
 };
 
 const Route = struct {
@@ -490,8 +515,8 @@ const session_routes = [_]Route{
     .{ .method = .DELETE, .path = "", .handler = deleteSession },
 };
 
-// Routes under /session/{id}; path is what follows the id ("" for the
-// session itself). The HTTP command surface goes here.
+// /session/{id} itself is session_routes; everything under it is a command
+// (http_command.parse).
 const SESSION_PREFIX = "/session/";
 
 const SESSION_ID_LEN = 36;
@@ -511,7 +536,10 @@ fn serveHTTP(server: *Server, conn: *Connection, req: *Connection.Request) !Serv
             return serveNotFound(server, conn, req);
         }
         req.session_id = path[SESSION_PREFIX.len..][0..SESSION_ID_LEN];
-        return dispatch(server, &session_routes, conn, req, tail);
+        if (tail.len == 0) {
+            return dispatch(server, &session_routes, conn, req, tail);
+        }
+        return serveSessionCommand(server, conn, req, tail);
     }
     return dispatch(server, &routes, conn, req, path);
 }
@@ -588,20 +616,32 @@ fn beginBody(server: *Server) !*std.Io.Writer {
     return &server.scratch.writer;
 }
 
-fn serveDynamicHTTPResponse(server: *Server, conn: *Connection, req: *const Connection.Request, comptime status: []const u8, comptime content_type: []const u8) !Served {
-    const header_format = "HTTP/1.1 " ++ status ++ "\r\n" ++
+fn serveDynamicHTTPResponse(server: *Server, conn: *Connection, req: *const Connection.Request, status: std.http.Status, comptime content_type: []const u8) !Served {
+    return serveHTTPResponse(server, conn, req, .{ .dynamic = fillHeader(server.scratch.written(), status, content_type) });
+}
+
+const JSON_CONTENT_TYPE = "application/json; charset=UTF-8";
+
+// `buf` is HEADER_RESERVE bytes followed by the body; returns the response,
+// its header right-aligned against the body.
+fn fillHeader(buf: []u8, status: std.http.Status, comptime content_type: []const u8) []u8 {
+    const header_format = "HTTP/1.1 {d} {s}\r\n" ++
         "Content-Length: {d}\r\n" ++
         "Content-Type: " ++ content_type ++ "\r\n\r\n";
 
-    // a usize prints as at most 20 digits
-    comptime std.debug.assert(header_format.len + 20 <= HEADER_RESERVE);
+    // 3 status digits, the longest std.http.Status phrase ("Network
+    // Authentication Required"), a usize's 20 digits
+    comptime std.debug.assert(header_format.len + 3 + 31 + 20 <= HEADER_RESERVE);
 
-    const buf = server.scratch.written();
     var header_buf: [HEADER_RESERVE]u8 = undefined;
-    const header = std.fmt.bufPrint(&header_buf, header_format, .{buf.len - HEADER_RESERVE}) catch unreachable;
+    const header = std.fmt.bufPrint(&header_buf, header_format, .{
+        @intFromEnum(status),
+        status.phrase() orelse "",
+        buf.len - HEADER_RESERVE,
+    }) catch unreachable;
     const start = HEADER_RESERVE - header.len;
     @memcpy(buf[start..HEADER_RESERVE], header);
-    return serveHTTPResponse(server, conn, req, .{ .dynamic = buf[start..] });
+    return buf[start..];
 }
 
 fn errorResponse(comptime status: u16, comptime body: []const u8) []const u8 {
@@ -656,7 +696,7 @@ fn serveJSONProtocol(server: *Server, conn: *Connection, req: *Connection.Reques
 fn serveMetrics(server: *Server, conn: *Connection, req: *Connection.Request) !Served {
     const writer = try beginBody(server);
     lp.metrics.write(writer);
-    return serveDynamicHTTPResponse(server, conn, req, "200 OK", "text/plain; version=0.0.4; charset=utf-8");
+    return serveDynamicHTTPResponse(server, conn, req, .ok, "text/plain; version=0.0.4; charset=utf-8");
 }
 
 // GET /status (webdriver)
@@ -678,20 +718,12 @@ fn newSession(server: *Server, conn: *Connection, req: *Connection.Request) !Ser
             firstMatch: ?[]const Capability = null,
         } = null,
     }, req.arena, req.body, .{ .ignore_unknown_fields = true }) catch {
-        return serveWebDriver(server, conn, req, "400 Bad Request", .{
-            .@"error" = "invalid argument",
-            .message = "invalid JSON body",
-            .stacktrace = "",
-        });
+        return serveWebDriverError(server, conn, req, "invalid argument", "invalid JSON body");
     };
 
     if (server.worker_pool.isFull()) {
         lp.metrics.serve_connection_limit.incr();
-        return serveWebDriver(server, conn, req, "500 Internal Server Error", .{
-            .@"error" = "session not created",
-            .message = "too many sessions",
-            .stacktrace = "",
-        });
+        return serveWebDriverError(server, conn, req, "session not created", "too many sessions");
     }
 
     var session_id: [36]u8 = undefined;
@@ -699,11 +731,7 @@ fn newSession(server: *Server, conn: *Connection, req: *Connection.Request) !Ser
 
     const worker = server.spawnWorker(.bidi, .{ .session = session_id }) catch |err| {
         log.err(.serve, "worker spawn", .{ .err = err });
-        return serveWebDriver(server, conn, req, "500 Internal Server Error", .{
-            .@"error" = "session not created",
-            .message = "failed to start the session",
-            .stacktrace = "",
-        });
+        return serveWebDriverError(server, conn, req, "session not created", "failed to start the session");
     };
     // The client never learns the id if we fail to answer (e.g. it hung up),
     // so nothing would ever DELETE this session.
@@ -731,7 +759,7 @@ fn newSession(server: *Server, conn: *Connection, req: *Connection.Request) !Ser
         break :blk null;
     };
 
-    return serveWebDriver(server, conn, req, "200 OK", .{
+    return serveWebDriver(server, conn, req, .ok, .{
         .sessionId = &session_id,
         .capabilities = bidi_session.Capabilities{
             .userAgent = server.app.config.http_headers.user_agent,
@@ -763,16 +791,43 @@ fn upgradeSession(server: *Server, conn: *Connection, req: *Connection.Request)
 // DELETE /session/ID (webdriver)
 fn deleteSession(server: *Server, conn: *Connection, req: *Connection.Request) !Served {
     const worker = server.findSession(req.session_id.?) orelse {
-        return serveWebDriver(server, conn, req, "404 Not Found", .{
-            .@"error" = "invalid session id",
-            .message = "no such session",
-            .stacktrace = "",
-        });
+        return serveNoSuchSession(server, conn, req);
     };
     server.quitSession(worker);
     return serveHTTPResponse(server, conn, req, .{ .static = delete_session_response });
 }
 
+// /session/ID/... (webdriver): parsed here, answered by the session's worker.
+fn serveSessionCommand(server: *Server, conn: *Connection, req: *Connection.Request, path: []const u8) !Served {
+    const worker = server.findSession(req.session_id.?) orelse {
+        return serveNoSuchSession(server, conn, req);
+    };
+    if (worker.http_request != null) {
+        return serveCommandInProgress(server, conn, req);
+    }
+
+    const arena = try server.app.arena_pool.acquire(req.body.len, "http command");
+    const command = http_command.parse(arena.allocator(), req.method, path, req.body) catch |err| {
+        arena.release();
+        switch (err) {
+            error.OutOfMemory => return err,
+            error.UnknownCommand => return serveWebDriverError(server, conn, req, "unknown command", "unknown command"),
+            error.UnknownMethod => return serveWebDriverError(server, conn, req, "unknown method", "unknown method"),
+            error.InvalidArgument => return serveWebDriverError(server, conn, req, "invalid argument", "invalid body"),
+        }
+    };
+    server.parkRequest(worker, conn, req.keepalive, arena, command);
+    return .parked;
+}
+
+fn serveCommandInProgress(server: *Server, conn: *Connection, req: *const Connection.Request) !Served {
+    return serveWebDriverError(server, conn, req, "unknown error", "a command is already in progress");
+}
+
+fn serveNoSuchSession(server: *Server, conn: *Connection, req: *const Connection.Request) !Served {
+    return serveWebDriverError(server, conn, req, "invalid session id", "no such session");
+}
+
 // CDP or Bidi directly creating a Worker from an websocket upgrade
 fn upgradeSpawn(server: *Server, conn: *Connection, req: *Connection.Request, protocol: Driver.Protocol) !Served {
     if (server.worker_pool.isFull()) {
@@ -783,10 +838,18 @@ fn upgradeSpawn(server: *Server, conn: *Connection, req: *Connection.Request, pr
 }
 
 // Answers a HTTP WebDriver request with {"value": value}.
-fn serveWebDriver(server: *Server, conn: *Connection, req: *const Connection.Request, comptime status: []const u8, value: anytype) !Served {
+fn serveWebDriver(server: *Server, conn: *Connection, req: *const Connection.Request, status: std.http.Status, value: anytype) !Served {
     const writer = try beginBody(server);
     try std.json.Stringify.value(.{ .value = value }, .{}, writer);
-    return serveDynamicHTTPResponse(server, conn, req, status, "application/json; charset=UTF-8");
+    return serveDynamicHTTPResponse(server, conn, req, status, JSON_CONTENT_TYPE);
+}
+
+fn serveWebDriverError(server: *Server, conn: *Connection, req: *const Connection.Request, code: []const u8, message: []const u8) !Served {
+    return serveWebDriver(server, conn, req, webDriverErrorStatus(code), .{
+        .@"error" = code,
+        .message = message,
+        .stacktrace = "",
+    });
 }
 
 fn serveNotFound(server: *Server, conn: *Connection, req: *Connection.Request) !Served {
@@ -823,6 +886,81 @@ fn serveHTTPResponse(server: *Server, conn: *Connection, req: *const Connection.
     return .responded;
 }
 
+// A parked connection's response is ready, join the loop so that we can start
+// writing the response.
+pub fn resumeParked(server: *Server, conn: *Connection, req_keepalive: bool, response: Connection.Writing.Data, now: u64) void {
+    const allocator = server.app.allocator;
+    // beginShutdown closed the http connections, this one was off the loop then
+    const keepalive = req_keepalive and server.shutdown_begun == false;
+    var writing: Connection.Writing = .{ .pos = 0, .data = response, .keepalive = keepalive };
+
+    server.io_engine.monitorHTTP(conn) catch |err| {
+        log.err(.serve, "resume monitor", .{ .err = err });
+        writing.deinit(allocator);
+        sys_net.close(conn.socket);
+        return recycle(server, conn);
+    };
+    conn.deadline = now + IDLE_TIMEOUT_MS;
+    server.http_connections.append(&conn.node);
+
+    const data = writing.remaining();
+    recordResponse(data);
+    writing.pos = write(conn.socket, data) catch |err| {
+        log.debug(.serve, "resume write", .{ .err = err });
+        writing.deinit(allocator);
+        return disconnect(server, conn);
+    };
+
+    if (writing.pos < data.len) {
+        // disconnect frees it from here
+        conn.pending = writing;
+        server.io_engine.waitWritable(conn) catch |err| {
+            log.err(.serve, "wait writable", .{ .err = err });
+            return disconnect(server, conn);
+        };
+        return;
+    }
+
+    writing.deinit(allocator);
+    if (keepalive == false) {
+        disconnect(server, conn);
+    }
+}
+
+// A complete {"value": value} response, built by a worker for resumeParked,
+// in an arena the loop releases once it's written.
+pub fn webDriverResponse(arena: *lp.Arena, status: std.http.Status, value: anytype) ![]const u8 {
+    var aw: std.Io.Writer.Allocating = try .initCapacity(arena.allocator(), 512);
+    try aw.writer.splatByteAll(0, HEADER_RESERVE);
+    try std.json.Stringify.value(.{ .value = value }, .{}, &aw.writer);
+    return fillHeader(aw.written(), status, JSON_CONTENT_TYPE);
+}
+
+// W3C WebDriver's error table; every code not listed is a 500.
+pub fn webDriverErrorStatus(code: []const u8) std.http.Status {
+    const statuses = std.StaticStringMap(std.http.Status).initComptime(.{
+        .{ "detached shadow root", .not_found },
+        .{ "element click intercepted", .bad_request },
+        .{ "element not interactable", .bad_request },
+        .{ "insecure certificate", .bad_request },
+        .{ "invalid argument", .bad_request },
+        .{ "invalid cookie domain", .bad_request },
+        .{ "invalid element state", .bad_request },
+        .{ "invalid selector", .bad_request },
+        .{ "invalid session id", .not_found },
+        .{ "no such alert", .not_found },
+        .{ "no such cookie", .not_found },
+        .{ "no such element", .not_found },
+        .{ "no such frame", .not_found },
+        .{ "no such shadow root", .not_found },
+        .{ "no such window", .not_found },
+        .{ "stale element reference", .not_found },
+        .{ "unknown command", .not_found },
+        .{ "unknown method", .method_not_allowed },
+    });
+    return statuses.get(code) orelse .internal_server_error;
+}
+
 // HTTP-phase teardown. Websockets tear down via releaseWorker.
 pub fn disconnect(server: *Server, conn: *Connection) void {
     server.io_engine.remove(conn.socket);