From e58f99da18a6f3961c2d329d02f2ddecbe2637d0 Mon Sep 17 00:00:00 2001 From: Halil Durak Date: Wed, 19 Aug 2026 16:07:03 +0300 Subject: [PATCH] add `--load-resources` CLI arg, supporting `image` param --- src/Config.zig | 33 +++++ src/browser/Frame.zig | 149 ++++++++++++++++++++++ src/browser/Session.zig | 11 ++ src/browser/webapi/element/html/Image.zig | 75 +++++++++-- src/cdp/domains/lp.zig | 3 + src/cdp/domains/network.zig | 1 + src/network/HttpClient.zig | 102 ++++++++++++++- 7 files changed, 353 insertions(+), 21 deletions(-) diff --git a/src/Config.zig b/src/Config.zig index e2dcd8813..b986066b7 100644 --- a/src/Config.zig +++ b/src/Config.zig @@ -211,6 +211,26 @@ fn caPathValidator( } } +// Sub-resources the browser actually requests. A comma-separated list so +// more can be opted into individually as they land; only `image` fetches +// today. +pub const LoadResources = packed struct(u1) { + image: bool = false, +}; + +fn loadResourcesValidator(_: Allocator, args: *std.process.Args.Iterator, target: *LoadResources) !void { + const resources = args.next() orelse return error.MissingArgument; + + var it = std.mem.splitScalar(u8, resources, ','); + while (it.next()) |resource| { + inline for (@typeInfo(LoadResources).@"struct".fields) |field| { + if (std.mem.eql(u8, field.name, resource)) { + @field(target, field.name) = true; + } + } + } +} + /// Common CLI args. const CommonOptions = .{ .{ .name = "obey_robots", .type = bool }, @@ -245,6 +265,12 @@ const CommonOptions = .{ .{ .name = "disable_subframes", .type = bool }, .{ .name = "disable_workers", .type = bool }, .{ .name = "enable_external_stylesheets", .type = bool }, + .{ + .name = "load_resources", + .type = LoadResources, + .default = LoadResources{}, + .validator = loadResourcesValidator, + }, .{ .name = "v8_flags_unsafe", .type = ?[]const u8 }, .{ .name = "v8_max_heap_mb", .type = ?u32 }, .{ .name = "watchdog_ms", .type = ?u32 }, @@ -526,6 +552,13 @@ pub fn enableExternalStylesheets(self: *const Config) bool { }; } +pub fn loadResources(self: *const Config) LoadResources { + return switch (self.mode) { + inline .serve, .fetch, .mcp, .agent => |opts| opts.load_resources, + else => unreachable, + }; +} + pub fn v8Flags(self: *const Config) ?[]const u8 { return switch (self.mode) { inline .serve, .fetch, .mcp, .agent => |opts| opts.v8_flags_unsafe, diff --git a/src/browser/Frame.zig b/src/browser/Frame.zig index afd14ae2a..66ed65457 100644 --- a/src/browser/Frame.zig +++ b/src/browser/Frame.zig @@ -2384,6 +2384,155 @@ pub fn loadExternalStylesheet(self: *Frame, link: *Element.Html.Link, href: []co try self.fireElementEvent(element, comptime .wrap("load")); } +// Asynchronous, unlike `loadExternalStylesheet`: nothing in the document +// depends on the result, so there's no reason to block the parser on it. +// The request does take a `_pending_loads` slot, so the window load event +// waits for it the way the HTML spec says it should. +pub fn loadImage(self: *Frame, image: *Element.Html.Image, src: []const u8) !void { + const session = self._session; + // Fragment-parsed images (innerHTML, DOMParser, ...) may never be + // attached, and may belong to another Document. Same call as + // `loadExternalStylesheet` makes. They still get the synthetic load the + // no-fetch path would have given them, so the two modes agree. + if (self._parse_mode == .fragment) { + return self.queueLoad(Factory.protoOf(image)); + } + + const arena = try session.getArena(.small, "Frame.loadImage"); + defer arena.release(); + + const resolved = URL.resolve(arena.allocator(), self.base(), src, .{ .encoding = self.charset }) catch |err| { + // An unresolvable src is a load failure the same way a 404 is, and + // unlike a real fetch we can settle it without a request. No pending + // load was taken out yet, so queue rather than decrementing. + log.info(.http, "image resolve", .{ .err = err, .src = src }); + image._complete = true; + return self.queueElementEvent(Factory.protoOf(image), .@"error"); + }; + + // an in-flight image is a fixed-size record we can hand back the moment it + // settles, and a page can create a lot of them. + const load = try self._factory.create(ImageLoad{ + .frame = self, + .image = image, + .generation = image._generation, + }); + errdefer self._factory.destroy(load); + + // New load event always sets `_complete` back to false. + image._complete = false; + self._pending_loads += 1; + errdefer { + image._complete = true; + self._pending_loads -= 1; + } + + const transfer = try session.browser.http_client.newRequest(.{ + .ctx = load, + .url = resolved, + .method = .GET, + .frame_id = self._frame_id, + .loader_id = self._loader_id, + .cookie_jar = &session.cookie_jar, + .cookie_origin = self.url, + .resource_type = .image, + .notification = session.notification, + .headers_only = true, + .header_callback = ImageLoad.headerCallback, + .data_callback = ImageLoad.dataCallback, + .done_callback = ImageLoad.doneCallback, + .error_callback = ImageLoad.errorCallback, + .shutdown_callback = ImageLoad.shutdownCallback, + }, &self._http_owner); + { + // `deinit` fires no callbacks, so the errdefers above are still the + // ones responsible for undoing our state on this path. + errdefer transfer.deinit(); + // Part of mimicking the real request: origins that content-negotiate + // (or that turn away clients which don't look like browsers) key off + // exactly this header. + try transfer.addHeader("Accept", "image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8", .{}); + try self.headersForRequest(transfer); + } + + // From here the transfer owns `load`. `submit` either succeeds or has + // already routed the failure through error_callback, which settles the + // ImageLoad and gives the pending-load slot back. + transfer.submit() catch |err| { + log.warn(.http, "image fetch", .{ .err = err, .url = resolved }); + }; +} + +// One in-flight image fetch. Exactly one of done/error/shutdown runs, and +// each releases the `_pending_loads` slot taken in `loadImage`. +const ImageLoad = struct { + frame: *Frame, + image: *Element.Html.Image, + generation: u32, + status: u16 = 0, + + fn headerCallback(transfer: *HttpClient.Transfer) !HttpClient.Transfer.HeaderResult { + const self: *ImageLoad = @ptrCast(@alignCast(transfer.req.ctx)); + self.status = transfer.responseStatus() orelse 0; + return .proceed; + } + + fn dataCallback(_: *HttpClient.Transfer, _: []const u8) !void { + // Nothing should ever reach here. + unreachable; + } + + fn doneCallback(ctx: *anyopaque) !void { + const self: *ImageLoad = @ptrCast(@alignCast(ctx)); + const ok = self.status >= 200 and self.status < 300; + self.settle(if (ok) .load else .@"error"); + } + + fn errorCallback(ctx: *anyopaque, err: anyerror) void { + const self: *ImageLoad = @ptrCast(@alignCast(ctx)); + log.info(.http, "image fetch", .{ .err = err, .status = self.status }); + self.settle(.@"error"); + } + + fn shutdownCallback(ctx: *anyopaque) void { + const self: *ImageLoad = @ptrCast(@alignCast(ctx)); + const frame = self.frame; + self.image._complete = true; + frame._factory.destroy(self); + + // Teardown or a superseding navigation. Release the slot directly: + // `pendingLoadCompleted` could reach `documentIsComplete`, and + // running the load event on a frame that's being dismantled is + // exactly what we're being told to stop doing. + frame._pending_loads -|= 1; + } + + fn settle(self: *ImageLoad, kind: QueuedEvent.Kind) void { + const frame = self.frame; + const image = self.image; + defer frame._factory.destroy(self); + + // A later src assignment owns the element's events now; this + // response is only still here to give its pending-load slot back. + const superseded = self.generation != image._generation; + if (!superseded) { + image._complete = true; + } + + if (!superseded and !frame.isGoingAway()) { + const name: String = switch (kind) { + .load => comptime .wrap("load"), + .@"error" => comptime .wrap("error"), + }; + frame.fireElementEvent(image.asElement(), name) catch |err| { + log.warn(.js, "image event", .{ .err = err, .kind = kind }); + }; + } + + frame.pendingLoadCompleted(); + } +}; + fn fireElementEvent(self: *Frame, el: *Element, name: String) !void { const event = try Event.initTrusted(name, .{}, self._page); try self._event_manager.dispatch(el.asEventTarget(), event); diff --git a/src/browser/Session.zig b/src/browser/Session.zig index 09233a32b..811612cf6 100644 --- a/src/browser/Session.zig +++ b/src/browser/Session.zig @@ -20,6 +20,7 @@ const std = @import("std"); const lp = @import("lightpanda"); const App = @import("../App.zig"); +const Config = @import("../Config.zig"); const History = @import("webapi/History.zig"); const storage = @import("webapi/storage/storage.zig"); @@ -115,6 +116,15 @@ _console_capture: bool = false, // (synchronous fetch + parse + register on `document.styleSheets`). load_external_stylesheets: bool = false, +// Sub-resources to actually request. Off by default: a driver that only +// reads the DOM shouldn't pay for bytes it never looks at. Set from the +// `--load-resources` CLI flag at session init; the LP.configureLoading CDP +// method can flip it per-session. When `image` is set, +// `Image.imageAddedCallback` routes to `Frame.loadImage` and an 's +// load/error event reflects the real HTTP status instead of always being +// a synthetic load. +load_resources: Config.LoadResources = .{}, + /// Caller-supplied cancellation probe. `Runner._wait` polls it between /// ticks; once `check` returns true the wait returns `error.Cancelled`. /// The agent installs this so SIGINT can abort an in-flight tool call @@ -180,6 +190,7 @@ pub fn init(self: *Session, browser: *Browser, notification: *Notification) !voi .worker_loading_enabled = !browser.app.config.disableWorkers(), ._console_messages = .init(allocator), .load_external_stylesheets = browser.app.config.enableExternalStylesheets(), + .load_resources = browser.app.config.loadResources(), }; errdefer self._console_messages.deinit(); } diff --git a/src/browser/webapi/element/html/Image.zig b/src/browser/webapi/element/html/Image.zig index 5d08a42e1..4538353a7 100644 --- a/src/browser/webapi/element/html/Image.zig +++ b/src/browser/webapi/element/html/Image.zig @@ -1,5 +1,6 @@ const lp = @import("lightpanda"); const std = @import("std"); +const log = lp.log; const js = @import("../../../js/js.zig"); const Factory = @import("../../../Factory.zig"); const Frame = @import("../../../Frame.zig"); @@ -7,10 +8,20 @@ const Node = @import("../../Node.zig"); const Element = @import("../../Element.zig"); const HtmlElement = @import("../Html.zig"); +const String = lp.String; + const Image = @This(); pub const Proto = HtmlElement; -_pad: bool = false, + +// Bumped on every src assignment. An in-flight fetch carries the value it +// was issued under; when they no longer match, a newer src has taken over +// and the older response must not fire events on this element. +_generation: u32 = 0, +// Per spec, false only while a fetch is in flight. Without +// `--load-resources image` there is never a fetch, so it never leaves true. +_complete: bool = true, + _proto_canary: if (lp.IS_DEBUG) *HtmlElement else void = undefined, pub fn constructor(w_: ?u32, h_: ?u32, frame: *Frame) !*Image { @@ -48,10 +59,12 @@ pub fn getSrc(self: *const Image, frame: *Frame) ![]const u8 { } pub fn setSrc(self: *Image, value: []const u8, frame: *Frame) !void { - const element = self.asElement(); - try element.setAttributeSafe(comptime .wrap("src"), .wrap(value), frame); - // No need to check if `Image` is connected to DOM; this is a special case. - return self.imageAddedCallback(frame); + // Setting the attribute is enough: `_put` dispatches to + // `Build.attributeChange`, which starts the load. Calling + // `imageAddedCallback` here too would issue the request twice (the first + // one immediately superseded, so it costs a request and shows nothing). + // Connectivity still isn't checked — a detached `new Image()` loads. + return self.asElement().setAttributeSafe(comptime .wrap("src"), .wrap(value), frame); } pub fn getLoading(self: *const Image) []const u8 { @@ -74,17 +87,21 @@ pub fn getNaturalHeight(_: *const Image) u32 { return 0; } -pub fn getComplete(_: *const Image) bool { - // Per spec, complete is true when: no src/srcset, src is empty, - // image is fully available, or image is broken (with no pending request). - // Since we never fetch images, they are in the "broken" state, which has - // complete=true. This is consistent with naturalWidth/naturalHeight=0. - return true; +pub fn getComplete(self: *const Image) bool { + // Per spec, complete is true when: no src/srcset, src is empty, the + // image is fully available, or the image is broken with no pending + // request. Every one of those is "no fetch in flight", which is exactly + // what `_complete` tracks. Without `--load-resources image` nothing ever + // clears it. + return self._complete; } -/// Used in `Page.nodeIsReady`. +/// The one funnel for "this element's src became current": parser-created +/// images (`Build.created`), `img.src = ...` (`setSrc`) and +/// `setAttribute("src", ...)` (`Build.attributeChange`) all land here. pub fn imageAddedCallback(self: *Image, frame: *Frame) !void { - // if we're planning on navigating to another frame, don't trigger load event. + // if we're planning on navigating to another frame, don't trigger a load event + // or start fetching a resource. if (frame.isGoingAway()) { return; } @@ -94,7 +111,28 @@ pub fn imageAddedCallback(self: *Image, frame: *Frame) !void { const src = element.getAttributeSafe(comptime .wrap("src")) orelse return; if (src.len == 0) return; - try frame.queueLoad(Factory.protoOf(self)); + // If image loading not desired, we just do fake "load" event. + if (frame._session.load_resources.image == false) { + return frame.queueLoad(Factory.protoOf(self)); + } + + // A fetch still in flight for the previous src is stale as of right now, + // and this is the only place that knows it. Wrapping is fine: colliding + // needs a request to still be in flight 2^32 src assignments later, which + // outlives any transfer timeout by several orders of magnitude. + self._generation +%= 1; + + // Deliberately not propagated. `newRequest` is declared `anyerror`, and + // this runs on DOM mutation paths with narrow declared error sets + // (`Node.cloneNode`'s `CloneError`, for one). Failing to even issue the + // request is the same observable outcome as the request failing, so + // report it the same way — `loadImage` has already unwound `_complete` + // and the pending-load slot by the time it returns an error. + frame.loadImage(self, src) catch |err| { + log.warn(.http, "image fetch", .{ .err = err, .src = src }); + // On failure, queue "error" event. + return frame.queueElementEvent(Factory.protoOf(self), .@"error"); + }; } pub const JsApi = struct { @@ -140,6 +178,15 @@ pub const Build = struct { const self = node.as(Image); return self.imageAddedCallback(frame); } + + // `img.src = ...` routes through `setSrc`, but `setAttribute("src", ...)` + // only reaches us here, and both have to (re)do fetch. + pub fn attributeChange(element: *Element, name: String, _: String, frame: *Frame) !void { + if (!name.eql(comptime .wrap("src"))) { + return; + } + return element.as(Image).imageAddedCallback(frame); + } }; const testing = @import("../../../../testing.zig"); diff --git a/src/cdp/domains/lp.zig b/src/cdp/domains/lp.zig index a331714f7..659d5ef98 100644 --- a/src/cdp/domains/lp.zig +++ b/src/cdp/domains/lp.zig @@ -20,6 +20,7 @@ const std = @import("std"); const lp = @import("lightpanda"); const CDP = @import("../CDP.zig"); +const Config = @import("../../Config.zig"); const Node = @import("../Node.zig"); const DOMNode = @import("../../browser/webapi/Node.zig"); @@ -98,12 +99,14 @@ fn configureLoading(cmd: *CDP.Command) !void { subFrame: ?bool = null, worker: ?bool = null, externalStylesheets: ?bool = null, + images: ?bool = null, })) orelse return error.InvalidParams; const bc = cmd.browser_context orelse return error.NoBrowserContext; if (params.subFrame) |v| bc.session.subframe_loading_enabled = v; if (params.worker) |v| bc.session.worker_loading_enabled = v; if (params.externalStylesheets) |v| bc.session.load_external_stylesheets = v; + if (params.images) |v| bc.session.load_resources.image = v; return cmd.sendResult(null, .{}); } diff --git a/src/cdp/domains/network.zig b/src/cdp/domains/network.zig index c5fd99de7..df83a30a0 100644 --- a/src/cdp/domains/network.zig +++ b/src/cdp/domains/network.zig @@ -650,6 +650,7 @@ fn initialPriority(resource_type: HttpClient.Request.ResourceType) []const u8 { return switch (resource_type) { .document, .stylesheet => "VeryHigh", .script, .xhr, .fetch, .eventsource => "High", + .image => "Low", }; } diff --git a/src/network/HttpClient.zig b/src/network/HttpClient.zig index cf7bdaf53..d1151d1a8 100644 --- a/src/network/HttpClient.zig +++ b/src/network/HttpClient.zig @@ -539,7 +539,7 @@ pub fn activity(self: *const Client) Activity { .http = self.http_active + self.dispatch_count + self.intercepted + self.delayed_count, .ws_events = self.ws_dispatch_count, .ws_conns = self.ws_active, - .pending = self.pending_queue.first != null, + .pending = self.pending_queue.first != null or self.pending_low_queue.first != null, }; } @@ -831,13 +831,24 @@ fn isGated(self: *const Client, transfer: *const Transfer) bool { return transfer.id != blocking_id; } +// Resources the page's progress doesn't depend on. Images are fetched for +// their status (so load/error is honest) and, with `--fetch-images headers`, +// a page can queue dozens of them in one parse — none of which should come +// ahead of the script that's blocking the parser. +fn isLowPriority(resource_type: Request.ResourceType) bool { + return switch (resource_type) { + .image => false, // EXPERIMENT + .document, .xhr, .script, .fetch, .stylesheet, .eventsource => false, + }; +} + fn startPending(self: *Client) !void { try self.startDelayed(); while (self.pending_queue.popFirst()) |queue_node| { const transfer: *Transfer = @fieldParentPtr("_node", queue_node); const conn = self.network.getConnection() orelse { - self.pending_queue.prepend(queue_node); - return; + queue.prepend(queue_node); + return false; }; // Bridge state to .created so a failure inside makeRequest before // any commit cleans up via the failAsync below. makeRequest flips to @@ -853,6 +864,7 @@ fn startPending(self: *Client) !void { return err; }; } + return true; } // Enter the pipeline for every delayed transfer whose time has come. @@ -1107,6 +1119,13 @@ fn cacheStore(self: *Client, transfer: *Transfer) void { // entry a second time on any early return below. transfer._cache_intent = .none; + // A headers_only transfer never read the body. Storing it would put an + // empty entry under the real cache key and every later full fetch of + // that URL (an XHR, a `full` image fetch) would hit it and get nothing. + if (transfer.req.headers_only) { + return; + } + // could have been disabled while waiting of the response const cache = self.cache.active() orelse return; @@ -1285,11 +1304,19 @@ pub fn syncRequest(self: *Client, transfer: *Transfer) !SyncResponse { } fn processTransfer(self: *Client, transfer: *Transfer) !void { - if (self.network.getConnection()) |conn| { - return self.makeRequest(conn, transfer); + const low = isLowPriority(transfer.req.resource_type); + if (!low or self.pending_queue.first == null) { + if (self.network.getConnection()) |conn| { + return self.makeRequest(conn, transfer); + } } - self.pending_queue.append(&transfer._node); + transfer._queued_low = low; + if (low) { + self.pending_low_queue.append(&transfer._node); + } else { + self.pending_queue.append(&transfer._node); + } transfer.state = .queued; } @@ -1509,6 +1536,11 @@ fn processOneMessage(self: *Client, msg: http.Handles.MultiMessage, transfer: *T log.debug(.http, "WriteError downgraded", .{ .url = transfer.req.url, .bytes = transfer.res.bytes_received }); break :blk null; } + // Our own headers_only abort, not a failure: fall through so the + // response is materialized and delivered with an empty body. + if (err == error.WriteError and transfer.res.headers_only_abort) { + break :blk null; + } break :blk err; } else null; @@ -1751,6 +1783,7 @@ pub const Request = struct { fetch, stylesheet, eventsource, + image, // Allowed Values: Document, Stylesheet, Image, Media, Font, Script, // TextTrack, XHR, Fetch, Prefetch, EventSource, WebSocket, Manifest, @@ -1764,6 +1797,7 @@ pub const Request = struct { .fetch => "Fetch", .stylesheet => "Stylesheet", .eventsource => "EventSource", + .image => "Image", }; } }; @@ -1772,6 +1806,12 @@ pub const Request = struct { // internal requests transparently following redirects. pub const RedirectMode = enum { follow, manual, @"error" }; + // Largest body a headers_only transfer will read to the end rather than + // abort. Draining costs bandwidth but keeps the connection poolable; + // aborting saves bandwidth but forces a reconnect. 16 KiB is the rough + // break-even: about ten segments, versus a TCP handshake plus a TLS one. + pub const HEADERS_ONLY_DRAIN_MAX: usize = 16 * 1024; + frame_id: u32, loader_id: u32, method: Method, @@ -1787,6 +1827,16 @@ pub const Request = struct { timeout_ms: u32 = 0, skip_cache: bool = false, + // Tear the transfer off the wire as soon as the first body byte arrives: + // the caller wants the status and the response headers, not the body. + // Unlike a HEAD, the request itself is byte-for-byte a normal GET, so + // origins and CDNs see (and answer) exactly what a real browser sends. + // The consumer still gets the usual start/header/done sequence with an + // empty body; `data_callback` never fires. Note the cost: aborting + // mid-response means the connection can't be drained, so libcurl closes + // it instead of returning it to the pool. + headers_only: bool = false, + // The document frame this request belongs to, for CDP attribution. // This will be different than frame_id for Workers. document_frame_id: ?u32 = null, @@ -2080,6 +2130,11 @@ pub const Transfer = struct { // node is always free by then). _node: std.DoublyLinkedList.Node = .{}, + // Which of the two pending queues _node is linked into. Only meaningful + // while state == .queued; set at enqueue, read when unlinking, because + // removing from the wrong list would corrupt both. + _queued_low: bool = false, + // Buffered response ordered events awaiting dispatch. _events: std.ArrayList(Event) = .empty, @@ -2643,7 +2698,11 @@ pub const Transfer = struct { } } - if (opts.check_content_length) { + // headers_only is exempt: the cap exists to bound how much body we + // buffer, and this transfer buffers none of it. Failing a 4 MB image + // we were never going to read would turn the size limit into a + // spurious `error` event on a perfectly good response. + if (opts.check_content_length and !self.req.headers_only) { if (self.getContentLength()) |cl| { if (cl > self.client.max_response_size) { return error.ResponseTooLarge; @@ -3056,6 +3115,29 @@ pub const Transfer = struct { return @intCast(chunk_len); } + if (transfer.req.headers_only) { + const drainable = if (transfer.getContentLength()) |cl| + cl <= Request.HEADERS_ONLY_DRAIN_MAX + else + // No Content-Length (chunked): we can't tell how much is + // coming, so don't gamble on it being small. + false; + + if (drainable) { + // Reuses the redirect machinery: consumed, never buffered, + // so the response still completes with an empty body. + res.skip_body = true; + return @intCast(chunk_len); + } + + // Returning writefunc_error is the only way to end a transfer + // early from a write callback; processOneMessage recognises + // the flag and treats the resulting CURLE_WRITE_ERROR as a + // completed response with an empty body. + res.headers_only_abort = true; + return http.writefunc_error; + } + // Pre-size buffer from Content-Length. if (transfer.getContentLength()) |cl| { if (cl > transfer.client.max_response_size) { @@ -3338,6 +3420,12 @@ const Response = struct { skip_body: bool = false, first_data_received: bool = false, + // Set when dataCallback deliberately killed the transfer to satisfy + // `Request.headers_only`. processOneMessage uses it to tell our own + // abort apart from a real CURLE_WRITE_ERROR and deliver the response + // (headers, status, empty body) as a success. + headers_only_abort: bool = false, + // Response body. Filled by dataCallback, consumed in processMessages. // See Stream.spare to see how this works in streaming mode buffer: std.ArrayList(u8) = .empty,