mirror/browser
1
0
Fork 0
mirror of https://github.com/lightpanda-io/browser.git synced 2026-06-11 01:25:53 -04:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #2079 from lightpanda-io/http-client-layering

Browse Source 
Layering HTTP Client
This commit is contained in:
Pierre Tachoire
2026-04-28 16:23:42 +02:00
committed by GitHub
parent 86217dd78f 1057b9de8d
commit 427cdfe3ce
18 changed files with 1524 additions and 869 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
src/Notification.zig
View File

@@ -21,6 +21,9 @@ const lp = @import("lightpanda");
const Frame = @import("browser/Frame.zig");
const Transfer = @import("browser/HttpClient.zig").Transfer;
const Request = @import("browser/HttpClient.zig").Request;
const Response = @import("browser/HttpClient.zig").Response;
const InterceptContext = @import("network/layer/InterceptionLayer.zig").InterceptContext;
const log = lp.log;
const List = std.DoublyLinkedList;
@@ -162,11 +165,11 @@ pub const FrameLoaded = struct {
};
pub const RequestStart = struct {
transfer: *Transfer,
request: *Request,
};
pub const RequestIntercept = struct {
transfer: *Transfer,
request: *Request,
wait_for_interception: *bool,
};
@@ -177,19 +180,21 @@ pub const RequestAuthRequired = struct {
pub const ResponseData = struct {
data: []const u8,
transfer: *Transfer,
request: *Request,
};
pub const ResponseHeaderDone = struct {
transfer: *Transfer,
request: *Request,
response: *const Response,
};
pub const RequestDone = struct {
transfer: *Transfer,
request: *Request,
content_length: usize,
};
pub const RequestFail = struct {
transfer: *Transfer,
request: *Request,
err: anyerror,
};

22
src/browser/Frame.zig
View File

@@ -647,16 +647,18 @@ pub fn navigate(self: *Frame, request_url: [:0]const u8, opts: NavigateOpts) !vo
http_client.request(.{
.ctx = self,
.url = self.url,
.frame_id = self._frame_id,
.loader_id = self._loader_id,
.method = opts.method,
.headers = headers,
.body = opts.body,
.cookie_jar = &session.cookie_jar,
.cookie_origin = self.url,
.resource_type = .document,
.notification = self._session.notification,
.params = .{
.url = self.url,
.frame_id = self._frame_id,
.loader_id = self._loader_id,
.method = opts.method,
.headers = headers,
.body = opts.body,
.cookie_jar = &session.cookie_jar,
.cookie_origin = self.url,
.resource_type = .document,
.notification = self._session.notification,
},
.header_callback = frameHeaderDoneCallback,
.data_callback = frameDataCallback,
.done_callback = frameDoneCallback,

937
src/browser/HttpClient.zig
View File

File diff suppressed because it is too large Load Diff

4
src/browser/Runner.zig
View File

@@ -185,7 +185,7 @@ fn _tick(self: *Runner, comptime is_cdp: bool, opts: TickOpts) !CDPTickResult {
try frame.dispatchLoad();
const http_active = http_client.http_active;
const total_network_activity = http_active + http_client.intercepted;
const total_network_activity = http_active + http_client.interception_layer.intercepted;
if (frame._notified_network_almost_idle.check(total_network_activity <= 2)) {
frame.notifyNetworkAlmostIdle();
}
@@ -211,7 +211,7 @@ fn _tick(self: *Runner, comptime is_cdp: bool, opts: TickOpts) !CDPTickResult {
// because is_cdp is false, and that can only be
// the case when interception isn't possible.
if (comptime IS_DEBUG) {
std.debug.assert(http_client.intercepted == 0);
std.debug.assert(http_client.interception_layer.intercepted == 0);
}
if (browser.hasBackgroundTasks()) {

128
src/browser/ScriptManager.zig
View File

@@ -265,13 +265,6 @@ pub fn addFromElement(self: *ScriptManager, comptime from_parser: bool, script_e
}
if (remote_url) |url| {
errdefer {
if (is_blocking == false) {
self.scriptList(script).remove(&script.node);
}
// Let the outer errdefer handle releasing the arena if client.request fails
}
if (comptime IS_DEBUG) {
var ls: js.Local.Scope = undefined;
frame.js.localScope(&ls);
@@ -285,23 +278,48 @@ pub fn addFromElement(self: *ScriptManager, comptime from_parser: bool, script_e
});
}
{
const was_evaluating = self.is_evaluating;
self.is_evaluating = true;
defer self.is_evaluating = was_evaluating;
const was_evaluating = self.is_evaluating;
self.is_evaluating = true;
defer self.is_evaluating = was_evaluating;
try self.client.request(.{
const headers = try self.getHeaders();
errdefer headers.deinit();
if (is_blocking) {
const response = try self.client.syncRequest(arena, .{
.url = url,
.ctx = script,
.method = .GET,
.frame_id = frame._frame_id,
.loader_id = frame._loader_id,
.headers = try self.getHeaders(),
.blocking = is_blocking,
.headers = headers,
.cookie_jar = &frame._session.cookie_jar,
.cookie_origin = frame.url,
.resource_type = .script,
.notification = frame._session.notification,
});
script.source = .{ .remote = response.body };
script.status = response.status;
script.complete = true;
} else {
errdefer {
self.scriptList(script).remove(&script.node);
// Let the outer errdefer handle releasing the arena if client.request fails
}
try self.client.request(.{
.ctx = script,
.params = .{
.url = url,
.method = .GET,
.frame_id = frame._frame_id,
.loader_id = frame._loader_id,
.headers = headers,
.cookie_jar = &frame._session.cookie_jar,
.cookie_origin = frame.url,
.resource_type = .script,
.notification = frame._session.notification,
},
.start_callback = if (log.enabled(.http, .debug)) Script.startCallback else null,
.header_callback = Script.headerCallback,
.data_callback = Script.dataCallback,
@@ -317,29 +335,21 @@ pub fn addFromElement(self: *ScriptManager, comptime from_parser: bool, script_e
return;
}
// this is <script src="..."></script>, it needs to block the caller
// until it's evaluated
var client = self.client;
while (true) {
if (!script.complete) {
_ = try client.tick(200);
continue;
}
if (script.status == 0) {
// an error (that we already logged)
script.deinit();
return;
}
// could have already been evaluating if this is dynamically added
const was_evaluating = self.is_evaluating;
self.is_evaluating = true;
defer {
self.is_evaluating = was_evaluating;
script.deinit();
}
return script.eval(frame);
if (script.status == 0) {
// an error (that we already logged)
script.deinit();
return;
}
// could have already been evaluating if this is dynamically added
const was_evaluating = self.is_evaluating;
self.is_evaluating = true;
defer {
self.is_evaluating = was_evaluating;
script.deinit();
}
script.eval(frame);
}
fn scriptList(self: *ScriptManager, script: *const Script) *std.DoublyLinkedList {
@@ -407,16 +417,18 @@ pub fn preloadImport(self: *ScriptManager, url: [:0]const u8, referrer: []const
self.async_scripts.append(&script.node);
self.client.request(.{
.url = url,
.ctx = script,
.method = .GET,
.frame_id = frame._frame_id,
.loader_id = frame._loader_id,
.headers = try self.getHeaders(),
.cookie_jar = &frame._session.cookie_jar,
.cookie_origin = frame.url,
.resource_type = .script,
.notification = frame._session.notification,
.params = .{
.url = url,
.method = .GET,
.frame_id = frame._frame_id,
.loader_id = frame._loader_id,
.headers = try self.getHeaders(),
.cookie_jar = &frame._session.cookie_jar,
.cookie_origin = frame.url,
.resource_type = .script,
.notification = frame._session.notification,
},
.start_callback = if (log.enabled(.http, .debug)) Script.startCallback else null,
.header_callback = Script.headerCallback,
.data_callback = Script.dataCallback,
@@ -513,16 +525,18 @@ pub fn getAsyncImport(self: *ScriptManager, url: [:0]const u8, cb: ImportAsync.C
self.async_scripts.append(&script.node);
self.client.request(.{
.url = url,
.method = .GET,
.frame_id = frame._frame_id,
.loader_id = frame._loader_id,
.headers = try self.getHeaders(),
.ctx = script,
.resource_type = .script,
.cookie_jar = &frame._session.cookie_jar,
.cookie_origin = frame.url,
.notification = frame._session.notification,
.params = .{
.url = url,
.method = .GET,
.frame_id = frame._frame_id,
.loader_id = frame._loader_id,
.headers = try self.getHeaders(),
.resource_type = .script,
.cookie_jar = &frame._session.cookie_jar,
.cookie_origin = frame.url,
.notification = frame._session.notification,
},
.start_callback = if (log.enabled(.http, .debug)) Script.startCallback else null,
.header_callback = Script.headerCallback,
.data_callback = Script.dataCallback,
@@ -664,7 +678,6 @@ pub const Script = struct {
debug_transfer_aborted: bool = false,
debug_transfer_bytes_received: usize = 0,
debug_transfer_notified_fail: bool = false,
debug_transfer_intercept_state: u8 = 0,
debug_transfer_auth_challenge: bool = false,
debug_transfer_easy_id: usize = 0,
@@ -740,7 +753,6 @@ pub const Script = struct {
.a3 = self.debug_transfer_aborted,
.a4 = self.debug_transfer_bytes_received,
.a5 = self.debug_transfer_notified_fail,
.a7 = self.debug_transfer_intercept_state,
.a8 = self.debug_transfer_auth_challenge,
.a9 = self.debug_transfer_easy_id,
.b1 = transfer.id,
@@ -748,7 +760,6 @@ pub const Script = struct {
.b3 = transfer.aborted,
.b4 = transfer.bytes_received,
.b5 = transfer._notified_fail,
.b7 = @intFromEnum(transfer._intercept_state),
.b8 = transfer._auth_challenge != null,
.b9 = if (transfer._conn) |c| @intFromPtr(c._easy) else 0,
});
@@ -758,7 +769,6 @@ pub const Script = struct {
self.debug_transfer_aborted = transfer.aborted;
self.debug_transfer_bytes_received = transfer.bytes_received;
self.debug_transfer_notified_fail = transfer._notified_fail;
self.debug_transfer_intercept_state = @intFromEnum(transfer._intercept_state);
self.debug_transfer_auth_challenge = transfer._auth_challenge != null;
self.debug_transfer_easy_id = if (transfer._conn) |c| @intFromPtr(c._easy) else 0;
},

20
src/browser/webapi/Worker.zig
View File

@@ -95,15 +95,17 @@ pub fn init(url: []const u8, exec: *Execution) !*Worker {
const http_client = session.browser.http_client;
http_client.request(.{
.ctx = self,
.url = resolved_url,
.method = .GET,
.headers = try http_client.newHeaders(),
.frame_id = self._frame_id,
.loader_id = self._loader_id,
.resource_type = .script,
.cookie_jar = &session.cookie_jar,
.cookie_origin = resolved_url,
.notification = session.notification,
.params = .{
.url = resolved_url,
.method = .GET,
.headers = try http_client.newHeaders(),
.frame_id = self._frame_id,
.loader_id = self._loader_id,
.resource_type = .script,
.cookie_jar = &session.cookie_jar,
.cookie_origin = resolved_url,
.notification = session.notification,
},
.header_callback = httpHeaderCallback,
.data_callback = httpDataCallback,
.done_callback = httpDoneCallback,

22
src/browser/webapi/net/Fetch.zig
View File

@@ -94,16 +94,18 @@ pub fn init(input: Input, options: ?InitOpts, frame: *Frame) !js.Promise {
try http_client.request(.{
.ctx = fetch,
.url = request._url,
.method = request._method,
.frame_id = frame._frame_id,
.loader_id = frame._loader_id,
.body = request._body,
.headers = headers,
.resource_type = .fetch,
.cookie_jar = cookie_jar,
.cookie_origin = frame.url,
.notification = frame._session.notification,
.params = .{
.url = request._url,
.method = request._method,
.frame_id = frame._frame_id,
.loader_id = frame._loader_id,
.body = request._body,
.headers = headers,
.resource_type = .fetch,
.cookie_jar = cookie_jar,
.cookie_origin = frame.url,
.notification = frame._session.notification,
},
.start_callback = httpStartCallback,
.header_callback = httpHeaderDoneCallback,
.data_callback = httpDataCallback,

24
src/browser/webapi/net/XMLHttpRequest.zig
View File

@@ -257,17 +257,19 @@ pub fn send(self: *XMLHttpRequest, body_: ?[]const u8) !void {
http_client.request(.{
.ctx = self,
.url = self._url,
.method = self._method,
.headers = headers,
.frame_id = frame._frame_id,
.loader_id = frame._loader_id,
.body = self._request_body,
.cookie_jar = if (cookie_support) &frame._session.cookie_jar else null,
.cookie_origin = frame.url,
.resource_type = .xhr,
.timeout_ms = self._timeout,
.notification = frame._session.notification,
.params = .{
.url = self._url,
.method = self._method,
.headers = headers,
.frame_id = frame._frame_id,
.loader_id = frame._loader_id,
.body = self._request_body,
.cookie_jar = if (cookie_support) &frame._session.cookie_jar else null,
.cookie_origin = frame.url,
.resource_type = .xhr,
.timeout_ms = self._timeout,
.notification = frame._session.notification,
},
.start_callback = httpStartCallback,
.header_callback = httpHeaderDoneCallback,
.data_callback = httpDataCallback,

29
src/cdp/CDP.zig
View File

@@ -450,8 +450,25 @@ pub const BrowserContext = struct {
// abort all intercepted requests before closing the session/page
// since some of these might callback into the page/scriptmanager
for (self.intercept_state.pendingTransfers()) |transfer| {
transfer.abort(error.ClientDisconnect);
const http_client = browser.http_client;
for (self.intercept_state.pendingIntercepts()) |intercept| {
defer {
lp.assert(
http_client.interception_layer.intercepted > 0,
"BrowserContext.deinit.intercepted",
.{ .value = http_client.interception_layer.intercepted },
);
http_client.interception_layer.intercepted -= 1;
}
switch (intercept) {
.transfer => |t| {
t.abort(error.ClientDisconnect);
},
.request => |r| {
defer http_client.deinitRequest(r);
r.error_callback(r.ctx, error.ClientDisconnect);
},
}
}
for (self.isolated_worlds.items) |world| {
@@ -668,7 +685,7 @@ pub const BrowserContext = struct {
const arena = self.frame_arena;
// Prepare the captured response value.
const id = msg.transfer.id;
const id = msg.request.params.request_id;
const gop = try self.captured_responses.getOrPut(arena, id);
if (!gop.found_existing) {
gop.value_ptr.* = .{
@@ -676,8 +693,8 @@ pub const BrowserContext = struct {
// Encode the data in base64 by default, but don't encode
// for well known content-type.
.must_encode = blk: {
const transfer = msg.transfer;
if (transfer.response_header.?.contentType()) |ct| {
const response = msg.response;
if (response.contentType()) |ct| {
const mime = try Mime.parse(ct);
if (!mime.isText()) {
@@ -705,7 +722,7 @@ pub const BrowserContext = struct {
const self: *BrowserContext = @ptrCast(@alignCast(ctx));
const arena = self.frame_arena;
const id = msg.transfer.id;
const id = msg.request.params.request_id;
const resp = self.captured_responses.getPtr(id) orelse lp.assert(false, "onHttpResponseData missinf captured response", .{});
return resp.data.appendSlice(arena, msg.data);

138
src/cdp/domains/fetch.zig
View File

@@ -54,7 +54,12 @@ pub fn processMessage(cmd: *CDP.Command) !void {
// Stored in CDP
pub const InterceptState = struct {
allocator: Allocator,
waiting: std.AutoArrayHashMapUnmanaged(u32, *HttpClient.Transfer),
waiting: std.AutoArrayHashMapUnmanaged(u32, Pending),
const Pending = union(enum) {
transfer: *HttpClient.Transfer,
request: HttpClient.Request,
};
pub fn init(allocator: Allocator) !InterceptState {
return .{
@@ -67,11 +72,15 @@ pub const InterceptState = struct {
return self.waiting.count() == 0;
}
pub fn put(self: *InterceptState, transfer: *HttpClient.Transfer) !void {
return self.waiting.put(self.allocator, transfer.id, transfer);
pub fn putRequest(self: *InterceptState, request: HttpClient.Request) !void {
return self.waiting.put(self.allocator, request.params.request_id, .{ .request = request });
}
pub fn remove(self: *InterceptState, request_id: u32) ?*HttpClient.Transfer {
pub fn putTransfer(self: *InterceptState, transfer: *HttpClient.Transfer) !void {
return self.waiting.put(self.allocator, transfer.id, .{ .transfer = transfer });
}
pub fn remove(self: *InterceptState, request_id: u32) ?Pending {
const entry = self.waiting.fetchSwapRemove(request_id) orelse return null;
return entry.value;
}
@@ -80,7 +89,7 @@ pub const InterceptState = struct {
self.waiting.deinit(self.allocator);
}
pub fn pendingTransfers(self: *const InterceptState) []*HttpClient.Transfer {
pub fn pendingIntercepts(self: *const InterceptState) []Pending {
return self.waiting.values();
}
};
@@ -190,29 +199,28 @@ pub fn requestIntercept(bc: *CDP.BrowserContext, intercept: *const Notification.
const session_id = bc.session_id orelse return;
// We keep it around to wait for modifications to the request.
// NOTE: we assume whomever created the request created it with a lifetime of the Page.
// TODO: What to do when receiving replies for a previous frame's requests?
const transfer = intercept.transfer;
try bc.intercept_state.put(transfer);
const request = intercept.request;
try bc.intercept_state.putRequest(request.*);
try bc.cdp.sendEvent("Fetch.requestPaused", .{
.requestId = &id.toInterceptId(transfer.id),
.frameId = &id.toFrameId(transfer.req.frame_id),
.request = network.TransferAsRequestWriter.init(transfer),
.resourceType = switch (transfer.req.resource_type) {
.requestId = &id.toInterceptId(request.params.request_id),
.frameId = &id.toFrameId(request.params.frame_id),
.request = network.RequestWriter.init(request),
.resourceType = switch (request.params.resource_type) {
.script => "Script",
.xhr => "XHR",
.document => "Document",
.fetch => "Fetch",
},
.networkId = &id.toRequestId(transfer), // matches the Network REQ-ID
.networkId = &id.toRequestId(request), // matches the Network REQ-ID
}, .{ .session_id = session_id });
log.debug(.cdp, "request intercept", .{
.state = "paused",
.id = transfer.id,
.url = transfer.url,
.id = request.params.request_id,
.url = request.params.url,
});
// Await either continueRequest, failRequest or fulfillRequest
@@ -236,39 +244,50 @@ fn continueRequest(cmd: *CDP.Command) !void {
var intercept_state = &bc.intercept_state;
const request_id = try idFromRequestId(params.requestId);
const transfer = intercept_state.remove(request_id) orelse return error.RequestNotFound;
const pending = intercept_state.remove(request_id) orelse return error.RequestNotFound;
var request = pending.request;
log.debug(.cdp, "request intercept", .{
.state = "continue",
.id = transfer.id,
.url = transfer.url,
.id = request.params.request_id,
.url = request.params.url,
.new_url = params.url,
});
const arena = transfer.arena.allocator();
const arena = request.params.arena;
// Update the request with the new parameters
if (params.url) |url| {
try transfer.updateURL(try arena.dupeZ(u8, url));
request.params.url = try arena.dupeZ(u8, url);
}
if (params.method) |method| {
transfer.req.method = std.meta.stringToEnum(http.Method, method) orelse return error.InvalidParams;
request.params.method = std.meta.stringToEnum(http.Method, method) orelse return error.InvalidParams;
}
if (params.headers) |headers| {
// Not obvious, but cmd.arena is safe here, since the headers will get
// duped by libcurl. transfer.arena is more obvious/safe, but cmd.arena
// is more efficient (it's re-used)
try transfer.replaceRequestHeaders(cmd.arena, headers);
request.params.headers.deinit();
var buf: std.ArrayList(u8) = .empty;
var new_headers = try bc.cdp.browser.http_client.newHeaders();
for (headers) |hdr| {
defer buf.clearRetainingCapacity();
try std.fmt.format(buf.writer(cmd.arena), "{s}: {s}", .{ hdr.name, hdr.value });
try buf.append(cmd.arena, 0);
try new_headers.add(buf.items[0 .. buf.items.len - 1 :0]);
}
request.params.headers = new_headers;
}
if (params.postData) |b| {
const decoder = std.base64.standard.Decoder;
const body = try arena.alloc(u8, try decoder.calcSizeForSlice(b));
try decoder.decode(body, b);
transfer.req.body = body;
request.params.body = body;
}
try bc.cdp.browser.http_client.continueTransfer(transfer);
// todo: replace.
const client = bc.cdp.browser.http_client;
try client.interception_layer.continueRequest(client, request);
return cmd.sendResult(null, .{});
}
@@ -292,14 +311,18 @@ fn continueWithAuth(cmd: *CDP.Command) !void {
var intercept_state = &bc.intercept_state;
const request_id = try idFromRequestId(params.requestId);
const transfer = intercept_state.remove(request_id) orelse return error.RequestNotFound;
const pending = intercept_state.remove(request_id) orelse return error.RequestNotFound;
const transfer = pending.transfer;
const request = transfer.req;
log.debug(.cdp, "request intercept", .{
.state = "continue with auth",
.id = transfer.id,
.id = request.params.request_id,
.response = params.authChallengeResponse.response,
});
const client = bc.cdp.browser.http_client;
if (params.authChallengeResponse.response != .ProvideCredentials) {
transfer.abortAuthChallenge();
return cmd.sendResult(null, .{});
@@ -308,17 +331,18 @@ fn continueWithAuth(cmd: *CDP.Command) !void {
// cancel the request, deinit the transfer on error.
errdefer transfer.abortAuthChallenge();
// restart the request with the provided credentials.
const arena = transfer.arena.allocator();
transfer.updateCredentials(
try std.fmt.allocPrintSentinel(arena, "{s}:{s}", .{
const arena = request.params.arena;
transfer.updateCredentials(try std.fmt.allocPrintSentinel(
arena,
"{s}:{s}",
.{
params.authChallengeResponse.username,
params.authChallengeResponse.password,
}, 0),
);
},
0,
));
transfer.reset();
try bc.cdp.browser.http_client.continueTransfer(transfer);
try client.continueTransfer(transfer);
return cmd.sendResult(null, .{});
}
@@ -341,12 +365,14 @@ fn fulfillRequest(cmd: *CDP.Command) !void {
var intercept_state = &bc.intercept_state;
const request_id = try idFromRequestId(params.requestId);
const transfer = intercept_state.remove(request_id) orelse return error.RequestNotFound;
const pending = intercept_state.remove(request_id) orelse return error.RequestNotFound;
var request = pending.request;
log.debug(.cdp, "request intercept", .{
.state = "fulfilled",
.id = transfer.id,
.url = transfer.url,
.id = request.params.request_id,
.url = request.params.url,
.status = params.responseCode,
.body = params.body != null,
});
@@ -354,13 +380,13 @@ fn fulfillRequest(cmd: *CDP.Command) !void {
var body: ?[]const u8 = null;
if (params.body) |b| {
const decoder = std.base64.standard.Decoder;
const buf = try transfer.arena.allocator().alloc(u8, try decoder.calcSizeForSlice(b));
const buf = try request.params.arena.alloc(u8, try decoder.calcSizeForSlice(b));
try decoder.decode(buf, b);
body = buf;
}
try bc.cdp.browser.http_client.fulfillTransfer(transfer, params.responseCode, params.responseHeaders orelse &.{}, body);
const client = bc.cdp.browser.http_client;
try client.interception_layer.fulfillRequest(client, request, params.responseCode, params.responseHeaders orelse &.{}, body);
return cmd.sendResult(null, .{});
}
@@ -374,13 +400,16 @@ fn failRequest(cmd: *CDP.Command) !void {
var intercept_state = &bc.intercept_state;
const request_id = try idFromRequestId(params.requestId);
const transfer = intercept_state.remove(request_id) orelse return error.RequestNotFound;
defer bc.cdp.browser.http_client.abortTransfer(transfer);
const pending = intercept_state.remove(request_id) orelse return error.RequestNotFound;
const request = pending.request;
const client = bc.cdp.browser.http_client;
defer client.interception_layer.abortRequest(client, request);
log.info(.cdp, "request intercept", .{
.state = "fail",
.id = request_id,
.url = transfer.url,
.url = request.params.url,
.reason = params.errorReason,
});
return cmd.sendResult(null, .{});
@@ -396,15 +425,16 @@ pub fn requestAuthRequired(bc: *CDP.BrowserContext, intercept: *const Notificati
// TODO: What to do when receiving replies for a previous frame's requests?
const transfer = intercept.transfer;
try bc.intercept_state.put(transfer);
try bc.intercept_state.putTransfer(transfer);
var request = transfer.req;
const challenge = transfer._auth_challenge orelse return error.NullAuthChallenge;
try bc.cdp.sendEvent("Fetch.authRequired", .{
.requestId = &id.toInterceptId(transfer.id),
.frameId = &id.toFrameId(transfer.req.frame_id),
.request = network.TransferAsRequestWriter.init(transfer),
.resourceType = switch (transfer.req.resource_type) {
.requestId = &id.toInterceptId(request.params.request_id),
.frameId = &id.toFrameId(request.params.frame_id),
.request = network.RequestWriter.init(&request),
.resourceType = switch (request.params.resource_type) {
.script => "Script",
.xhr => "XHR",
.document => "Document",
@@ -416,13 +446,13 @@ pub fn requestAuthRequired(bc: *CDP.BrowserContext, intercept: *const Notificati
.scheme = if (challenge.scheme) |s| (if (s == .digest) "digest" else "basic") else "",
.realm = challenge.realm orelse "",
},
.networkId = &id.toRequestId(transfer),
.networkId = &id.toRequestId(&request),
}, .{ .session_id = session_id });
log.debug(.cdp, "request auth required", .{
.state = "paused",
.id = transfer.id,
.url = transfer.url,
.id = request.params.request_id,
.url = request.params.url,
});
// Await continueWithAuth

91
src/cdp/domains/network.zig
View File

@@ -27,6 +27,8 @@ const Mime = @import("../../browser/Mime.zig");
const Notification = @import("../../Notification.zig");
const timestamp = @import("../../datetime.zig").timestamp;
const Transfer = @import("../../browser/HttpClient.zig").Transfer;
const Request = @import("../../browser/HttpClient.zig").Request;
const Response = @import("../../browser/HttpClient.zig").Response;
const CdpStorage = @import("storage.zig");
@@ -260,7 +262,7 @@ pub fn httpRequestFail(bc: *CDP.BrowserContext, msg: *const Notification.Request
// We're missing a bunch of fields, but, for now, this seems like enough
try bc.cdp.sendEvent("Network.loadingFailed", .{
.requestId = &id.toRequestId(msg.transfer),
.requestId = &id.toRequestId(msg.request),
// Seems to be what chrome answers with. I assume it depends on the type of error?
.type = "Ping",
.errorText = msg.err,
@@ -273,24 +275,23 @@ pub fn httpRequestStart(bc: *CDP.BrowserContext, msg: *const Notification.Reques
// things, but no session.
const session_id = bc.session_id orelse return;
const transfer = msg.transfer;
const req = &transfer.req;
const frame_id = req.frame_id;
const req = msg.request;
const frame_id = req.params.frame_id;
const frame = bc.session.findFrameByFrameId(frame_id) orelse return;
// Modify request with extra CDP headers
for (bc.extra_headers.items) |extra| {
try req.headers.add(extra);
try req.params.headers.add(extra);
}
// We're missing a bunch of fields, but, for now, this eems like enough
try bc.cdp.sendEvent("Network.requestWillBeSent", .{
.frameId = &id.toFrameId(frame_id),
.requestId = &id.toRequestId(transfer),
.loaderId = &id.toLoaderId(req.loader_id),
.type = req.resource_type.string(),
.requestId = &id.toRequestId(req),
.loaderId = &id.toLoaderId(req.params.loader_id),
.type = req.params.resource_type.string(),
.documentURL = frame.url,
.request = TransferAsRequestWriter.init(transfer),
.request = RequestWriter.init(req),
.initiator = .{ .type = "other" },
.redirectHasExtraInfo = false, // TODO change after adding Network.requestWillBeSentExtraInfo
.hasUserGesture = false,
@@ -304,15 +305,14 @@ pub fn httpResponseHeaderDone(arena: Allocator, bc: *CDP.BrowserContext, msg: *c
// things, but no session.
const session_id = bc.session_id orelse return;
const transfer = msg.transfer;
const req = &transfer.req;
const req = msg.request;
// We're missing a bunch of fields, but, for now, this seems like enough
try bc.cdp.sendEvent("Network.responseReceived", .{
.frameId = &id.toFrameId(req.frame_id),
.requestId = &id.toRequestId(transfer),
.loaderId = &id.toLoaderId(req.loader_id),
.response = TransferAsResponseWriter.init(arena, transfer),
.frameId = &id.toFrameId(req.params.frame_id),
.requestId = &id.toRequestId(req),
.loaderId = &id.toLoaderId(req.params.loader_id),
.response = ResponseWriter.init(arena, msg.response),
.hasExtraInfo = false, // TODO change after adding Network.responseReceivedExtraInfo
}, .{ .session_id = session_id });
}
@@ -321,36 +321,37 @@ pub fn httpRequestDone(bc: *CDP.BrowserContext, msg: *const Notification.Request
// detachTarget could be called, in which case, we still have a frame doing
// things, but no session.
const session_id = bc.session_id orelse return;
const transfer = msg.transfer;
const req = msg.request;
try bc.cdp.sendEvent("Network.loadingFinished", .{
.requestId = &id.toRequestId(transfer),
.encodedDataLength = transfer.bytes_received,
.requestId = &id.toRequestId(req),
.encodedDataLength = msg.content_length,
}, .{ .session_id = session_id });
}
pub const TransferAsRequestWriter = struct {
transfer: *Transfer,
pub const RequestWriter = struct {
request: *Request,
pub fn init(transfer: *Transfer) TransferAsRequestWriter {
pub fn init(request: *Request) RequestWriter {
return .{
.transfer = transfer,
.request = request,
};
}
pub fn jsonStringify(self: *const TransferAsRequestWriter, jws: anytype) !void {
pub fn jsonStringify(self: *const RequestWriter, jws: anytype) !void {
self._jsonStringify(jws) catch return error.WriteFailed;
}
fn _jsonStringify(self: *const TransferAsRequestWriter, jws: anytype) !void {
const transfer = self.transfer;
fn _jsonStringify(self: *const RequestWriter, jws: anytype) !void {
const request = self.request;
try jws.beginObject();
{
try jws.objectField("url");
try jws.write(transfer.url);
try jws.write(request.params.url);
}
{
const frag = URL.getHash(transfer.url);
const frag = URL.getHash(request.params.url);
if (frag.len > 0) {
try jws.objectField("urlFragment");
try jws.write(frag);
@@ -359,23 +360,23 @@ pub const TransferAsRequestWriter = struct {
{
try jws.objectField("method");
try jws.write(@tagName(transfer.req.method));
try jws.write(@tagName(request.params.method));
}
{
try jws.objectField("hasPostData");
try jws.write(transfer.req.body != null);
try jws.write(request.params.body != null);
}
{
try jws.objectField("headers");
try jws.beginObject();
var it = transfer.req.headers.iterator();
var it = request.params.headers.iterator();
while (it.next()) |hdr| {
try jws.objectField(hdr.name);
try jws.write(hdr.value);
}
if (try transfer.getCookieString()) |cookies| {
if (try request.getCookieString()) |cookies| {
try jws.objectField("Cookie");
try jws.write(cookies[0 .. cookies.len - 1]);
}
@@ -385,34 +386,31 @@ pub const TransferAsRequestWriter = struct {
}
};
const TransferAsResponseWriter = struct {
const ResponseWriter = struct {
arena: Allocator,
transfer: *Transfer,
response: *const Response,
fn init(arena: Allocator, transfer: *Transfer) TransferAsResponseWriter {
fn init(arena: Allocator, response: *const Response) ResponseWriter {
return .{
.arena = arena,
.transfer = transfer,
.response = response,
};
}
pub fn jsonStringify(self: *const TransferAsResponseWriter, jws: anytype) !void {
pub fn jsonStringify(self: *const ResponseWriter, jws: anytype) !void {
self._jsonStringify(jws) catch return error.WriteFailed;
}
fn _jsonStringify(self: *const TransferAsResponseWriter, jws: anytype) !void {
const transfer = self.transfer;
fn _jsonStringify(self: *const ResponseWriter, jws: anytype) !void {
const response = self.response;
try jws.beginObject();
{
try jws.objectField("url");
try jws.write(transfer.url);
try jws.write(response.url());
}
if (transfer.response_header) |*rh| {
// it should not be possible for this to be false, but I'm not
// feeling brave today.
const status = rh.status;
if (response.status()) |status| {
try jws.objectField("status");
try jws.write(status);
@@ -422,7 +420,7 @@ const TransferAsResponseWriter = struct {
{
const mime: Mime = blk: {
if (transfer.response_header.?.contentType()) |ct| {
if (response.contentType()) |ct| {
break :blk try Mime.parse(ct);
}
break :blk .unknown;
@@ -437,7 +435,8 @@ const TransferAsResponseWriter = struct {
{
try jws.objectField("timing");
try jws.write(.{
.requestTime = transfer.start_time,
// TODO: fix
.requestTime = -1,
.connectEnd = -1,
.connectStart = -1,
.dnsEnd = -1,
@@ -458,7 +457,7 @@ const TransferAsResponseWriter = struct {
// common to get these from a server (e.g. for Cache-Control), but
// Chrome joins these. So we have to too.
const arena = self.arena;
var it = transfer.responseHeaderIterator();
var it = response.headerIterator();
var map: std.StringArrayHashMapUnmanaged([]const u8) = .empty;
while (it.next()) |hdr| {
const gop = try map.getOrPut(arena, hdr.name);

2
src/cdp/domains/page.zig
View File

@@ -145,7 +145,7 @@ fn setLifecycleEventsEnabled(cmd: *CDP.Command) !void {
const http_client = frame._session.browser.http_client;
const http_active = http_client.http_active;
const total_network_activity = http_active + http_client.intercepted;
const total_network_activity = http_active + http_client.interception_layer.intercepted;
if (frame._notified_network_almost_idle.check(total_network_activity <= 2)) {
try sendPageLifecycle(bc, "networkAlmostIdle", now, frame_id, loader_id);
}

11
src/cdp/id.zig
View File

@@ -40,15 +40,14 @@ pub fn toLoaderId(id: u32) [14]u8 {
// requestId has special requirements. If it's the main document navigation,
// then it should match the loader id.
const Transfer = @import("../browser/HttpClient.zig").Transfer;
pub fn toRequestId(transfer: *const Transfer) [14]u8 {
const req = transfer.req;
if (req.resource_type == .document) {
return toLoaderId(req.loader_id);
const Request = @import("../browser/HttpClient.zig").Request;
pub fn toRequestId(req: *const Request) [14]u8 {
if (req.params.resource_type == .document) {
return toLoaderId(req.params.loader_id);
}
var buf: [14]u8 = undefined;
_ = std.fmt.bufPrint(&buf, "REQ-{d:0>10}", .{transfer.id}) catch unreachable;
_ = std.fmt.bufPrint(&buf, "REQ-{d:0>10}", .{req.params.request_id}) catch unreachable;
return buf;
}

232
src/network/layer/CacheLayer.zig Normal file
View File

@@ -0,0 +1,232 @@
// Copyright (C) 2023-2026 Lightpanda (Selecy SAS)
//
// Francis Bouvier <francis@lightpanda.io>
// Pierre Tachoire <pierre@lightpanda.io>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
const std = @import("std");
const lp = @import("lightpanda");
const log = lp.log;
const http = @import("../http.zig");
const Client = @import("../../browser/HttpClient.zig").Client;
const Transfer = @import("../../browser/HttpClient.zig").Transfer;
const Request = @import("../../browser/HttpClient.zig").Request;
const Response = @import("../../browser/HttpClient.zig").Response;
const Layer = @import("../../browser/HttpClient.zig").Layer;
const Cache = @import("../cache/Cache.zig");
const CachedMetadata = @import("../cache/Cache.zig").CachedMetadata;
const CachedResponse = @import("../cache/Cache.zig").CachedResponse;
const Forward = @import("Forward.zig");
const CacheLayer = @This();
next: Layer = undefined,
pub fn layer(self: *CacheLayer) Layer {
return .{
.ptr = self,
.vtable = &.{
.request = request,
},
};
}
fn request(ptr: *anyopaque, client: *Client, req: Request) anyerror!void {
const self: *CacheLayer = @ptrCast(@alignCast(ptr));
const network = client.network;
if (req.params.method != .GET) {
return self.next.request(client, req);
}
const arena = req.params.arena;
var iter = req.params.headers.iterator();
const req_header_list = try iter.collect(arena);
if (network.cache.?.get(arena, .{
.url = req.params.url,
.timestamp = std.time.timestamp(),
.request_headers = req_header_list.items,
})) |cached| {
return serveFromCache(req, &cached);
}
const cache_ctx = try arena.create(CacheContext);
cache_ctx.* = .{
.arena = arena,
.client = client,
.forward = Forward.fromRequest(req),
.req_url = req.params.url,
.req_headers = req.params.headers,
};
const wrapped = cache_ctx.forward.wrapRequest(
req,
cache_ctx,
.{
.start = CacheContext.startCallback,
.header = CacheContext.headerCallback,
.done = CacheContext.doneCallback,
.shutdown = CacheContext.shutdownCallback,
.err = CacheContext.errorCallback,
},
);
return self.next.request(client, wrapped);
}
fn serveFromCache(req: Request, cached: *const CachedResponse) !void {
const response = Response.fromCached(req.ctx, cached);
defer switch (cached.data) {
.buffer => |_| {},
.file => |f| f.file.close(),
};
if (req.start_callback) |cb| {
try cb(response);
}
const proceed = try req.header_callback(response);
if (!proceed) {
return error.Abort;
}
switch (cached.data) {
.buffer => |data| {
if (data.len > 0) {
try req.data_callback(response, data);
}
},
.file => |f| {
const file = f.file;
var buf: [1024]u8 = undefined;
var file_reader = file.reader(&buf);
try file_reader.seekTo(f.offset);
const reader = &file_reader.interface;
var read_buf: [1024]u8 = undefined;
var remaining = f.len;
while (remaining > 0) {
const read_len = @min(read_buf.len, remaining);
const n = try reader.readSliceShort(read_buf[0..read_len]);
if (n == 0) break;
remaining -= n;
try req.data_callback(response, read_buf[0..n]);
}
},
}
try req.done_callback(req.ctx);
}
const CacheContext = struct {
arena: std.mem.Allocator,
client: *Client,
transfer: ?*Transfer = null,
forward: Forward,
req_url: [:0]const u8,
req_headers: http.Headers,
pending_metadata: ?*CachedMetadata = null,
fn startCallback(response: Response) anyerror!void {
const self: *CacheContext = @ptrCast(@alignCast(response.ctx));
self.transfer = response.inner.transfer;
return self.forward.forwardStart(response);
}
fn headerCallback(response: Response) anyerror!bool {
const self: *CacheContext = @ptrCast(@alignCast(response.ctx));
const allocator = self.arena;
const transfer = response.inner.transfer;
var rh = &transfer.response_header.?;
const conn = transfer._conn.?;
const vary = if (conn.getResponseHeader("vary", 0)) |h| h.value else null;
const maybe_cm = try Cache.tryCache(
allocator,
std.time.timestamp(),
transfer.url,
rh.status,
rh.contentType(),
if (conn.getResponseHeader("cache-control", 0)) |h| h.value else null,
vary,
if (conn.getResponseHeader("age", 0)) |h| h.value else null,
conn.getResponseHeader("set-cookie", 0) != null,
conn.getResponseHeader("authorization", 0) != null,
);
if (maybe_cm) |cm| {
var iter = transfer.responseHeaderIterator();
var header_list = try iter.collect(allocator);
const end_of_response = header_list.items.len;
if (vary) |vary_str| {
var req_it = self.req_headers.iterator();
while (req_it.next()) |hdr| {
var vary_iter = std.mem.splitScalar(u8, vary_str, ',');
while (vary_iter.next()) |part| {
const name = std.mem.trim(u8, part, &std.ascii.whitespace);
if (std.ascii.eqlIgnoreCase(hdr.name, name)) {
try header_list.append(allocator, .{
.name = try allocator.dupe(u8, hdr.name),
.value = try allocator.dupe(u8, hdr.value),
});
}
}
}
}
const metadata = try allocator.create(CachedMetadata);
metadata.* = cm;
metadata.headers = header_list.items[0..end_of_response];
metadata.vary_headers = header_list.items[end_of_response..];
self.pending_metadata = metadata;
}
return self.forward.forwardHeader(response);
}
fn doneCallback(ctx: *anyopaque) anyerror!void {
const self: *CacheContext = @ptrCast(@alignCast(ctx));
const transfer = self.transfer orelse @panic("Start Callback didn't set CacheLayer.transfer");
if (self.pending_metadata) |metadata| {
const cache = &self.client.network.cache.?;
log.debug(.browser, "http cache", .{ .key = self.req_url, .metadata = metadata });
cache.put(metadata.*, transfer._stream_buffer.items) catch |err| {
log.warn(.http, "cache put failed", .{ .err = err });
};
log.debug(.browser, "http.cache.put", .{ .url = self.req_url });
}
return self.forward.forwardDone();
}
fn shutdownCallback(ctx: *anyopaque) void {
const self: *CacheContext = @ptrCast(@alignCast(ctx));
self.forward.forwardShutdown();
}
fn errorCallback(ctx: *anyopaque, e: anyerror) void {
const self: *CacheContext = @ptrCast(@alignCast(ctx));
self.forward.forwardErr(e);
}
};

134
src/network/layer/Forward.zig Normal file
View File

@@ -0,0 +1,134 @@
// Copyright (C) 2023-2026 Lightpanda (Selecy SAS)
//
// Francis Bouvier <francis@lightpanda.io>
// Pierre Tachoire <pierre@lightpanda.io>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
const Request = @import("../../browser/HttpClient.zig").Request;
const Response = @import("../../browser/HttpClient.zig").Response;
const Forward = @This();
ctx: *anyopaque,
start: ?Request.StartCallback,
header: Request.HeaderCallback,
data: Request.DataCallback,
done: Request.DoneCallback,
err: Request.ErrorCallback,
shutdown: ?Request.ShutdownCallback,
pub fn fromRequest(req: Request) Forward {
return .{
.ctx = req.ctx,
.start = req.start_callback,
.header = req.header_callback,
.data = req.data_callback,
.done = req.done_callback,
.err = req.error_callback,
.shutdown = req.shutdown_callback,
};
}
pub const Overrides = struct {
start: ?Request.StartCallback = null,
header: ?Request.HeaderCallback = null,
data: ?Request.DataCallback = null,
done: ?Request.DoneCallback = null,
err: ?Request.ErrorCallback = null,
shutdown: ?Request.ShutdownCallback = null,
};
pub fn wrapRequest(
self: *Forward,
req: Request,
new_ctx: anytype,
overrides: Overrides,
) Request {
const T = @TypeOf(new_ctx.*);
const PassthroughT = makePassthrough(T, "forward");
var wrapped = req;
wrapped.ctx = new_ctx;
wrapped.start_callback = overrides.start orelse if (self.start != null) PassthroughT.start else null;
wrapped.header_callback = overrides.header orelse PassthroughT.header;
wrapped.data_callback = overrides.data orelse PassthroughT.data;
wrapped.done_callback = overrides.done orelse PassthroughT.done;
wrapped.error_callback = overrides.err orelse PassthroughT.err;
wrapped.shutdown_callback = overrides.shutdown orelse if (self.shutdown != null) PassthroughT.shutdown else null;
return wrapped;
}
fn makePassthrough(comptime T: type, comptime field: []const u8) type {
return struct {
pub fn start(response: Response) anyerror!void {
const self: *T = @ptrCast(@alignCast(response.ctx));
return @field(self, field).forwardStart(response);
}
pub fn header(response: Response) anyerror!bool {
const self: *T = @ptrCast(@alignCast(response.ctx));
return @field(self, field).forwardHeader(response);
}
pub fn data(response: Response, chunk: []const u8) anyerror!void {
const self: *T = @ptrCast(@alignCast(response.ctx));
return @field(self, field).forwardData(response, chunk);
}
pub fn done(ctx_ptr: *anyopaque) anyerror!void {
const self: *T = @ptrCast(@alignCast(ctx_ptr));
return @field(self, field).forwardDone();
}
pub fn err(ctx_ptr: *anyopaque, e: anyerror) void {
const self: *T = @ptrCast(@alignCast(ctx_ptr));
@field(self, field).forwardErr(e);
}
pub fn shutdown(ctx_ptr: *anyopaque) void {
const self: *T = @ptrCast(@alignCast(ctx_ptr));
@field(self, field).forwardShutdown();
}
};
}
pub fn forwardStart(self: Forward, response: Response) anyerror!void {
var fwd = response;
fwd.ctx = self.ctx;
if (self.start) |cb| try cb(fwd);
}
pub fn forwardHeader(self: Forward, response: Response) anyerror!bool {
var fwd = response;
fwd.ctx = self.ctx;
return self.header(fwd);
}
pub fn forwardData(self: Forward, response: Response, chunk: []const u8) anyerror!void {
var fwd = response;
fwd.ctx = self.ctx;
return self.data(fwd, chunk);
}
pub fn forwardDone(self: Forward) anyerror!void {
return self.done(self.ctx);
}
pub fn forwardErr(self: Forward, e: anyerror) void {
self.err(self.ctx, e);
}
pub fn forwardShutdown(self: Forward) void {
if (self.shutdown) |cb| cb(self.ctx);
}

270
src/network/layer/InterceptionLayer.zig Normal file
View File

@@ -0,0 +1,270 @@
// Copyright (C) 2023-2026 Lightpanda (Selecy SAS)
//
// Francis Bouvier <francis@lightpanda.io>
// Pierre Tachoire <pierre@lightpanda.io>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
const std = @import("std");
const builtin = @import("builtin");
const lp = @import("lightpanda");
const log = lp.log;
const IS_DEBUG = builtin.mode == .Debug;
const http = @import("../http.zig");
const Client = @import("../../browser/HttpClient.zig").Client;
const Request = @import("../../browser/HttpClient.zig").Request;
const Response = @import("../../browser/HttpClient.zig").Response;
const FulfilledResponse = @import("../../browser/HttpClient.zig").FulfilledResponse;
const Layer = @import("../../browser/HttpClient.zig").Layer;
const Forward = @import("Forward.zig");
const InterceptionLayer = @This();
// Count of intercepted requests. This is to help deal with intercepted requests.
// The client doesn't track intercepted transfers. If a request is intercepted,
// the client forgets about it and requires the interceptor to continue or abort
// it. That works well, except if we only rely on active, we might think there's
// no more network activity when, with interecepted requests, there might be more
// in the future. (We really only need this to properly emit a 'networkIdle' and
// 'networkAlmostIdle' Page.lifecycleEvent in CDP).
intercepted: usize = 0,
next: Layer = undefined,
pub fn layer(self: *InterceptionLayer) Layer {
return .{
.ptr = self,
.vtable = &.{ .request = request },
};
}
fn request(ptr: *anyopaque, client: *Client, in_req: Request) anyerror!void {
const self: *InterceptionLayer = @ptrCast(@alignCast(ptr));
const intercept_ctx = try in_req.params.arena.create(InterceptContext);
intercept_ctx.* = .{
.client = client,
.forward = Forward.fromRequest(in_req),
.layer = self,
.request = in_req,
};
var req = intercept_ctx.forward.wrapRequest(
in_req,
intercept_ctx,
.{
.start = InterceptContext.startCallback,
.header = InterceptContext.headerCallback,
.data = InterceptContext.dataCallback,
.done = InterceptContext.doneCallback,
.err = InterceptContext.errorCallback,
.shutdown = InterceptContext.shutdownCallback,
},
);
req.params.notification.dispatch(.http_request_start, &.{ .request = &req });
var wait_for_interception = false;
req.params.notification.dispatch(.http_request_intercept, &.{
.request = &req,
.wait_for_interception = &wait_for_interception,
});
log.debug(.http, "interception check", .{
.wait_for_interception = wait_for_interception,
.intercepted = self.intercepted,
.url = req.params.url,
});
if (!wait_for_interception) {
return self.next.request(client, req);
}
self.intercepted += 1;
if (comptime IS_DEBUG) {
log.debug(.http, "wait for interception", .{ .intercepted = self.intercepted });
}
}
pub const InterceptContext = struct {
client: *Client,
forward: Forward,
layer: *InterceptionLayer,
request: Request,
content_length: usize = 0,
fn startCallback(response: Response) anyerror!void {
const self: *InterceptContext = @ptrCast(@alignCast(response.ctx));
log.debug(.http, "intercept start", .{ .url = self.request.params.url });
return self.forward.forwardStart(response);
}
fn headerCallback(response: Response) anyerror!bool {
const self: *InterceptContext = @ptrCast(@alignCast(response.ctx));
log.debug(.http, "intercept header", .{
.url = self.request.params.url,
.status = response.status(),
.content_length = response.contentLength(),
});
self.content_length = response.contentLength() orelse 0;
self.request.params.notification.dispatch(.http_response_header_done, &.{
.request = &self.request,
.response = &response,
});
return self.forward.forwardHeader(response);
}
fn dataCallback(response: Response, chunk: []const u8) anyerror!void {
const self: *InterceptContext = @ptrCast(@alignCast(response.ctx));
log.debug(.http, "intercept data", .{
.url = self.request.params.url,
.len = chunk.len,
});
self.request.params.notification.dispatch(.http_response_data, &.{
.data = chunk,
.request = &self.request,
});
return self.forward.forwardData(response, chunk);
}
fn doneCallback(ctx: *anyopaque) anyerror!void {
const self: *InterceptContext = @ptrCast(@alignCast(ctx));
log.debug(.http, "intercept done", .{
.url = self.request.params.url,
.content_length = self.content_length,
});
self.request.params.notification.dispatch(.http_request_done, &.{
.request = &self.request,
.content_length = self.content_length,
});
return self.forward.forwardDone();
}
fn errorCallback(ctx: *anyopaque, err: anyerror) void {
const self: *InterceptContext = @ptrCast(@alignCast(ctx));
log.debug(.http, "intercept error", .{
.url = self.request.params.url,
.err = err,
});
self.request.params.notification.dispatch(.http_request_fail, &.{
.request = &self.request,
.err = err,
});
self.forward.forwardErr(err);
}
fn shutdownCallback(ctx: *anyopaque) void {
const self: *InterceptContext = @ptrCast(@alignCast(ctx));
log.debug(.http, "intercept shutdown", .{ .url = self.request.params.url });
self.request.params.notification.dispatch(.http_request_fail, &.{
.request = &self.request,
.err = error.Shutdown,
});
self.forward.forwardShutdown();
}
};
// CDP Callbacks
// These handle their own clean up on errors with `self.next.request`.
// This is because they don't pass their error up the chain as they are async callbacks.
pub fn continueRequest(self: *InterceptionLayer, client: *Client, req: Request) anyerror!void {
if (comptime IS_DEBUG) {
lp.assert(self.intercepted > 0, "InterceptionLayer.continueRequest", .{ .value = self.intercepted });
log.debug(.http, "continue transfer", .{ .intercepted = self.intercepted });
}
self.intercepted -= 1;
self.next.request(client, req) catch |err| {
const ctx: *InterceptContext = @ptrCast(@alignCast(req.ctx));
req.error_callback(req.ctx, err);
ctx.client.deinitRequest(req);
return err;
};
}
pub fn abortRequest(self: *InterceptionLayer, client: *Client, req: Request) void {
if (comptime IS_DEBUG) {
lp.assert(self.intercepted > 0, "InterceptionLayer.abortRequest", .{ .value = self.intercepted });
log.debug(.http, "abort transfer", .{ .intercepted = self.intercepted });
}
self.intercepted -= 1;
req.error_callback(req.ctx, error.Abort);
client.deinitRequest(req);
}
fn fulfillInner(
req: Request,
status: u16,
headers: []const http.Header,
body: ?[]const u8,
) !void {
const fulfilled = FulfilledResponse{
.status = status,
.url = req.params.url,
.headers = headers,
.body = body,
};
const response = Response.fromFulfilled(req.ctx, &fulfilled);
if (req.start_callback) |cb| {
try cb(response);
}
const proceed = try req.header_callback(response);
if (!proceed) {
return error.Abort;
}
if (body) |b| {
try req.data_callback(response, b);
}
try req.done_callback(req.ctx);
}
pub fn fulfillRequest(
self: *InterceptionLayer,
client: *Client,
req: Request,
status: u16,
headers: []const http.Header,
body: ?[]const u8,
) !void {
if (comptime IS_DEBUG) {
lp.assert(self.intercepted > 0, "InterceptionLayer.fulfillRequest", .{ .value = self.intercepted });
log.debug(.http, "fulfill transfer", .{ .intercepted = self.intercepted });
}
self.intercepted -= 1;
defer client.deinitRequest(req);
fulfillInner(req, status, headers, body) catch |err| {
req.error_callback(req.ctx, err);
return err;
};
}

261
src/network/layer/RobotsLayer.zig Normal file
View File

@@ -0,0 +1,261 @@
// Copyright (C) 2023-2026 Lightpanda (Selecy SAS)
//
// Francis Bouvier <francis@lightpanda.io>
// Pierre Tachoire <pierre@lightpanda.io>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
const std = @import("std");
const lp = @import("lightpanda");
const log = lp.log;
const URL = @import("../../browser/URL.zig");
const Robots = @import("../Robots.zig");
const Client = @import("../../browser/HttpClient.zig").Client;
const Request = @import("../../browser/HttpClient.zig").Request;
const Response = @import("../../browser/HttpClient.zig").Response;
const Layer = @import("../../browser/HttpClient.zig").Layer;
const Forward = @import("Forward.zig");
const RobotsLayer = @This();
next: Layer = undefined,
allocator: std.mem.Allocator,
pending: std.StringHashMapUnmanaged(std.ArrayListUnmanaged(Request)) = .empty,
pub fn layer(self: *RobotsLayer) Layer {
return .{
.ptr = self,
.vtable = &.{
.request = request,
},
};
}
pub fn deinit(self: *RobotsLayer, allocator: std.mem.Allocator) void {
var it = self.pending.iterator();
while (it.next()) |entry| {
entry.value_ptr.deinit(allocator);
}
self.pending.deinit(allocator);
}
fn request(ptr: *anyopaque, client: *Client, req: Request) anyerror!void {
const self: *RobotsLayer = @ptrCast(@alignCast(ptr));
const arena = req.params.arena;
const robots_url = try URL.getRobotsUrl(arena, req.params.url);
if (client.network.robot_store.get(robots_url)) |robot_entry| {
switch (robot_entry) {
.present => |robots| {
const path = URL.getPathname(req.params.url);
if (!robots.isAllowed(path)) {
log.warn(.http, "blocked by robots", .{ .url = req.params.url });
return error.RobotsBlocked;
}
},
.absent => {},
}
return self.next.request(client, req);
}
return self.fetchRobotsThenRequest(client, robots_url, req);
}
fn fetchRobotsThenRequest(
self: *RobotsLayer,
client: *Client,
robots_url: [:0]const u8,
req: Request,
) !void {
const entry = try self.pending.getOrPut(self.allocator, robots_url);
if (!entry.found_existing) {
errdefer std.debug.assert(self.pending.remove(robots_url));
entry.value_ptr.* = .empty;
// This arena is later owned by the Request. It does not need to be cleaned up by us because
// it will be cleaned up by the `Transfer.deinit()` or any `Request.deinit()` called on any sublayers.
const new_arena = try client.network.app.arena_pool.acquire(.small, "RobotsLayer.RobotsContext");
errdefer client.network.app.arena_pool.release(new_arena);
const robots_ctx = try new_arena.create(RobotsContext);
robots_ctx.* = .{
.layer = self,
.client = client,
.arena = new_arena,
.robots_url = robots_url,
.buffer = .empty,
};
const headers = try client.newHeaders();
log.debug(.browser, "fetching robots.txt", .{ .robots_url = robots_url });
try self.next.request(client, .{
.ctx = robots_ctx,
.params = .{
// We have to do this ourselves because we are not going through the top level `request`.
.arena = new_arena,
.request_id = client.incrReqId(),
.url = robots_url,
.method = .GET,
.headers = headers,
.frame_id = req.params.frame_id,
.loader_id = req.params.loader_id,
.cookie_jar = req.params.cookie_jar,
.cookie_origin = req.params.cookie_origin,
.notification = req.params.notification,
.resource_type = .fetch,
},
.header_callback = RobotsContext.headerCallback,
.data_callback = RobotsContext.dataCallback,
.done_callback = RobotsContext.doneCallback,
.error_callback = RobotsContext.errorCallback,
.shutdown_callback = RobotsContext.shutdownCallback,
});
}
try entry.value_ptr.append(self.allocator, req);
}
fn flushPending(self: *RobotsLayer, client: *Client, robots_url: [:0]const u8, allowed: bool) void {
var queued = self.pending.fetchRemove(robots_url) orelse
@panic("RobotsLayer.flushPending: missing queue");
defer queued.value.deinit(self.allocator);
for (queued.value.items) |queued_req| {
if (!allowed) {
log.warn(.http, "blocked by robots", .{ .url = queued_req.params.url });
defer client.deinitRequest(queued_req);
queued_req.error_callback(queued_req.ctx, error.RobotsBlocked);
} else {
self.next.request(client, queued_req) catch |e| {
defer client.deinitRequest(queued_req);
queued_req.error_callback(queued_req.ctx, e);
};
}
}
}
fn flushPendingShutdown(self: *RobotsLayer, robots_url: [:0]const u8, client: *Client) void {
var queued = self.pending.fetchRemove(robots_url) orelse
@panic("RobotsLayer.flushPendingShutdown: missing queue");
defer queued.value.deinit(self.allocator);
for (queued.value.items) |queued_req| {
defer client.deinitRequest(queued_req);
if (queued_req.shutdown_callback) |cb| cb(queued_req.ctx);
}
}
const RobotsContext = struct {
layer: *RobotsLayer,
arena: std.mem.Allocator,
client: *Client,
robots_url: [:0]const u8,
buffer: std.ArrayListUnmanaged(u8),
status: u16 = 0,
fn deinit(self: *RobotsContext) void {
self.buffer.deinit(self.layer.allocator);
self.layer.allocator.destroy(self);
}
fn headerCallback(response: Response) anyerror!bool {
const self: *RobotsContext = @ptrCast(@alignCast(response.ctx));
switch (response.inner) {
.transfer => |t| {
if (t.response_header) |hdr| {
log.debug(.browser, "robots status", .{ .status = hdr.status, .robots_url = self.robots_url });
self.status = hdr.status;
}
if (t.getContentLength()) |cl| {
try self.buffer.ensureTotalCapacity(self.arena, cl);
}
},
else => {},
}
return true;
}
fn dataCallback(response: Response, data: []const u8) anyerror!void {
const self: *RobotsContext = @ptrCast(@alignCast(response.ctx));
try self.buffer.appendSlice(self.arena, data);
}
fn doneCallback(ctx_ptr: *anyopaque) anyerror!void {
const self: *RobotsContext = @ptrCast(@alignCast(ctx_ptr));
const l = self.layer;
const client = self.client;
const robots_url = self.robots_url;
var allowed = true;
const network = client.network;
switch (self.status) {
200 => {
if (self.buffer.items.len > 0) {
const robots: ?Robots = network.robot_store.robotsFromBytes(
network.config.http_headers.user_agent,
self.buffer.items,
) catch blk: {
log.warn(.browser, "failed to parse robots", .{ .robots_url = robots_url });
try network.robot_store.putAbsent(robots_url);
break :blk null;
};
if (robots) |r| {
try network.robot_store.put(robots_url, r);
const path = URL.getPathname(l.pending.get(robots_url).?.items[0].params.url);
allowed = r.isAllowed(path);
}
}
},
404 => {
log.debug(.http, "robots not found", .{ .url = robots_url });
try network.robot_store.putAbsent(robots_url);
},
else => {
log.debug(.http, "unexpected status on robots", .{
.url = robots_url,
.status = self.status,
});
try network.robot_store.putAbsent(robots_url);
},
}
l.flushPending(client, robots_url, allowed);
}
fn errorCallback(ctx_ptr: *anyopaque, err: anyerror) void {
const self: *RobotsContext = @ptrCast(@alignCast(ctx_ptr));
const l = self.layer;
const client = self.client;
const robots_url = self.robots_url;
log.warn(.http, "robots fetch failed", .{ .err = err });
l.flushPending(client, robots_url, true);
}
fn shutdownCallback(ctx_ptr: *anyopaque) void {
const self: *RobotsContext = @ptrCast(@alignCast(ctx_ptr));
const l = self.layer;
const client = self.client;
const robots_url = self.robots_url;
log.debug(.http, "robots fetch shutdown", .{});
l.flushPendingShutdown(robots_url, client);
}
};

51
src/network/layer/WebBotAuthLayer.zig Normal file
View File

@@ -0,0 +1,51 @@
// Copyright (C) 2023-2026 Lightpanda (Selecy SAS)
//
// Francis Bouvier <francis@lightpanda.io>
// Pierre Tachoire <pierre@lightpanda.io>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
const std = @import("std");
const lp = @import("lightpanda");
const log = lp.log;
const URL = @import("../../browser/URL.zig");
const WebBotAuth = @import("../WebBotAuth.zig");
const Client = @import("../../browser/HttpClient.zig").Client;
const Request = @import("../../browser/HttpClient.zig").Request;
const Layer = @import("../../browser/HttpClient.zig").Layer;
const WebBotAuthLayer = @This();
next: Layer = undefined,
pub fn layer(self: *WebBotAuthLayer) Layer {
return .{
.ptr = self,
.vtable = &.{ .request = request },
};
}
fn request(ptr: *anyopaque, client: *Client, req: Request) anyerror!void {
const self: *WebBotAuthLayer = @ptrCast(@alignCast(ptr));
var our_req = req;
const wba = client.network.web_bot_auth orelse @panic("WebBotAuthLayer shouldn't be active without WebBotAuth");
const arena = req.params.arena;
const authority = URL.getHost(req.params.url);
try wba.signRequest(arena, &our_req.params.headers, authority);
return self.next.request(client, our_req);
}