mirror/browser
1
0
Fork 0
mirror of https://github.com/lightpanda-io/browser.git synced 2026-06-11 01:25:53 -04:00
Code Issues Packages Projects Releases Wiki Activity

Cleanup HttpClient.Transfer

Browse Source 
This is just moving fields around. The end result is that there's a
`transfer.req` and a `transfer.res`.

On the Request side, we use to have a nested `params: RequestParam` resulting
in a lot of `transfer.req.params.url`. This is now `transfer.req.url`. On the
Response side, we had the exact opposite: response fields splattered directly
in the transfer, `transfer.response_header`. This is now `transfer.res.header`.

There is now an HttpClient.Response, which is the actual final response (which
could be for a transfer or something else, e.g the cache). And an
HttpClient.Transfer.Response which captures the inflight response data (and is
one of the polymorphic variants of the HttpClient.Response). Probably still not
ideal, but I'm not sure how to make it cleaner, and even if this is just an
intermediary step, I consider it an small win.
This commit is contained in:
Karl Seguin
2026-05-15 12:39:42 +08:00
parent cb8c2bc4d8
commit a5162bea8f
15 changed files with 279 additions and 303 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
src/browser/Frame.zig
View File

@@ -659,18 +659,16 @@ pub fn navigate(self: *Frame, request_url: [:0]const u8, opts: NavigateOpts) !vo
self.makeRequest(.{
.ctx = self,
.params = .{
.url = self.url,
.frame_id = self._frame_id,
.loader_id = self._loader_id,
.method = opts.method,
.headers = headers,
.body = opts.body,
.cookie_jar = &session.cookie_jar,
.cookie_origin = self.url,
.resource_type = .document,
.notification = self._session.notification,
},
.url = self.url,
.frame_id = self._frame_id,
.loader_id = self._loader_id,
.method = opts.method,
.headers = headers,
.body = opts.body,
.cookie_jar = &session.cookie_jar,
.cookie_origin = self.url,
.resource_type = .document,
.notification = self._session.notification,
.header_callback = frameHeaderDoneCallback,
.data_callback = frameDataCallback,
.done_callback = frameDoneCallback,

255
src/browser/HttpClient.zig
View File

@@ -408,35 +408,34 @@ pub fn _request(_: *anyopaque, transfer: *Transfer) !void {
}
// Ownership contract: from the moment this function is entered, the
// HttpClient owns `req` — specifically `req.params.headers` (a curl_slist).
// HttpClient owns `req` — specifically `req.headers` (a curl_slist).
// On success, transfer.deinit eventually frees it. On any failure path
// inside this function, we free it before returning the error. Callers
// must NOT pair `request()` with their own `errdefer headers.deinit()`
// — that's a double-free.
pub fn request(self: *Client, req: Request, owner: ?*Owner) !void {
const arena = self.arena_pool.acquire(.small, "Request.arena") catch |err| {
req.params.headers.deinit();
req.headers.deinit();
return err;
};
const transfer = arena.create(Transfer) catch |err| {
req.params.headers.deinit();
req.headers.deinit();
self.arena_pool.release(arena);
return err;
};
transfer.* = .{
.req = req,
.url = req.params.url,
.client = self,
.arena = arena,
.id = self.incrReqId(),
.start_time = timestamp(.monotonic),
// owner is set AFTER we've actually appended to the owner list,
// so transfer.deinit's `if (self.owner)` branch only fires when
// we're truly linked. Otherwise we'd try to remove a node from
// a list it was never in.
.owner = null,
.arena = arena,
.id = self.incrReqId(),
.start_time = timestamp(.monotonic),
.owner_node = .{},
};
@@ -512,19 +511,18 @@ const SyncContext = struct {
}
};
pub fn syncRequest(self: *Client, allocator: Allocator, params: RequestParams) !SyncResponse {
pub fn syncRequest(self: *Client, allocator: Allocator, req: Request) !SyncResponse {
var sync_ctx = SyncContext{ .allocator = allocator, .body = .empty };
errdefer sync_ctx.body.deinit(allocator);
try self.request(.{
.params = params,
.ctx = &sync_ctx,
.header_callback = SyncContext.headerCallback,
.data_callback = SyncContext.dataCallback,
.done_callback = SyncContext.doneCallback,
.error_callback = SyncContext.errorCallback,
.shutdown_callback = SyncContext.shutdownCallback,
}, null);
var r = req;
r.ctx = &sync_ctx;
r.header_callback = SyncContext.headerCallback;
r.data_callback = SyncContext.dataCallback;
r.done_callback = SyncContext.doneCallback;
r.error_callback = SyncContext.errorCallback;
r.shutdown_callback = SyncContext.shutdownCallback;
try self.request(r, null);
while (sync_ctx.completion == .in_progress) {
const status = try self.tick(200);
@@ -692,7 +690,7 @@ fn processOneMessage(self: *Client, msg: http.Handles.MultiMessage, transfer: *T
// TODO give a way to configure the number of auth retries.
if (transfer._auth_challenge != null and transfer._tries < 10) {
var wait_for_interception = false;
transfer.req.params.notification.dispatch(
transfer.req.notification.dispatch(
.http_request_auth_required,
&.{ .transfer = transfer, .wait_for_interception = &wait_for_interception },
);
@@ -756,11 +754,11 @@ fn processOneMessage(self: *Client, msg: http.Handles.MultiMessage, transfer: *T
defer transfer._performing = false;
if (msg.err != null and !is_conn_close_recv) {
transfer.requestFailed(transfer._callback_error orelse msg.err.?, true);
transfer.requestFailed(transfer.res.callback_error orelse msg.err.?, true);
return true;
}
if (!transfer._header_done_called) {
if (!transfer.res.header_done_called) {
// In case of request w/o data, we need to call the header done
// callback now.
const proceed = try transfer.headerDoneCallback(msg.conn);
@@ -770,10 +768,10 @@ fn processOneMessage(self: *Client, msg: http.Handles.MultiMessage, transfer: *T
}
}
const body = transfer._stream_buffer.items;
const body = transfer.res.stream_buffer.items;
// Replay buffered body through user's data_callback.
if (transfer._stream_buffer.items.len > 0) {
if (body.len > 0) {
try transfer.req.data_callback(Response.fromTransfer(transfer), body);
if (transfer.aborted) {
@@ -897,22 +895,15 @@ fn ensureNoActiveConnection(self: *const Client) !void {
}
}
pub const RequestParams = struct {
frame_id: u32,
loader_id: u32,
method: Method,
url: [:0]const u8,
headers: http.Headers,
body: ?[]const u8 = null,
cookie_jar: ?*CookieJar,
cookie_origin: [:0]const u8,
resource_type: ResourceType,
credentials: ?[:0]const u8 = null,
notification: *Notification,
timeout_ms: u32 = 0,
skip_robots: bool = false,
pub const Request = struct {
pub const StartCallback = *const fn (response: Response) anyerror!void;
pub const HeaderCallback = *const fn (response: Response) anyerror!bool;
pub const DataCallback = *const fn (response: Response, data: []const u8) anyerror!void;
pub const DoneCallback = *const fn (ctx: *anyopaque) anyerror!void;
pub const ErrorCallback = *const fn (ctx: *anyopaque, err: anyerror) void;
pub const ShutdownCallback = *const fn (ctx: *anyopaque) void;
const ResourceType = enum {
pub const ResourceType = enum {
document,
xhr,
script,
@@ -932,37 +923,37 @@ pub const RequestParams = struct {
}
};
pub fn deinit(self: *const RequestParams) void {
self.headers.deinit();
}
};
frame_id: u32,
loader_id: u32,
method: Method,
url: [:0]const u8,
headers: http.Headers,
body: ?[]const u8 = null,
cookie_jar: ?*CookieJar,
cookie_origin: [:0]const u8,
resource_type: ResourceType,
credentials: ?[:0]const u8 = null,
notification: *Notification,
timeout_ms: u32 = 0,
skip_robots: bool = false,
pub const Request = struct {
pub const StartCallback = *const fn (response: Response) anyerror!void;
pub const HeaderCallback = *const fn (response: Response) anyerror!bool;
pub const DataCallback = *const fn (response: Response, data: []const u8) anyerror!void;
pub const DoneCallback = *const fn (ctx: *anyopaque) anyerror!void;
pub const ErrorCallback = *const fn (ctx: *anyopaque, err: anyerror) void;
pub const ShutdownCallback = *const fn (ctx: *anyopaque) void;
params: RequestParams,
// arbitrary data that can be associated with this request
ctx: *anyopaque = undefined,
start_callback: ?StartCallback = null,
header_callback: HeaderCallback,
data_callback: DataCallback,
done_callback: DoneCallback,
error_callback: ErrorCallback,
header_callback: HeaderCallback = Noop.headerCallback,
data_callback: DataCallback = Noop.dataCallback,
done_callback: DoneCallback = Noop.doneCallback,
error_callback: ErrorCallback = Noop.errorCallback,
shutdown_callback: ?ShutdownCallback = null,
pub fn getCookieString(self: *Request, arena: Allocator) !?[:0]const u8 {
const jar = self.params.cookie_jar orelse return null;
const jar = self.cookie_jar orelse return null;
var aw: std.Io.Writer.Allocating = .init(arena);
try jar.forRequest(self.params.url, &aw.writer, .{
try jar.forRequest(self.url, &aw.writer, .{
.is_http = true,
.origin_url = self.params.cookie_origin,
.is_navigation = self.params.resource_type == .document,
.origin_url = self.cookie_origin,
.is_navigation = self.resource_type == .document,
});
const written = aw.written();
if (written.len == 0) return null;
@@ -971,7 +962,7 @@ pub const Request = struct {
}
pub fn deinit(self: *const Request) void {
self.params.deinit();
self.headers.deinit();
}
};
@@ -1011,7 +1002,7 @@ pub const Response = struct {
pub fn status(self: Response) ?u16 {
return switch (self.inner) {
.transfer => |t| if (t.response_header) |rh| rh.status else null,
.transfer => |t| if (t.res.header) |rh| rh.status else null,
.cached => |c| c.metadata.status,
.fulfilled => |f| f.status,
};
@@ -1019,7 +1010,7 @@ pub const Response = struct {
pub fn contentType(self: Response) ?[]const u8 {
return switch (self.inner) {
.transfer => |t| if (t.response_header) |*rh| rh.contentType() else null,
.transfer => |t| if (t.res.header) |*rh| rh.contentType() else null,
.cached => |c| c.metadata.content_type,
.fulfilled => |f| f.contentType(),
};
@@ -1038,14 +1029,14 @@ pub const Response = struct {
pub fn redirectCount(self: Response) ?u32 {
return switch (self.inner) {
.transfer => |t| if (t.response_header) |rh| rh.redirect_count else null,
.transfer => |t| if (t.res.header) |rh| rh.redirect_count else null,
.cached, .fulfilled => 0,
};
}
pub fn url(self: Response) [:0]const u8 {
return switch (self.inner) {
.transfer => |t| t.url,
.transfer => |t| t.req.url,
.cached => |c| c.metadata.url,
.fulfilled => |f| f.url,
};
@@ -1107,21 +1098,12 @@ pub const Transfer = struct {
_queued: bool = false,
req: Request,
url: [:0]const u8,
res: Transfer.Response = .{},
client: *Client,
// total bytes received in the response, including the response status line,
// the headers, and the [encoded] body.
bytes_received: usize = 0,
start_time: u64,
aborted: bool = false,
// We'll store the response header here
response_header: ?ResponseHead = null,
// track if the header callbacks done have been called.
_header_done_called: bool = false,
_notified_fail: bool = false,
_conn: ?*http.Connection = null,
@@ -1137,14 +1119,6 @@ pub const Transfer = struct {
_tries: u8 = 0,
_performing: bool = false,
_redirect_count: u8 = 0,
_skip_body: bool = false,
_first_data_received: bool = false,
// Buffered response body. Filled by dataCallback, consumed in processMessages.
_stream_buffer: std.ArrayList(u8) = .{},
// Error captured in dataCallback to be reported in processMessages.
_callback_error: ?anyerror = null,
// for when a Transfer is queued in the client.queue
_node: std.DoublyLinkedList.Node = .{},
@@ -1297,15 +1271,15 @@ pub const Transfer = struct {
try conn.setProxy(client.http_proxy);
try conn.setTlsVerify(client.tls_verify, client.use_proxy);
try conn.setURL(req.params.url);
try conn.setMethod(req.params.method);
if (req.params.body) |b| {
try conn.setURL(req.url);
try conn.setMethod(req.method);
if (req.body) |b| {
try conn.setBody(b);
} else {
try conn.setGetMode();
}
var header_list = req.params.headers;
var header_list = req.headers;
try conn.secretHeaders(&header_list, &client.network.config.http_headers);
try conn.setHeaders(&header_list);
@@ -1317,12 +1291,12 @@ pub const Transfer = struct {
conn.transport = .{ .http = self };
// Per-request timeout override (e.g. XHR timeout)
if (req.params.timeout_ms > 0) {
try conn.setTimeout(req.params.timeout_ms);
if (req.timeout_ms > 0) {
try conn.setTimeout(req.timeout_ms);
}
// add credentials
if (req.params.credentials) |creds| {
if (req.credentials) |creds| {
if (self._auth_challenge != null and self._auth_challenge.?.source == .proxy) {
try conn.setProxyCredentials(creds);
} else {
@@ -1332,21 +1306,19 @@ pub const Transfer = struct {
}
pub fn reset(self: *Transfer) void {
// Note: do NOT reset _auth_challenge here. It is needed by makeRequest
// to determine whether to use setProxyCredentials vs setCredentials.
// Note: do NOT reset _auth_challenge or _redirect_count here. They
// span retries — _auth_challenge tells makeRequest whether to use
// setProxyCredentials vs setCredentials; _redirect_count caps the
// total hops. The rest of the response state is per-attempt.
self._notified_fail = false;
self.response_header = null;
self.bytes_received = 0;
self._tries += 1;
self._stream_buffer.clearRetainingCapacity();
self._callback_error = null;
self._skip_body = false;
self._first_data_received = false;
self.res.stream_buffer.clearRetainingCapacity();
self.res = .{ .stream_buffer = self.res.stream_buffer };
}
fn buildResponseHeader(self: *Transfer, conn: *const http.Connection) !void {
if (comptime IS_DEBUG) {
std.debug.assert(self.response_header == null);
std.debug.assert(self.res.header == null);
}
const url = try conn.getEffectiveUrl();
@@ -1356,14 +1328,14 @@ pub const Transfer = struct {
else
try conn.getResponseCode();
self.response_header = .{
self.res.header = .{
.url = url,
.status = status,
.redirect_count = self._redirect_count,
};
if (conn.getResponseHeader("content-type", 0)) |ct| {
var hdr = &self.response_header.?;
var hdr = &self.res.header.?;
const value = ct.value;
const len = @min(value.len, ResponseHead.MAX_CONTENT_TYPE_LEN);
hdr._content_type_len = len;
@@ -1373,15 +1345,11 @@ pub const Transfer = struct {
pub fn format(self: *Transfer, writer: *std.Io.Writer) !void {
const req = self.req;
return writer.print("{s} {s}", .{ @tagName(req.params.method), req.params.url });
return writer.print("{s} {s}", .{ @tagName(req.method), req.url });
}
pub fn updateURL(self: *Transfer, url: [:0]const u8) !void {
// for cookies
self.url = url;
// for the request itself
self.req.params.url = url;
self.req.url = url;
}
fn handleRedirect(transfer: *Transfer) !void {
@@ -1395,10 +1363,10 @@ pub const Transfer = struct {
}
// retrieve cookies from the redirect's response.
if (req.params.cookie_jar) |jar| {
if (req.cookie_jar) |jar| {
var i: usize = 0;
while (conn.getResponseHeader("set-cookie", i)) |ct| : (i += 1) {
try jar.populateFromResponse(transfer.url, ct.value);
try jar.populateFromResponse(transfer.req.url, ct.value);
if (i >= ct.amount) {
break;
@@ -1426,7 +1394,7 @@ pub const Transfer = struct {
// URL.resolve follows RFC 3986 §5.3, which drops the base fragment when
// the relative ref has none, so we re-attach it here.
if (URL.getHash(resolved).len == 0) {
const original_hash = URL.getHash(transfer.url);
const original_hash = URL.getHash(transfer.req.url);
if (original_hash.len != 0) {
break :blk try std.mem.joinZ(arena, "", &.{ resolved, original_hash });
}
@@ -1439,8 +1407,8 @@ pub const Transfer = struct {
// 307, 308 → keep method and body.
const status = try conn.getResponseCode();
if (status == 301 or status == 302 or status == 303) {
req.params.method = .GET;
req.params.body = null;
req.method = .GET;
req.body = null;
}
}
@@ -1467,11 +1435,11 @@ pub const Transfer = struct {
}
pub fn updateCredentials(self: *Transfer, userpwd: [:0]const u8) void {
self.req.params.credentials = userpwd;
self.req.credentials = userpwd;
}
pub fn replaceRequestHeaders(self: *Transfer, allocator: Allocator, headers: []const http.Header) !void {
self.req.params.headers.deinit();
self.req.headers.deinit();
var buf: std.ArrayList(u8) = .empty;
var new_headers = try self.client.newHeaders();
@@ -1483,7 +1451,7 @@ pub const Transfer = struct {
try buf.append(allocator, 0); // null terminated
try new_headers.add(buf.items[0 .. buf.items.len - 1 :0]);
}
self.req.params.headers = new_headers;
self.req.headers = new_headers;
}
// abortAuthChallenge is called when an auth challenge interception is
@@ -1503,17 +1471,17 @@ pub const Transfer = struct {
// It can be called either on dataCallback or once the request for those
// w/o body.
fn headerDoneCallback(transfer: *Transfer, conn: *const http.Connection) !bool {
lp.assert(transfer._header_done_called == false, "Transfer.headerDoneCallback", .{});
defer transfer._header_done_called = true;
lp.assert(transfer.res.header_done_called == false, "Transfer.headerDoneCallback", .{});
defer transfer.res.header_done_called = true;
try transfer.buildResponseHeader(conn);
if (transfer.req.params.cookie_jar) |jar| {
if (transfer.req.cookie_jar) |jar| {
var i: usize = 0;
while (true) {
const ct = conn.getResponseHeader("set-cookie", i);
if (ct == null) break;
jar.populateFromResponse(transfer.url, ct.?.value) catch |err| {
jar.populateFromResponse(transfer.req.url, ct.?.value) catch |err| {
log.err(.http, "set cookie", .{ .err = err, .req = transfer });
return err;
};
@@ -1528,7 +1496,7 @@ pub const Transfer = struct {
}
}
const proceed = transfer.req.header_callback(Response.fromTransfer(transfer)) catch |err| {
const proceed = transfer.req.header_callback(Client.Response.fromTransfer(transfer)) catch |err| {
log.err(.http, "header_callback", .{ .err = err, .req = transfer });
return err;
};
@@ -1544,9 +1512,10 @@ pub const Transfer = struct {
const conn: *http.Connection = @ptrCast(@alignCast(data));
var transfer = conn.transport.http;
const res = &transfer.res;
if (!transfer._first_data_received) {
transfer._first_data_received = true;
if (!res.first_data_received) {
res.first_data_received = true;
// Skip body for responses that will be retried (redirects, auth challenges).
const status = conn.getResponseCode() catch |err| {
@@ -1554,31 +1523,31 @@ pub const Transfer = struct {
return http.writefunc_error;
};
if ((status >= 300 and status <= 399) or status == 401 or status == 407) {
transfer._skip_body = true;
res.skip_body = true;
return @intCast(chunk_len);
}
// Pre-size buffer from Content-Length.
if (transfer.getContentLength()) |cl| {
if (cl > transfer.client.max_response_size) {
transfer._callback_error = error.ResponseTooLarge;
res.callback_error = error.ResponseTooLarge;
return http.writefunc_error;
}
transfer._stream_buffer.ensureTotalCapacity(transfer.arena, cl) catch {};
res.stream_buffer.ensureTotalCapacity(transfer.arena, cl) catch {};
}
}
if (transfer._skip_body) return @intCast(chunk_len);
if (res.skip_body) return @intCast(chunk_len);
transfer.bytes_received += chunk_len;
if (transfer.bytes_received > transfer.client.max_response_size) {
transfer._callback_error = error.ResponseTooLarge;
res.bytes_received += chunk_len;
if (res.bytes_received > transfer.client.max_response_size) {
res.callback_error = error.ResponseTooLarge;
return http.writefunc_error;
}
const chunk = buffer[0..chunk_len];
transfer._stream_buffer.appendSlice(transfer.arena, chunk) catch |err| {
transfer._callback_error = err;
res.stream_buffer.appendSlice(transfer.arena, chunk) catch |err| {
res.callback_error = err;
return http.writefunc_error;
};
@@ -1618,7 +1587,7 @@ pub const Transfer = struct {
// doneCallback. OR, maybe this is a "fulfilled" request. Let's check
// the injected headers (if we have any).
const rh = self.response_header orelse return null;
const rh = self.res.header orelse return null;
for (rh._injected_headers) |hdr| {
if (std.ascii.eqlIgnoreCase(hdr.name, "content-length")) {
return hdr.value;
@@ -1627,6 +1596,32 @@ pub const Transfer = struct {
return null;
}
// Response-state owned by this transfer's currently-in-flight response.
// Reset on every retry (auth retry, redirect) via Transfer.reset — only
// the cross-retry counters (_auth_challenge, _redirect_count) live on
// Transfer itself. `Transfer.Response` is the on-Transfer storage; the
// top-level `Client.Response` is the actual Response (which is a union, e.g.
// for a cached response)
const Response = struct {
header: ?ResponseHead = null,
// total bytes received in the response, including the response status
// line, the headers, and the [encoded] body.
bytes_received: usize = 0,
// track if the header callbacks done have been called.
header_done_called: bool = false,
skip_body: bool = false,
first_data_received: bool = false,
// Buffered response body. Filled by dataCallback, consumed in processMessages.
stream_buffer: std.ArrayList(u8) = .{},
// Error captured in dataCallback to be reported in processMessages.
callback_error: ?anyerror = null,
};
};
pub fn continueTransfer(self: *Client, transfer: *Transfer) !void {

20
src/browser/ScriptManager.zig
View File

@@ -258,17 +258,15 @@ pub fn addFromElement(self: *ScriptManager, comptime from_parser: bool, script_e
try frame.makeRequest(.{
.ctx = script,
.params = .{
.url = url,
.method = .GET,
.frame_id = frame._frame_id,
.loader_id = frame._loader_id,
.headers = headers,
.cookie_jar = &frame._session.cookie_jar,
.cookie_origin = frame.url,
.resource_type = .script,
.notification = frame._session.notification,
},
.url = url,
.method = .GET,
.frame_id = frame._frame_id,
.loader_id = frame._loader_id,
.headers = headers,
.cookie_jar = &frame._session.cookie_jar,
.cookie_origin = frame.url,
.resource_type = .script,
.notification = frame._session.notification,
.start_callback = if (log.enabled(.http, .debug)) Script.startCallback else null,
.header_callback = Script.headerCallback,
.data_callback = Script.dataCallback,

44
src/browser/ScriptManagerBase.zig
View File

@@ -265,17 +265,15 @@ pub fn preloadImport(self: *ScriptManagerBase, url: [:0]const u8, referrer: []co
const session = owner.session();
owner.makeRequest(.{
.ctx = script,
.params = .{
.url = url,
.method = .GET,
.frame_id = owner.frameId(),
.loader_id = owner.loaderId(),
.headers = try self.getHeaders(),
.cookie_jar = &session.cookie_jar,
.cookie_origin = owner.url(),
.resource_type = .script,
.notification = session.notification,
},
.url = url,
.method = .GET,
.frame_id = owner.frameId(),
.loader_id = owner.loaderId(),
.headers = try self.getHeaders(),
.cookie_jar = &session.cookie_jar,
.cookie_origin = owner.url(),
.resource_type = .script,
.notification = session.notification,
.start_callback = if (log.enabled(.http, .debug)) Script.startCallback else null,
.header_callback = Script.headerCallback,
.data_callback = Script.dataCallback,
@@ -372,17 +370,15 @@ pub fn getAsyncImport(self: *ScriptManagerBase, url: [:0]const u8, cb: ImportAsy
self.async_scripts.append(&script.node);
owner.makeRequest(.{
.ctx = script,
.params = .{
.url = url,
.method = .GET,
.frame_id = owner.frameId(),
.loader_id = owner.loaderId(),
.headers = try self.getHeaders(),
.resource_type = .script,
.cookie_jar = &session.cookie_jar,
.cookie_origin = owner.url(),
.notification = session.notification,
},
.url = url,
.method = .GET,
.frame_id = owner.frameId(),
.loader_id = owner.loaderId(),
.headers = try self.getHeaders(),
.resource_type = .script,
.cookie_jar = &session.cookie_jar,
.cookie_origin = owner.url(),
.notification = session.notification,
.start_callback = if (log.enabled(.http, .debug)) Script.startCallback else null,
.header_callback = Script.headerCallback,
.data_callback = Script.dataCallback,
@@ -581,7 +577,7 @@ pub const Script = struct {
.b1 = transfer.id,
.b2 = transfer._tries,
.b3 = transfer.aborted,
.b4 = transfer.bytes_received,
.b4 = transfer.res.bytes_received,
.b5 = transfer._notified_fail,
.b8 = transfer._auth_challenge != null,
.b9 = if (transfer._conn) |c| @intFromPtr(c._easy) else 0,
@@ -590,7 +586,7 @@ pub const Script = struct {
self.debug_transfer_id = transfer.id;
self.debug_transfer_tries = transfer._tries;
self.debug_transfer_aborted = transfer.aborted;
self.debug_transfer_bytes_received = transfer.bytes_received;
self.debug_transfer_bytes_received = transfer.res.bytes_received;
self.debug_transfer_notified_fail = transfer._notified_fail;
self.debug_transfer_auth_challenge = transfer._auth_challenge != null;
self.debug_transfer_easy_id = if (transfer._conn) |c| @intFromPtr(c._easy) else 0;

20
src/browser/webapi/Worker.zig
View File

@@ -102,17 +102,15 @@ pub fn init(url: []const u8, frame: *Frame) !*Worker {
const headers = try session.browser.http_client.newHeaders();
frame.makeRequest(.{
.ctx = self,
.params = .{
.method = .GET,
.headers = headers,
.url = resolved_url,
.frame_id = self._frame_id,
.loader_id = self._loader_id,
.resource_type = .script,
.cookie_jar = &session.cookie_jar,
.cookie_origin = resolved_url,
.notification = session.notification,
},
.method = .GET,
.headers = headers,
.url = resolved_url,
.frame_id = self._frame_id,
.loader_id = self._loader_id,
.resource_type = .script,
.cookie_jar = &session.cookie_jar,
.cookie_origin = resolved_url,
.notification = session.notification,
.header_callback = httpHeaderCallback,
.data_callback = httpDataCallback,
.done_callback = httpDoneCallback,

22
src/browser/webapi/net/Fetch.zig
View File

@@ -101,18 +101,16 @@ pub fn init(input: Input, options: ?InitOpts, exec: *const Execution) !js.Promis
// fire the `errdefer response.deinit` above and double-free the arena.
exec.makeRequest(.{
.ctx = fetch,
.params = .{
.url = request._url,
.method = request._method,
.frame_id = exec.frameId(),
.loader_id = exec.loaderId(),
.body = request._body,
.headers = headers,
.resource_type = .fetch,
.cookie_jar = cookie_jar,
.cookie_origin = exec.url.*,
.notification = session.notification,
},
.url = request._url,
.method = request._method,
.frame_id = exec.frameId(),
.loader_id = exec.loaderId(),
.body = request._body,
.headers = headers,
.resource_type = .fetch,
.cookie_jar = cookie_jar,
.cookie_origin = exec.url.*,
.notification = session.notification,
.start_callback = httpStartCallback,
.header_callback = httpHeaderDoneCallback,
.data_callback = httpDataCallback,

24
src/browser/webapi/net/XMLHttpRequest.zig
View File

@@ -270,19 +270,17 @@ pub fn send(self: *XMLHttpRequest, body_: ?BodyInit, exec_: *const Execution) !v
exec.makeRequest(.{
.ctx = self,
.params = .{
.url = self._url,
.method = self._method,
.headers = headers,
.frame_id = exec.frameId(),
.loader_id = exec.loaderId(),
.body = self._request_body,
.cookie_jar = if (cookie_support) &session.cookie_jar else null,
.cookie_origin = exec.url.*,
.resource_type = .xhr,
.timeout_ms = self._timeout,
.notification = session.notification,
},
.url = self._url,
.method = self._method,
.headers = headers,
.frame_id = exec.frameId(),
.loader_id = exec.loaderId(),
.body = self._request_body,
.cookie_jar = if (cookie_support) &session.cookie_jar else null,
.cookie_origin = exec.url.*,
.resource_type = .xhr,
.timeout_ms = self._timeout,
.notification = session.notification,
.start_callback = httpStartCallback,
.header_callback = httpHeaderDoneCallback,
.data_callback = httpDataCallback,

4
src/cdp/CDP.zig
View File

@@ -796,8 +796,8 @@ pub const BrowserContext = struct {
}
fn keyFromTransfer(transfer: *const Transfer) CDP.BrowserContext.CapturedResponseKey {
return if (transfer.req.params.resource_type == .document)
.{ .kind = .loader, .id = transfer.req.params.loader_id }
return if (transfer.req.resource_type == .document)
.{ .kind = .loader, .id = transfer.req.loader_id }
else
.{ .kind = .request, .id = transfer.id };
}

28
src/cdp/domains/fetch.zig
View File

@@ -202,9 +202,9 @@ pub fn requestIntercept(bc: *CDP.BrowserContext, intercept: *const Notification.
try bc.cdp.sendEvent("Fetch.requestPaused", .{
.requestId = &id.toInterceptId(transfer.id),
.frameId = &id.toFrameId(transfer.req.params.frame_id),
.frameId = &id.toFrameId(transfer.req.frame_id),
.request = network.RequestWriter.init(transfer),
.resourceType = switch (transfer.req.params.resource_type) {
.resourceType = switch (transfer.req.resource_type) {
.script => "Script",
.xhr => "XHR",
.document => "Document",
@@ -216,7 +216,7 @@ pub fn requestIntercept(bc: *CDP.BrowserContext, intercept: *const Notification.
log.debug(.cdp, "request intercept", .{
.state = "paused",
.id = transfer.id,
.url = transfer.url,
.url = transfer.req.url,
});
// Await either continueRequest, failRequest or fulfillRequest
@@ -254,7 +254,7 @@ fn continueRequest(cmd: *CDP.Command) !void {
log.debug(.cdp, "request intercept", .{
.state = "continue",
.id = transfer.id,
.url = transfer.url,
.url = transfer.req.url,
.new_url = params.url,
});
@@ -262,14 +262,14 @@ fn continueRequest(cmd: *CDP.Command) !void {
const request = &transfer.req;
// Update the request with the new parameters
if (params.url) |url| {
request.params.url = try arena.dupeZ(u8, url);
request.url = try arena.dupeZ(u8, url);
}
if (params.method) |method| {
request.params.method = std.meta.stringToEnum(http.Method, method) orelse return error.InvalidParams;
request.method = std.meta.stringToEnum(http.Method, method) orelse return error.InvalidParams;
}
if (params.headers) |headers| {
request.params.headers.deinit();
request.headers.deinit();
var buf: std.ArrayList(u8) = .empty;
var new_headers = try bc.cdp.browser.http_client.newHeaders();
@@ -279,14 +279,14 @@ fn continueRequest(cmd: *CDP.Command) !void {
try buf.append(cmd.arena, 0);
try new_headers.add(buf.items[0 .. buf.items.len - 1 :0]);
}
request.params.headers = new_headers;
request.headers = new_headers;
}
if (params.postData) |b| {
const decoder = std.base64.standard.Decoder;
const body = try arena.alloc(u8, try decoder.calcSizeForSlice(b));
try decoder.decode(body, b);
request.params.body = body;
request.body = body;
}
try client.interception_layer.continueRequest(transfer);
@@ -384,7 +384,7 @@ fn fulfillRequest(cmd: *CDP.Command) !void {
log.debug(.cdp, "request intercept", .{
.state = "fulfilled",
.id = transfer.id,
.url = transfer.url,
.url = transfer.req.url,
.status = params.responseCode,
.body = params.body != null,
});
@@ -423,7 +423,7 @@ fn failRequest(cmd: *CDP.Command) !void {
log.info(.cdp, "request intercept", .{
.state = "fail",
.id = transfer.id,
.url = transfer.url,
.url = transfer.req.url,
.reason = params.errorReason,
});
return cmd.sendResult(null, .{});
@@ -446,9 +446,9 @@ pub fn requestAuthRequired(bc: *CDP.BrowserContext, intercept: *const Notificati
try bc.cdp.sendEvent("Fetch.authRequired", .{
.requestId = &id.toInterceptId(transfer.id),
.frameId = &id.toFrameId(request.params.frame_id),
.frameId = &id.toFrameId(request.frame_id),
.request = network.RequestWriter.init(transfer),
.resourceType = switch (request.params.resource_type) {
.resourceType = switch (request.resource_type) {
.script => "Script",
.xhr => "XHR",
.document => "Document",
@@ -466,7 +466,7 @@ pub fn requestAuthRequired(bc: *CDP.BrowserContext, intercept: *const Notificati
log.debug(.cdp, "request auth required", .{
.state = "paused",
.id = transfer.id,
.url = transfer.url,
.url = transfer.req.url,
});
// Await continueWithAuth

22
src/cdp/domains/network.zig
View File

@@ -308,20 +308,20 @@ pub fn httpRequestStart(bc: *CDP.BrowserContext, msg: *const Notification.Reques
const transfer = msg.transfer;
const req = &transfer.req;
const frame_id = req.params.frame_id;
const frame_id = req.frame_id;
const frame = bc.session.findFrameByFrameId(frame_id) orelse return;
// Modify request with extra CDP headers
for (bc.extra_headers.items) |extra| {
try req.params.headers.add(extra);
try req.headers.add(extra);
}
// We're missing a bunch of fields, but, for now, this eems like enough
try bc.cdp.sendEvent("Network.requestWillBeSent", .{
.frameId = &id.toFrameId(frame_id),
.requestId = &id.toRequestId(transfer),
.loaderId = &id.toLoaderId(req.params.loader_id),
.type = req.params.resource_type.string(),
.loaderId = &id.toLoaderId(req.loader_id),
.type = req.resource_type.string(),
.documentURL = frame.url,
.request = RequestWriter.init(transfer),
.initiator = .{ .type = "other" },
@@ -342,9 +342,9 @@ pub fn httpResponseHeaderDone(arena: Allocator, bc: *CDP.BrowserContext, msg: *c
// We're missing a bunch of fields, but, for now, this seems like enough
try bc.cdp.sendEvent("Network.responseReceived", .{
.frameId = &id.toFrameId(req.params.frame_id),
.frameId = &id.toFrameId(req.frame_id),
.requestId = &id.toRequestId(transfer),
.loaderId = &id.toLoaderId(req.params.loader_id),
.loaderId = &id.toLoaderId(req.loader_id),
.response = ResponseWriter.init(arena, msg.response),
.hasExtraInfo = false, // TODO change after adding Network.responseReceivedExtraInfo
}, .{ .session_id = session_id });
@@ -389,11 +389,11 @@ pub const RequestWriter = struct {
try jws.beginObject();
{
try jws.objectField("url");
try jws.write(request.params.url);
try jws.write(request.url);
}
{
const frag = URL.getHash(request.params.url);
const frag = URL.getHash(request.url);
if (frag.len > 0) {
try jws.objectField("urlFragment");
try jws.write(frag);
@@ -402,18 +402,18 @@ pub const RequestWriter = struct {
{
try jws.objectField("method");
try jws.write(@tagName(request.params.method));
try jws.write(@tagName(request.method));
}
{
try jws.objectField("hasPostData");
try jws.write(request.params.body != null);
try jws.write(request.body != null);
}
{
try jws.objectField("headers");
try jws.beginObject();
var it = request.params.headers.iterator();
var it = request.headers.iterator();
while (it.next()) |hdr| {
try jws.objectField(hdr.name);
try jws.write(hdr.value);

4
src/cdp/id.zig
View File

@@ -42,8 +42,8 @@ pub fn toLoaderId(id: u32) [14]u8 {
// then it should match the loader id.
const Transfer = @import("../browser/HttpClient.zig").Transfer;
pub fn toRequestId(transfer: *const Transfer) [14]u8 {
if (transfer.req.params.resource_type == .document) {
return toLoaderId(transfer.req.params.loader_id);
if (transfer.req.resource_type == .document) {
return toLoaderId(transfer.req.loader_id);
}
var buf: [14]u8 = undefined;

24
src/network/layer/CacheLayer.zig
View File

@@ -32,6 +32,7 @@ const CachedResponse = @import("../cache/Cache.zig").CachedResponse;
const Forward = @import("Forward.zig");
const log = lp.log;
const IS_DEBUG = @import("builtin").mode == .Debug;
const CacheLayer = @This();
@@ -51,22 +52,22 @@ fn request(ptr: *anyopaque, transfer: *Transfer) anyerror!void {
const self: *CacheLayer = @ptrCast(@alignCast(ptr));
const req = &transfer.req;
if (self.disabled or req.params.method != .GET) {
if (self.disabled or req.method != .GET) {
return self.next.request(transfer);
}
const arena = transfer.arena;
var iter = req.params.headers.iterator();
var iter = req.headers.iterator();
const req_header_list = try iter.collect(arena);
if (transfer.client.network.cache.?.get(arena, .{
.url = transfer.url,
.url = req.url,
.timestamp = std.time.timestamp(),
.request_headers = req_header_list.items,
})) |cached| {
// Dispatch that the Request was served from the Cache.
transfer.req.params.notification.dispatch(
transfer.req.notification.dispatch(
.http_request_served_from_cache,
&.{ .transfer = transfer },
);
@@ -86,8 +87,8 @@ fn request(ptr: *anyopaque, transfer: *Transfer) anyerror!void {
.arena = arena,
.transfer = transfer,
.forward = Forward.capture(req),
.req_url = transfer.url,
.req_headers = req.params.headers,
.req_url = req.url,
.req_headers = req.headers,
};
req.ctx = ctx;
@@ -185,11 +186,11 @@ const CacheContext = struct {
const conn = transfer._conn.?;
const vary = if (conn.getResponseHeader("vary", 0)) |h| h.value else null;
var rh = &transfer.response_header.?;
var rh = &transfer.res.header.?;
const maybe_cm = try Cache.tryCache(
arena,
std.time.timestamp(),
transfer.url,
self.req_url,
rh.status,
rh.contentType(),
if (conn.getResponseHeader("cache-control", 0)) |h| h.value else null,
@@ -237,11 +238,12 @@ const CacheContext = struct {
if (self.pending_metadata) |metadata| {
const cache = &transfer.client.network.cache.?;
log.debug(.browser, "http cache", .{ .key = self.req_url, .metadata = metadata });
cache.put(metadata.*, transfer._stream_buffer.items) catch |err| {
if (comptime IS_DEBUG) {
log.debug(.browser, "http cache", .{ .key = self.req_url, .metadata = metadata });
}
cache.put(metadata.*, transfer.res.stream_buffer.items) catch |err| {
log.warn(.http, "cache put failed", .{ .err = err });
};
log.debug(.browser, "http.cache.put", .{ .url = self.req_url });
}
return self.forward.forwardDone();

30
src/network/layer/InterceptionLayer.zig
View File

@@ -72,10 +72,10 @@ fn request(ptr: *anyopaque, transfer: *Transfer) anyerror!void {
req.error_callback = InterceptContext.errorCallback;
if (ctx.forward.shutdown != null) req.shutdown_callback = InterceptContext.shutdownCallback;
req.params.notification.dispatch(.http_request_start, &.{ .transfer = transfer });
req.notification.dispatch(.http_request_start, &.{ .transfer = transfer });
var wait_for_interception = false;
req.params.notification.dispatch(.http_request_intercept, &.{
req.notification.dispatch(.http_request_intercept, &.{
.transfer = transfer,
.wait_for_interception = &wait_for_interception,
});
@@ -83,7 +83,7 @@ fn request(ptr: *anyopaque, transfer: *Transfer) anyerror!void {
log.debug(.http, "interception check", .{
.wait_for_interception = wait_for_interception,
.intercepted = self.intercepted,
.url = req.params.url,
.url = req.url,
});
if (!wait_for_interception) {
@@ -109,21 +109,21 @@ pub const InterceptContext = struct {
fn startCallback(response: Response) anyerror!void {
const self: *InterceptContext = @ptrCast(@alignCast(response.ctx));
log.debug(.http, "intercept start", .{ .url = self.transfer.url });
log.debug(.http, "intercept start", .{ .url = self.transfer.req.url });
return self.forward.forwardStart(response);
}
fn headerCallback(response: Response) anyerror!bool {
const self: *InterceptContext = @ptrCast(@alignCast(response.ctx));
log.debug(.http, "intercept header", .{
.url = self.transfer.url,
.url = self.transfer.req.url,
.status = response.status(),
.content_length = response.contentLength(),
});
self.content_length = response.contentLength() orelse 0;
self.transfer.req.params.notification.dispatch(.http_response_header_done, &.{
self.transfer.req.notification.dispatch(.http_response_header_done, &.{
.transfer = self.transfer,
.response = &response,
});
@@ -134,11 +134,11 @@ pub const InterceptContext = struct {
fn dataCallback(response: Response, chunk: []const u8) anyerror!void {
const self: *InterceptContext = @ptrCast(@alignCast(response.ctx));
log.debug(.http, "intercept data", .{
.url = self.transfer.url,
.url = self.transfer.req.url,
.len = chunk.len,
});
self.transfer.req.params.notification.dispatch(.http_response_data, &.{
self.transfer.req.notification.dispatch(.http_response_data, &.{
.data = chunk,
.transfer = self.transfer,
});
@@ -150,11 +150,11 @@ pub const InterceptContext = struct {
const self: *InterceptContext = @ptrCast(@alignCast(ctx));
log.debug(.http, "intercept done", .{
.url = self.transfer.url,
.url = self.transfer.req.url,
.content_length = self.content_length,
});
self.transfer.req.params.notification.dispatch(.http_request_done, &.{
self.transfer.req.notification.dispatch(.http_request_done, &.{
.transfer = self.transfer,
.content_length = self.content_length,
});
@@ -165,10 +165,10 @@ pub const InterceptContext = struct {
const self: *InterceptContext = @ptrCast(@alignCast(ctx));
log.debug(.http, "intercept error", .{
.url = self.transfer.url,
.url = self.transfer.req.url,
.err = err,
});
self.transfer.req.params.notification.dispatch(.http_request_fail, &.{
self.transfer.req.notification.dispatch(.http_request_fail, &.{
.transfer = self.transfer,
.err = err,
});
@@ -178,8 +178,8 @@ pub const InterceptContext = struct {
fn shutdownCallback(ctx: *anyopaque) void {
const self: *InterceptContext = @ptrCast(@alignCast(ctx));
log.debug(.http, "intercept shutdown", .{ .url = self.transfer.url });
self.transfer.req.params.notification.dispatch(.http_request_fail, &.{
log.debug(.http, "intercept shutdown", .{ .url = self.transfer.req.url });
self.transfer.req.notification.dispatch(.http_request_fail, &.{
.transfer = self.transfer,
.err = error.Shutdown,
});
@@ -257,7 +257,7 @@ fn fulfillInner(
) !void {
const fulfilled = FulfilledResponse{
.status = status,
.url = req.params.url,
.url = req.url,
.headers = headers,
.body = body,
};

59
src/network/layer/RobotsLayer.zig
View File

@@ -60,11 +60,11 @@ pub fn deinit(self: *RobotsLayer, allocator: Allocator) void {
fn request(ptr: *anyopaque, transfer: *Transfer) anyerror!void {
const self: *RobotsLayer = @ptrCast(@alignCast(ptr));
if (transfer.req.params.skip_robots) {
if (transfer.req.skip_robots) {
return self.next.request(transfer);
}
const url = transfer.url;
const url = transfer.req.url;
const robots_url = try URL.getRobotsUrl(transfer.arena, url);
if (self.network.robot_store.get(robots_url)) |robot_entry| {
@@ -111,37 +111,30 @@ fn fetchRobotsThenRequest(
.robots_url = robots_url,
};
var params = transfer.req.params;
if (@typeInfo(@TypeOf(params)) != .@"struct") {
// protect against mutating the original request
@compileError("expected request.params to be a struct");
}
// CRITICAL: build a fresh Headers for the inner robots fetch.
// params is value-copied from the parent's req.params, but
// Headers is a struct wrapping a *curl_slist — value copy shares
// the pointer. Letting Client.request take ownership of a shared
// headers list means both transfers will free it at deinit time
// -> double-free. The robots.txt fetch is a system-level GET
// anyway, no need to inherit the parent's user headers.
params.headers = try transfer.client.newHeaders();
errdefer params.headers.deinit();
params.method = .GET;
params.url = robots_url;
params.skip_robots = true;
params.resource_type = .fetch;
params.body = null;
// We value-copy req from the parent, but Headers is a struct wrapping
// a *curl_slist — value copy shares the pointer. Letting Client.request
// take ownership of a shared headers list means both transfers will
// free it at deinit time -> double-free. The robots.txt fetch is a
// system-level GET anyway, no need to inherit the parent's user headers.
var new_req = transfer.req;
new_req.headers = try transfer.client.newHeaders();
errdefer new_req.headers.deinit();
new_req.method = .GET;
new_req.url = robots_url;
new_req.skip_robots = true;
new_req.resource_type = .fetch;
new_req.body = null;
new_req.ctx = robots_ctx;
new_req.start_callback = null;
new_req.header_callback = RobotsContext.headerCallback;
new_req.data_callback = RobotsContext.dataCallback;
new_req.done_callback = RobotsContext.doneCallback;
new_req.error_callback = RobotsContext.errorCallback;
new_req.shutdown_callback = RobotsContext.shutdownCallback;
log.debug(.browser, "fetching robots.txt", .{ .robots_url = robots_url });
try transfer.client.request(.{
.ctx = robots_ctx,
.params = params,
.header_callback = RobotsContext.headerCallback,
.data_callback = RobotsContext.dataCallback,
.done_callback = RobotsContext.doneCallback,
.error_callback = RobotsContext.errorCallback,
.shutdown_callback = RobotsContext.shutdownCallback,
}, transfer.owner);
try transfer.client.request(new_req, transfer.owner);
} else {
// Already one in flight, just queue behind.
try entry.value_ptr.append(self.allocator, transfer);
@@ -160,7 +153,7 @@ fn flushPending(self: *RobotsLayer, robots_url: [:0]const u8, allowed: bool) voi
for (queued.value.items) |transfer| {
if (!allowed) {
log.warn(.http, "blocked by robots", .{ .url = transfer.url });
log.warn(.http, "blocked by robots", .{ .url = transfer.req.url });
transfer.abort(error.RobotsBlocked);
} else {
// Reset ownership: handing back to the layer chain. If a downstream
@@ -207,7 +200,7 @@ const RobotsContext = struct {
const self: *RobotsContext = @ptrCast(@alignCast(response.ctx));
switch (response.inner) {
.transfer => |t| {
if (t.response_header) |hdr| {
if (t.res.header) |hdr| {
log.debug(.browser, "robots status", .{ .status = hdr.status, .robots_url = self.robots_url });
self.status = hdr.status;
}
@@ -246,7 +239,7 @@ const RobotsContext = struct {
};
if (robots) |r| {
try network.robot_store.put(robots_url, r);
const path = URL.getPathname(l.pending.get(robots_url).?.items[0].req.params.url);
const path = URL.getPathname(l.pending.get(robots_url).?.items[0].req.url);
allowed = r.isAllowed(path);
}
}

4
src/network/layer/WebBotAuthLayer.zig
View File

@@ -44,8 +44,8 @@ fn request(ptr: *anyopaque, transfer: *Transfer) anyerror!void {
const wba = transfer.client.network.web_bot_auth orelse @panic("WebBotAuthLayer shouldn't be active without WebBotAuth");
const authority = URL.getHost(transfer.url);
try wba.signRequest(transfer.arena, &transfer.req.params.headers, authority);
const authority = URL.getHost(transfer.req.url);
try wba.signRequest(transfer.arena, &transfer.req.headers, authority);
return self.next.request(transfer);
}