diff --git a/src/Notification.zig b/src/Notification.zig index 96b8f822..f429e7b9 100644 --- a/src/Notification.zig +++ b/src/Notification.zig @@ -174,8 +174,7 @@ pub const RequestIntercept = struct { }; pub const RequestAuthRequired = struct { - request: *Request, - intercept_ctx: *InterceptContext, + transfer: *Transfer, wait_for_interception: *bool, }; diff --git a/src/browser/HttpClient.zig b/src/browser/HttpClient.zig index 042b5406..7600ad5a 100644 --- a/src/browser/HttpClient.zig +++ b/src/browser/HttpClient.zig @@ -639,6 +639,33 @@ fn perform(self: *Client, timeout_ms: c_int) anyerror!PerformStatus { } fn processOneMessage(self: *Client, msg: http.Handles.MultiMessage, transfer: *Transfer) !bool { + if (msg.err == null or msg.err.? == error.RecvError) { + transfer.detectAuthChallenge(msg.conn); + } + + // In case of auth challenge + // TODO give a way to configure the number of auth retries. + if (transfer._auth_challenge != null and transfer._tries < 10) { + var wait_for_interception = false; + transfer.req.params.notification.dispatch( + .http_request_auth_required, + &.{ .transfer = transfer, .wait_for_interception = &wait_for_interception }, + ); + if (wait_for_interception) { + self.interception_layer.intercepted += 1; + if (comptime IS_DEBUG) { + log.debug(.http, "wait for auth interception", .{ .intercepted = self.interception_layer.intercepted }); + } + + // Whether or not this is a blocking request, we're not going + // to process it now. We can end the transfer, which will + // release the easy handle back into the pool. The transfer + // is still valid/alive (just has no handle). + transfer.releaseConn(); + return false; + } + } + // Handle redirects: reuse the same connection to preserve TCP state. if (msg.err == null) { const status = try msg.conn.getResponseCode(); @@ -1068,7 +1095,6 @@ pub const Transfer = struct { // for when a Transfer is queued in the client.queue _node: std.DoublyLinkedList.Node = .{}, - _intercept_state: InterceptState = .not_intercepted, const InterceptState = union(enum) { not_intercepted, @@ -1312,24 +1338,25 @@ pub const Transfer = struct { } } - pub fn detectAuthChallenge(conn: *const http.Connection) ?http.AuthChallenge { - const status = conn.getResponseCode() catch return null; - const connect_status = conn.getConnectCode() catch return null; + fn detectAuthChallenge(transfer: *Transfer, conn: *const http.Connection) void { + const status = conn.getResponseCode() catch return; + const connect_status = conn.getConnectCode() catch return; if (status != 401 and status != 407 and connect_status != 401 and connect_status != 407) { - return null; + transfer._auth_challenge = null; + return; } if (conn.getResponseHeader("WWW-Authenticate", 0)) |hdr| { - return http.AuthChallenge.parse(status, .server, hdr.value) catch null; + transfer._auth_challenge = http.AuthChallenge.parse(status, .server, hdr.value) catch null; } else if (conn.getConnectHeader("WWW-Authenticate", 0)) |hdr| { - return http.AuthChallenge.parse(status, .server, hdr.value) catch null; + transfer._auth_challenge = http.AuthChallenge.parse(status, .server, hdr.value) catch null; } else if (conn.getResponseHeader("Proxy-Authenticate", 0)) |hdr| { - return http.AuthChallenge.parse(status, .proxy, hdr.value) catch null; + transfer._auth_challenge = http.AuthChallenge.parse(status, .proxy, hdr.value) catch null; } else if (conn.getConnectHeader("Proxy-Authenticate", 0)) |hdr| { - return http.AuthChallenge.parse(status, .proxy, hdr.value) catch null; + transfer._auth_challenge = http.AuthChallenge.parse(status, .proxy, hdr.value) catch null; } else { - return .{ .status = status, .source = null, .scheme = null, .realm = null }; + transfer._auth_challenge = .{ .status = status, .source = null, .scheme = null, .realm = null }; } } @@ -1358,16 +1385,12 @@ pub const Transfer = struct { // before interception process. pub fn abortAuthChallenge(self: *Transfer) void { if (comptime IS_DEBUG) { - std.debug.assert(self._intercept_state != .not_intercepted); - log.debug(.http, "abort auth transfer", .{ .intercepted = self.client.intercepted }); + log.debug(.http, "abort auth transfer", .{ .intercepted = self.client.interception_layer.intercepted }); } - self.client.intercepted -= 1; - if (!self.req.params.blocking) { - self.abort(error.AbortAuthChallenge); - return; - } - self._intercept_state = .{ .abort = error.AbortAuthChallenge }; + self.client.interception_layer.intercepted -= 1; + self.abort(error.AbortAuthChallenge); + return; } // headerDoneCallback is called once the headers have been read. @@ -1551,6 +1574,15 @@ pub const Transfer = struct { } }; +pub fn continueTransfer(self: *Client, transfer: *Transfer) !void { + if (comptime IS_DEBUG) { + log.debug(.http, "continue transfer", .{ .intercepted = self.interception_layer.intercepted }); + } + + self.interception_layer.intercepted -= 1; + return self.process(transfer); +} + const Noop = struct { fn headerCallback(_: Response) !bool { return true; diff --git a/src/browser/ScriptManager.zig b/src/browser/ScriptManager.zig index 28a3aefe..f707f390 100644 --- a/src/browser/ScriptManager.zig +++ b/src/browser/ScriptManager.zig @@ -680,7 +680,6 @@ pub const Script = struct { debug_transfer_aborted: bool = false, debug_transfer_bytes_received: usize = 0, debug_transfer_notified_fail: bool = false, - debug_transfer_intercept_state: u8 = 0, debug_transfer_auth_challenge: bool = false, debug_transfer_easy_id: usize = 0, @@ -756,7 +755,6 @@ pub const Script = struct { .a3 = self.debug_transfer_aborted, .a4 = self.debug_transfer_bytes_received, .a5 = self.debug_transfer_notified_fail, - .a7 = self.debug_transfer_intercept_state, .a8 = self.debug_transfer_auth_challenge, .a9 = self.debug_transfer_easy_id, .b1 = transfer.id, @@ -764,7 +762,6 @@ pub const Script = struct { .b3 = transfer.aborted, .b4 = transfer.bytes_received, .b5 = transfer._notified_fail, - .b7 = @intFromEnum(transfer._intercept_state), .b8 = transfer._auth_challenge != null, .b9 = if (transfer._conn) |c| @intFromPtr(c._easy) else 0, }); @@ -774,7 +771,6 @@ pub const Script = struct { self.debug_transfer_aborted = transfer.aborted; self.debug_transfer_bytes_received = transfer.bytes_received; self.debug_transfer_notified_fail = transfer._notified_fail; - self.debug_transfer_intercept_state = @intFromEnum(transfer._intercept_state); self.debug_transfer_auth_challenge = transfer._auth_challenge != null; self.debug_transfer_easy_id = if (transfer._conn) |c| @intFromPtr(c._easy) else 0; }, diff --git a/src/cdp/CDP.zig b/src/cdp/CDP.zig index 023f8bf2..326e93c5 100644 --- a/src/cdp/CDP.zig +++ b/src/cdp/CDP.zig @@ -450,9 +450,16 @@ pub const BrowserContext = struct { // abort all intercepted requests before closing the session/page // since some of these might callback into the page/scriptmanager - for (self.intercept_state.pendingRequests()) |request| { - defer request.deinit(); - request.error_callback(request.ctx, error.ClientDisconnect); + for (self.intercept_state.pendingIntercepts()) |intercept| { + switch (intercept) { + .transfer => |t| { + t.abort(error.ClientDisconnect); + }, + .request => |r| { + defer r.deinit(); + r.error_callback(r.ctx, error.ClientDisconnect); + }, + } } for (self.isolated_worlds.items) |world| { diff --git a/src/cdp/domains/fetch.zig b/src/cdp/domains/fetch.zig index 0f638b81..7eff321e 100644 --- a/src/cdp/domains/fetch.zig +++ b/src/cdp/domains/fetch.zig @@ -54,7 +54,12 @@ pub fn processMessage(cmd: *CDP.Command) !void { // Stored in CDP pub const InterceptState = struct { allocator: Allocator, - waiting: std.AutoArrayHashMapUnmanaged(u32, HttpClient.Request), + waiting: std.AutoArrayHashMapUnmanaged(u32, Pending), + + const Pending = union(enum) { + transfer: *HttpClient.Transfer, + request: HttpClient.Request, + }; pub fn init(allocator: Allocator) !InterceptState { return .{ @@ -67,11 +72,15 @@ pub const InterceptState = struct { return self.waiting.count() == 0; } - pub fn put(self: *InterceptState, request: HttpClient.Request) !void { - return self.waiting.put(self.allocator, request.params.request_id, request); + pub fn putRequest(self: *InterceptState, request: HttpClient.Request) !void { + return self.waiting.put(self.allocator, request.params.request_id, .{ .request = request }); } - pub fn remove(self: *InterceptState, request_id: u32) ?HttpClient.Request { + pub fn putTransfer(self: *InterceptState, transfer: *HttpClient.Transfer) !void { + return self.waiting.put(self.allocator, transfer.id, .{ .transfer = transfer }); + } + + pub fn remove(self: *InterceptState, request_id: u32) ?Pending { const entry = self.waiting.fetchSwapRemove(request_id) orelse return null; return entry.value; } @@ -80,7 +89,7 @@ pub const InterceptState = struct { self.waiting.deinit(self.allocator); } - pub fn pendingRequests(self: *const InterceptState) []HttpClient.Request { + pub fn pendingIntercepts(self: *const InterceptState) []Pending { return self.waiting.values(); } }; @@ -193,7 +202,7 @@ pub fn requestIntercept(bc: *CDP.BrowserContext, intercept: *const Notification. // TODO: What to do when receiving replies for a previous frame's requests? const request = intercept.request; - try bc.intercept_state.put(request.*); + try bc.intercept_state.putRequest(request.*); try bc.cdp.sendEvent("Fetch.requestPaused", .{ .requestId = &id.toInterceptId(request.params.request_id), @@ -235,7 +244,9 @@ fn continueRequest(cmd: *CDP.Command) !void { var intercept_state = &bc.intercept_state; const request_id = try idFromRequestId(params.requestId); - var request = intercept_state.remove(request_id) orelse return error.RequestNotFound; + + const pending = intercept_state.remove(request_id) orelse return error.RequestNotFound; + var request = pending.request; log.debug(.cdp, "request intercept", .{ .state = "continue", @@ -300,7 +311,9 @@ fn continueWithAuth(cmd: *CDP.Command) !void { var intercept_state = &bc.intercept_state; const request_id = try idFromRequestId(params.requestId); - var request = intercept_state.remove(request_id) orelse return error.RequestNotFound; + const pending = intercept_state.remove(request_id) orelse return error.RequestNotFound; + const transfer = pending.transfer; + var request = transfer.req; log.debug(.cdp, "request intercept", .{ .state = "continue with auth", @@ -311,15 +324,15 @@ fn continueWithAuth(cmd: *CDP.Command) !void { const client = bc.cdp.browser.http_client; if (params.authChallengeResponse.response != .ProvideCredentials) { - client.interception_layer.abortAuthChallenge(request); + transfer.abortAuthChallenge(); return cmd.sendResult(null, .{}); } // cancel the request, deinit the transfer on error. - errdefer client.interception_layer.abortAuthChallenge(request); + errdefer transfer.abortAuthChallenge(); const arena = request.params.arena.allocator(); - request.params.credentials = try std.fmt.allocPrintSentinel( + transfer.updateCredentials(try std.fmt.allocPrintSentinel( arena, "{s}:{s}", .{ @@ -327,9 +340,9 @@ fn continueWithAuth(cmd: *CDP.Command) !void { params.authChallengeResponse.password, }, 0, - ); + )); - try client.interception_layer.continueRequest(client, request); + try client.continueTransfer(transfer); return cmd.sendResult(null, .{}); } @@ -352,7 +365,9 @@ fn fulfillRequest(cmd: *CDP.Command) !void { var intercept_state = &bc.intercept_state; const request_id = try idFromRequestId(params.requestId); - var request = intercept_state.remove(request_id) orelse return error.RequestNotFound; + + const pending = intercept_state.remove(request_id) orelse return error.RequestNotFound; + var request = pending.request; log.debug(.cdp, "request intercept", .{ .state = "fulfilled", @@ -385,7 +400,8 @@ fn failRequest(cmd: *CDP.Command) !void { var intercept_state = &bc.intercept_state; const request_id = try idFromRequestId(params.requestId); - const request = intercept_state.remove(request_id) orelse return error.RequestNotFound; + const pending = intercept_state.remove(request_id) orelse return error.RequestNotFound; + const request = pending.request; const client = bc.cdp.browser.http_client; defer client.interception_layer.abortRequest(client, request); @@ -408,16 +424,16 @@ pub fn requestAuthRequired(bc: *CDP.BrowserContext, intercept: *const Notificati // NOTE: we assume whomever created the request created it with a lifetime of the Page. // TODO: What to do when receiving replies for a previous frame's requests? - const intercept_ctx = intercept.intercept_ctx; - const request = intercept.request; - try bc.intercept_state.put(request.*); + const transfer = intercept.transfer; + try bc.intercept_state.putTransfer(transfer); + var request = transfer.req; - const challenge = intercept_ctx.auth_challenge orelse return error.NullAuthChallenge; + const challenge = transfer._auth_challenge orelse return error.NullAuthChallenge; try bc.cdp.sendEvent("Fetch.authRequired", .{ .requestId = &id.toInterceptId(request.params.request_id), .frameId = &id.toFrameId(request.params.frame_id), - .request = network.RequestWriter.init(request), + .request = network.RequestWriter.init(&request), .resourceType = switch (request.params.resource_type) { .script => "Script", .xhr => "XHR", @@ -430,7 +446,7 @@ pub fn requestAuthRequired(bc: *CDP.BrowserContext, intercept: *const Notificati .scheme = if (challenge.scheme) |s| (if (s == .digest) "digest" else "basic") else "", .realm = challenge.realm orelse "", }, - .networkId = &id.toRequestId2(request), + .networkId = &id.toRequestId2(&request), }, .{ .session_id = session_id }); log.debug(.cdp, "request auth required", .{ diff --git a/src/cdp/domains/network.zig b/src/cdp/domains/network.zig index 06581c15..1458bb0c 100644 --- a/src/cdp/domains/network.zig +++ b/src/cdp/domains/network.zig @@ -386,63 +386,6 @@ pub const RequestWriter = struct { } }; -pub const TransferAsRequestWriter = struct { - transfer: *Transfer, - - pub fn init(transfer: *Transfer) TransferAsRequestWriter { - return .{ - .transfer = transfer, - }; - } - - pub fn jsonStringify(self: *const TransferAsRequestWriter, jws: anytype) !void { - self._jsonStringify(jws) catch return error.WriteFailed; - } - fn _jsonStringify(self: *const TransferAsRequestWriter, jws: anytype) !void { - const transfer = self.transfer; - - try jws.beginObject(); - { - try jws.objectField("url"); - try jws.write(transfer.url); - } - - { - const frag = URL.getHash(transfer.url); - if (frag.len > 0) { - try jws.objectField("urlFragment"); - try jws.write(frag); - } - } - - { - try jws.objectField("method"); - try jws.write(@tagName(transfer.req.params.method)); - } - - { - try jws.objectField("hasPostData"); - try jws.write(transfer.req.params.body != null); - } - - { - try jws.objectField("headers"); - try jws.beginObject(); - var it = transfer.req.params.headers.iterator(); - while (it.next()) |hdr| { - try jws.objectField(hdr.name); - try jws.write(hdr.value); - } - if (try transfer.req.getCookieString()) |cookies| { - try jws.objectField("Cookie"); - try jws.write(cookies[0 .. cookies.len - 1]); - } - try jws.endObject(); - } - try jws.endObject(); - } -}; - const ResponseWriter = struct { arena: Allocator, response: *const Response, @@ -533,98 +476,6 @@ const ResponseWriter = struct { } }; -const TransferAsResponseWriter = struct { - arena: Allocator, - transfer: *Transfer, - - fn init(arena: Allocator, transfer: *Transfer) TransferAsResponseWriter { - return .{ - .arena = arena, - .transfer = transfer, - }; - } - - pub fn jsonStringify(self: *const TransferAsResponseWriter, jws: anytype) !void { - self._jsonStringify(jws) catch return error.WriteFailed; - } - - fn _jsonStringify(self: *const TransferAsResponseWriter, jws: anytype) !void { - const transfer = self.transfer; - - try jws.beginObject(); - { - try jws.objectField("url"); - try jws.write(transfer.url); - } - - if (transfer.response_header) |*rh| { - // it should not be possible for this to be false, but I'm not - // feeling brave today. - const status = rh.status; - try jws.objectField("status"); - try jws.write(status); - - try jws.objectField("statusText"); - try jws.write(@as(std.http.Status, @enumFromInt(status)).phrase() orelse "Unknown"); - } - - { - const mime: Mime = blk: { - if (transfer.response_header.?.contentType()) |ct| { - break :blk try Mime.parse(ct); - } - break :blk .unknown; - }; - - try jws.objectField("mimeType"); - try jws.write(mime.contentTypeString()); - try jws.objectField("charset"); - try jws.write(mime.charsetString()); - } - - { - try jws.objectField("timing"); - try jws.write(.{ - .requestTime = transfer.start_time, - .connectEnd = -1, - .connectStart = -1, - .dnsEnd = -1, - .dnsStart = -1, - .proxyEnd = -1, - .proxyStart = -1, - .receiveHeadersEnd = -1, - .receiveHeadersStart = -1, - .sendEnd = -1, - .sendStart = -1, - .sslEnd = -1, - .sslStart = -1, - }); - } - - { - // chromedp doesn't like having duplicate header names. It's pretty - // common to get these from a server (e.g. for Cache-Control), but - // Chrome joins these. So we have to too. - const arena = self.arena; - var it = transfer.responseHeaderIterator(); - var map: std.StringArrayHashMapUnmanaged([]const u8) = .empty; - while (it.next()) |hdr| { - const gop = try map.getOrPut(arena, hdr.name); - if (gop.found_existing) { - // yes, chrome joins multi-value headers with a \n - gop.value_ptr.* = try std.mem.join(arena, "\n", &.{ gop.value_ptr.*, hdr.value }); - } else { - gop.value_ptr.* = hdr.value; - } - } - - try jws.objectField("headers"); - try jws.write(std.json.ArrayHashMap([]const u8){ .map = map }); - } - try jws.endObject(); - } -}; - fn idFromRequestId(request_id: []const u8) !u64 { // The requesIid for the original document is its loaderId. if (!std.mem.startsWith(u8, request_id, "REQ-") and !std.mem.startsWith(u8, request_id, "LID-")) { diff --git a/src/network/layer/InterceptionLayer.zig b/src/network/layer/InterceptionLayer.zig index 823a5314..fd868af1 100644 --- a/src/network/layer/InterceptionLayer.zig +++ b/src/network/layer/InterceptionLayer.zig @@ -107,9 +107,6 @@ pub const InterceptContext = struct { request: Request, content_length: usize = 0, - auth_challenge: ?http.AuthChallenge = null, - tries: usize = 0, - fn startCallback(response: Response) anyerror!void { const self: *InterceptContext = @ptrCast(@alignCast(response.ctx)); log.debug(.http, "intercept start", .{ .url = self.request.params.url }); @@ -126,16 +123,6 @@ pub const InterceptContext = struct { self.content_length = response.contentLength() orelse 0; - switch (response.inner) { - .transfer => |t| { - const status = t.response_header.?.status; - if (status == 401 or status == 407) { - self.auth_challenge = Transfer.detectAuthChallenge(t._conn.?); - } - }, - else => {}, - } - self.request.params.notification.dispatch(.http_response_header_done, &.{ .request = &self.request, .response = &response, @@ -166,26 +153,6 @@ pub const InterceptContext = struct { .content_length = self.content_length, }); - // if (self.auth_challenge != null and self.tries < 10) { - // var wait_for_interception = false; - // self.request.params.notification.dispatch(.http_request_auth_required, &.{ - // .request = &self.request, - // .intercept_ctx = self, - // .wait_for_interception = &wait_for_interception, - // }); - - // if (wait_for_interception) { - // log.debug(.http, "intercept auth required", .{ - // .url = self.request.params.url, - // .intercepted = self.layer.intercepted, - // }); - // self.layer.intercepted += 1; - // self.tries += 1; - // // Don't forward done — CDP owns this now, will retry via continueWithAuth - // return; - // } - // } - self.request.params.notification.dispatch(.http_request_done, &.{ .request = &self.request, .content_length = self.content_length,