From dad2ed42c9b326fd0540c2cf3077c0bc552d7b14 Mon Sep 17 00:00:00 2001 From: Karl Seguin Date: Mon, 31 Aug 2026 17:20:55 +0800 Subject: [PATCH] CDP: enforce Network.enable limits This enforces a limit on the captures responses based on the Network.enable's maxTotalBufferSize and maxResourceBufferSize fields. It also limits the size of the post body we echo (e.g. in requestWillBeSent) based on the `maxPostDataSize` field. --- src/cdp/CDP.zig | 107 ++++++++++++++--- src/cdp/domains/fetch.zig | 4 +- src/cdp/domains/network.zig | 232 ++++++++++++++++++++++++++++++++++-- src/cdp/domains/page.zig | 2 +- 4 files changed, 316 insertions(+), 29 deletions(-) diff --git a/src/cdp/CDP.zig b/src/cdp/CDP.zig index 593f6c732..b2e29c49b 100644 --- a/src/cdp/CDP.zig +++ b/src/cdp/CDP.zig @@ -39,6 +39,7 @@ const Label = @import("../browser/webapi/element/html/Label.zig"); const Connection = @import("Connection.zig"); const Incrementing = @import("id.zig").Incrementing; +const network_domain = @import("domains/network.zig"); const InterceptState = @import("domains/fetch.zig").InterceptState; const log = lp.log; @@ -420,7 +421,7 @@ fn dispatchCommand(command: *Command, method: []const u8) !void { 7 => switch (@as(u56, @bitCast(domain[0..7].*))) { asUint(u56, "Browser") => return @import("domains/browser.zig").processMessage(command), asUint(u56, "Runtime") => return @import("domains/runtime.zig").processMessage(command), - asUint(u56, "Network") => return @import("domains/network.zig").processMessage(command), + asUint(u56, "Network") => return network_domain.processMessage(command), asUint(u56, "Storage") => return @import("domains/storage.zig").processMessage(command), asUint(u56, "Console") => return @import("domains/console.zig").processMessage(command), else => {}, @@ -515,7 +516,9 @@ pub const BrowserContext = struct { const CapturedResponse = struct { must_encode: bool, - data: std.ArrayList(u8), + // null once evicted (size limit). We keep the map entry so that we + // can give an "evicted" error instead of "unknown" + data: ?std.ArrayList(u8), }; // Key for `captured_responses` / `captured_requests`. Documents are @@ -616,8 +619,10 @@ pub const BrowserContext = struct { // memory longer than they have to. In fact, the main request is only // ever streamed. So if CDP is the only thing that needs bodies in // memory for an arbitrary amount of time, then that's where we're going - // to store the, - captured_responses: std.AutoHashMapUnmanaged(CapturedKey, CapturedResponse), + // to store them. + captured_responses_size: usize = 0, + captured_responses: std.AutoArrayHashMapUnmanaged(CapturedKey, CapturedResponse), + network_limits: network_domain.BufferLimits = .{}, notification: *Notification, @@ -723,6 +728,7 @@ pub const BrowserContext = struct { // rely on those notifications to do our normal cleanup?) self.notification.unregisterAll(self); + self.clearCapturedResponses(); // If the session has a frame, we need to clear it first. The page // context is always nested inside of the isolated world context, @@ -852,7 +858,22 @@ pub const BrowserContext = struct { }; } - pub fn networkEnable(self: *BrowserContext) !void { + pub fn networkEnable(self: *BrowserContext, limits: network_domain.BufferLimits) !void { + const previous = self.network_limits; + self.network_limits = limits; + if (limits.resource < previous.resource) { + // evict responses which are over the new small limit + for (self.captured_responses.values()) |*resp| { + const data = resp.data orelse continue; + if (data.items.len > limits.resource) { + self.evictCapturedResponse(resp); + } + } + } + if (limits.total < previous.total) { + self.ensureCapturedSpace(0); + } + try self.notification.register(.http_request_fail, self, onHttpRequestFail); try self.notification.register(.http_request_start, self, onHttpRequestStart); try self.notification.register(.http_request_done, self, onHttpRequestDone); @@ -868,6 +889,41 @@ pub const BrowserContext = struct { self.notification.unregister(.http_response_data, self); self.notification.unregister(.http_response_header_done, self); self.notification.unregister(.http_request_served_from_cache, self); + self.clearCapturedResponses(); + } + + pub fn clearCapturedResponses(self: *BrowserContext) void { + const allocator = self.cdp.allocator; + for (self.captured_responses.values()) |*resp| { + if (resp.data) |*data| { + data.deinit(allocator); + } + } + self.captured_responses.deinit(allocator); + self.captured_responses = .empty; + self.captured_responses_size = 0; + } + + fn evictCapturedResponse(self: *BrowserContext, resp: *CapturedResponse) void { + if (resp.data) |*data| { + self.captured_responses_size -= data.items.len; + data.deinit(self.cdp.allocator); + resp.data = null; + } + } + + // Evict captured responses, ordered by age, until we can fit `size` more bytes + fn ensureCapturedSpace(self: *BrowserContext, size: usize) void { + for (self.captured_responses.values()) |*resp| { + if (self.captured_responses_size + size <= self.network_limits.total) { + return; + } + const data = resp.data orelse continue; + if (data.items.len == 0) { + continue; + } + self.evictCapturedResponse(resp); + } } pub fn fetchEnable(self: *BrowserContext, authRequests: bool, session_id: []const u8) !void { @@ -1015,7 +1071,7 @@ pub const BrowserContext = struct { const transfer = msg.transfer; const key = keyFromTransfer(transfer); const body = transfer.req.body orelse ""; - if (body.len == 0 or body.len > @import("domains/network.zig").max_post_data_size) { + if (body.len == 0 or body.len > network_domain.max_post_data_size) { _ = self.captured_requests.remove(key); } else { const owned_body = try self.frame_arena.dupe(u8, body); @@ -1023,7 +1079,7 @@ pub const BrowserContext = struct { } } defer self.resetNotificationArena(); - try @import("domains/network.zig").httpRequestStart(self.notification_arena, self, msg); + try network_domain.httpRequestStart(self.notification_arena, self, msg); } pub fn onHttpRequestIntercept(ctx: *anyopaque, msg: *const Notification.RequestIntercept) !void { @@ -1034,7 +1090,7 @@ pub const BrowserContext = struct { pub fn onHttpRequestFail(ctx: *anyopaque, msg: *const Notification.RequestFail) !void { const self: *BrowserContext = @ptrCast(@alignCast(ctx)); - return @import("domains/network.zig").httpRequestFail(self, msg); + return network_domain.httpRequestFail(self, msg); } pub fn onFrameDOMContentLoaded(ctx: *anyopaque, msg: *const Notification.FrameDOMContentLoaded) !void { @@ -1063,14 +1119,18 @@ pub const BrowserContext = struct { const self: *BrowserContext = @ptrCast(@alignCast(ctx)); defer self.resetNotificationArena(); - const arena = self.frame_arena; - // Prepare the captured response value. const key = keyFromTransfer(msg.transfer); - const gop = try self.captured_responses.getOrPut(arena, key); + const gop = try self.captured_responses.getOrPut(self.cdp.allocator, key); if (!gop.found_existing) { gop.value_ptr.* = .{ - .data = .empty, + .data = blk: { + const cl = msg.transfer.getContentLength() orelse break :blk .empty; + if (cl > self.network_limits.resource) { + break :blk null; + } + break :blk try std.ArrayList(u8).initCapacity(self.cdp.allocator, cl); + }, // Encode the data in base64 by default, but don't encode // for well known content-type. .must_encode = blk: { @@ -1081,7 +1141,7 @@ pub const BrowserContext = struct { break :blk true; } - if (std.mem.eql(u8, "UTF-8", mime.charsetString())) { + if (std.ascii.eqlIgnoreCase("UTF-8", mime.charsetString())) { break :blk false; } } @@ -1090,22 +1150,33 @@ pub const BrowserContext = struct { }; } - return @import("domains/network.zig").httpResponseHeaderDone(self.notification_arena, self, msg); + return network_domain.httpResponseHeaderDone(self.notification_arena, self, msg); } pub fn onHttpRequestDone(ctx: *anyopaque, msg: *const Notification.RequestDone) !void { const self: *BrowserContext = @ptrCast(@alignCast(ctx)); - return @import("domains/network.zig").httpRequestDone(self, msg); + return network_domain.httpRequestDone(self, msg); } pub fn onHttpResponseData(ctx: *anyopaque, msg: *const Notification.ResponseData) !void { const self: *BrowserContext = @ptrCast(@alignCast(ctx)); - const arena = self.frame_arena; const key = keyFromTransfer(msg.transfer); const resp = self.captured_responses.getPtr(key) orelse lp.assert(false, "onHttpResponseData missing captured response", .{}); - return resp.data.appendSlice(arena, msg.data); + const data = &(resp.data orelse return); + const chunk = msg.data; + const limits = &self.network_limits; + if (data.items.len + chunk.len > limits.resource or chunk.len > limits.total) { + return self.evictCapturedResponse(resp); + } + // Can evict `resp` itself when it's the oldest, hence the re-check. + self.ensureCapturedSpace(chunk.len); + if (resp.data == null) { + return; + } + try data.appendSlice(self.cdp.allocator, chunk); + self.captured_responses_size += chunk.len; } pub fn onHttpRequestAuthRequired(ctx: *anyopaque, data: *const Notification.RequestAuthRequired) !void { @@ -1116,7 +1187,7 @@ pub const BrowserContext = struct { pub fn onHttpRequestServedFromCache(ctx: *anyopaque, msg: *const Notification.RequestServedFromCache) !void { const self: *BrowserContext = @ptrCast(@alignCast(ctx)); - return @import("domains/network.zig").httpServedFromCache(self, msg); + return network_domain.httpServedFromCache(self, msg); } pub fn onConsoleMessage(ctx: *anyopaque, msg: *const Notification.ConsoleMessage) !void { diff --git a/src/cdp/domains/fetch.zig b/src/cdp/domains/fetch.zig index 733dd8b77..b5a7179c1 100644 --- a/src/cdp/domains/fetch.zig +++ b/src/cdp/domains/fetch.zig @@ -209,7 +209,7 @@ pub fn requestIntercept(arena: Allocator, bc: *CDP.BrowserContext, intercept: *c try bc.cdp.sendEvent("Fetch.requestPaused", .{ .requestId = &id.toInterceptId(intercept_id), .frameId = &id.toFrameId(transfer.req.frame_id), - .request = network.RequestWriter.init(arena, transfer), + .request = network.RequestWriter.init(arena, transfer, bc.network_limits.post_data), .resourceType = transfer.req.resource_type.string(), .networkId = &id.toRequestId(transfer), // matches the Network REQ-ID }, .{ .session_id = session_id }); @@ -435,7 +435,7 @@ pub fn requestAuthRequired(arena: Allocator, bc: *CDP.BrowserContext, intercept: try bc.cdp.sendEvent("Fetch.authRequired", .{ .requestId = &id.toInterceptId(intercept_id), .frameId = &id.toFrameId(request.frame_id), - .request = network.RequestWriter.init(arena, transfer), + .request = network.RequestWriter.init(arena, transfer, bc.network_limits.post_data), .resourceType = request.resource_type.string(), .authChallenge = .{ .origin = "", // TODO get origin, could be the proxy address for example. diff --git a/src/cdp/domains/network.zig b/src/cdp/domains/network.zig index 9271ee9ce..113e6de0e 100644 --- a/src/cdp/domains/network.zig +++ b/src/cdp/domains/network.zig @@ -39,6 +39,15 @@ const log = lp.log; const Allocator = std.mem.Allocator; pub const max_post_data_size = 64 * 1024; +// Network.enable's buffer parameters. +pub const BufferLimits = struct { + // total bytes that we'll capture before evicting older entries + total: usize = 200 * 1000 * 1000, + // max bytes-per-capture that we'll retain + resource: usize = 20 * 1000 * 1000, + post_data: usize = max_post_data_size, +}; + pub fn processMessage(cmd: *CDP.Command) !void { const action = std.meta.stringToEnum(enum { enable, @@ -103,8 +112,26 @@ fn emulateNetworkConditions(cmd: *CDP.Command) !void { } fn enable(cmd: *CDP.Command) !void { + const Params = struct { + maxTotalBufferSize: ?u32 = null, + maxResourceBufferSize: ?u32 = null, + maxPostDataSize: ?u32 = null, + }; + const params = (try cmd.params(Params)) orelse Params{}; const bc = cmd.browser_context orelse return error.BrowserContextNotLoaded; - try bc.networkEnable(); + + var limits: BufferLimits = .{}; + if (params.maxTotalBufferSize) |max| { + limits.total = max; + } + if (params.maxResourceBufferSize) |max| { + limits.resource = max; + } + if (params.maxPostDataSize) |max| { + // 0 is Chrome's "no limit". + limits.post_data = if (max == 0) std.math.maxInt(usize) else max; + } + try bc.networkEnable(limits); return cmd.sendResult(null, .{}); } @@ -325,19 +352,22 @@ fn getResponseBody(cmd: *CDP.Command) !void { const key = try keyFromRequestId(params.requestId); const bc = cmd.browser_context orelse return error.BrowserContextNotLoaded; const resp = bc.captured_responses.getPtr(key) orelse return error.RequestNotFound; + const data = resp.data orelse { + return cmd.sendError(-32000, "Request content was evicted from inspector cache", .{}); + }; // must_encode trusts the declared charset; a server can declare UTF-8 and // still send invalid bytes. - if (!resp.must_encode and std.unicode.utf8ValidateSlice(resp.data.items)) { + if (!resp.must_encode and std.unicode.utf8ValidateSlice(data.items)) { return cmd.sendResult(.{ - .body = resp.data.items, + .body = data.items, .base64Encoded = false, }, .{}); } - const encoded_len = std.base64.standard.Encoder.calcSize(resp.data.items.len); + const encoded_len = std.base64.standard.Encoder.calcSize(data.items.len); const encoded = try cmd.arena.alloc(u8, encoded_len); - _ = std.base64.standard.Encoder.encode(encoded, resp.data.items); + _ = std.base64.standard.Encoder.encode(encoded, data.items); return cmd.sendResult(.{ .body = encoded, @@ -403,7 +433,7 @@ pub fn httpRequestStart(arena: Allocator, bc: *CDP.BrowserContext, msg: *const N .loaderId = &id.toLoaderId(req.loader_id), .type = req.resource_type.string(), .documentURL = frame.url, - .request = RequestWriter.init(arena, transfer), + .request = RequestWriter.init(arena, transfer, bc.network_limits.post_data), .initiator = .{ .type = "other" }, .redirectResponse = if (msg.redirect_response) ResponseWriter.init(arena, transfer) @@ -459,11 +489,13 @@ pub fn httpServedFromCache(bc: *CDP.BrowserContext, msg: *const Notification.Req pub const RequestWriter = struct { arena: Allocator, transfer: *Transfer, + max_post_data: usize, - pub fn init(arena: Allocator, transfer: *Transfer) RequestWriter { + pub fn init(arena: Allocator, transfer: *Transfer, max_post_data: usize) RequestWriter { return .{ .arena = arena, .transfer = transfer, + .max_post_data = max_post_data, }; } @@ -500,7 +532,7 @@ pub const RequestWriter = struct { } if (request.body) |body| { - if (body.len <= max_post_data_size) { + if (body.len <= self.max_post_data) { try jws.objectField("postData"); try jws.write(SafeString.wrap(body)); @@ -1242,6 +1274,190 @@ test "cdp.Network: POST body exposed as postData" { try ctx.expectSentError(-31998, "RequestNotFound", .{ .id = 3 }); } +// Drives POST /echo_body (the response echoes the body, so its size is the +// request body's) to completion and returns the wire requestId. +const EchoDriver = struct { + done: bool = false, + err: ?anyerror = null, + + fn doneCallback(raw: *anyopaque) !void { + const self: *EchoDriver = @ptrCast(@alignCast(raw)); + self.done = true; + } + + fn errorCallback(raw: *anyopaque, err: anyerror) void { + const self: *EchoDriver = @ptrCast(@alignCast(raw)); + self.err = err; + } + + fn run(bc: *CDP.BrowserContext, frame_id: u32, body: []const u8) ![14]u8 { + const client = &bc.cdp.browser.http_client; + var request_id: [14]u8 = undefined; + _ = std.fmt.bufPrint(&request_id, "REQ-{d:0>10}", .{client.next_request_id +% 1}) catch unreachable; + + var driver: EchoDriver = .{}; + try client.request(.{ + .frame_id = frame_id, + .loader_id = 1, + .method = .POST, + .url = "http://127.0.0.1:9582/echo_body", + .body = body, + .cookie_jar = null, + .cookie_origin = "http://127.0.0.1:9582/", + .resource_type = .fetch, + .notification = bc.session.notification, + .ctx = &driver, + .done_callback = doneCallback, + .error_callback = errorCallback, + .shutdown_callback = HttpClient.noopShutdown, + }, null); + + for (0..50) |_| { + if (driver.done or driver.err != null) break; + _ = try client.tick(20); + } + try testing.expectEqual(null, driver.err); + try testing.expect(driver.done); + return request_id; + } +}; + +test "cdp.Network: enable maxResourceBufferSize evicts oversized bodies" { + var ctx = try testing.context(); + defer ctx.deinit(); + + const bc = try ctx.loadBrowserContext(.{ .id = "BID-RBS", .session_id = "SID-RBS" }); + const page = try bc.session.createPage(); + + try ctx.processMessage(.{ + .id = 1, + .method = "Network.enable", + .params = .{ .maxResourceBufferSize = 4 }, + }); + try ctx.expectSentResult(null, .{ .id = 1 }); + + const big = try EchoDriver.run(bc, page.frame_id, "12345678"); + const small = try EchoDriver.run(bc, page.frame_id, "123"); + + try ctx.processMessage(.{ + .id = 2, + .method = "Network.getResponseBody", + .params = .{ .requestId = &big }, + }); + try ctx.expectSentError(-32000, "Request content was evicted from inspector cache", .{ .id = 2 }); + + try ctx.processMessage(.{ + .id = 3, + .method = "Network.getResponseBody", + .params = .{ .requestId = &small }, + }); + try ctx.expectSentResult(.{ .body = "123", .base64Encoded = false }, .{ .id = 3 }); + try testing.expectEqual(3, bc.captured_responses_size); + + // Re-enabling with a tighter limit evicts what's already captured. + try ctx.processMessage(.{ + .id = 4, + .method = "Network.enable", + .params = .{ .maxResourceBufferSize = 2 }, + }); + try ctx.expectSentResult(null, .{ .id = 4 }); + try ctx.processMessage(.{ + .id = 5, + .method = "Network.getResponseBody", + .params = .{ .requestId = &small }, + }); + try ctx.expectSentError(-32000, "Request content was evicted from inspector cache", .{ .id = 5 }); + try testing.expectEqual(0, bc.captured_responses_size); +} + +test "cdp.Network: enable maxTotalBufferSize evicts oldest bodies first" { + var ctx = try testing.context(); + defer ctx.deinit(); + + const bc = try ctx.loadBrowserContext(.{ .id = "BID-TBS", .session_id = "SID-TBS" }); + const page = try bc.session.createPage(); + + try ctx.processMessage(.{ + .id = 1, + .method = "Network.enable", + .params = .{ .maxTotalBufferSize = 10 }, + }); + try ctx.expectSentResult(null, .{ .id = 1 }); + + const first = try EchoDriver.run(bc, page.frame_id, "aaaaaa"); + const second = try EchoDriver.run(bc, page.frame_id, "bbbbbb"); + + try ctx.processMessage(.{ + .id = 2, + .method = "Network.getResponseBody", + .params = .{ .requestId = &first }, + }); + try ctx.expectSentError(-32000, "Request content was evicted from inspector cache", .{ .id = 2 }); + + try ctx.processMessage(.{ + .id = 3, + .method = "Network.getResponseBody", + .params = .{ .requestId = &second }, + }); + try ctx.expectSentResult(.{ .body = "bbbbbb", .base64Encoded = false }, .{ .id = 3 }); + try testing.expectEqual(6, bc.captured_responses_size); + + // Network.disable releases everything retained. + try ctx.processMessage(.{ .id = 4, .method = "Network.disable" }); + try ctx.expectSentResult(null, .{ .id = 4 }); + try testing.expectEqual(0, bc.captured_responses.count()); + try testing.expectEqual(0, bc.captured_responses_size); + try ctx.processMessage(.{ + .id = 5, + .method = "Network.getResponseBody", + .params = .{ .requestId = &second }, + }); + try ctx.expectSentError(-31998, "RequestNotFound", .{ .id = 5 }); +} + +test "cdp.Network: enable maxPostDataSize omits inline postData" { + var ctx = try testing.context(); + defer ctx.deinit(); + + const bc = try ctx.loadBrowserContext(.{ .id = "BID-PDS", .session_id = "SID-PDS" }); + const page = try bc.session.createPage(); + + try ctx.processMessage(.{ + .id = 1, + .method = "Network.enable", + .params = .{ .maxPostDataSize = 8 }, + }); + try ctx.expectSentResult(null, .{ .id = 1 }); + + const body = "{\"source\":\"xhr\",\"pageSize\":100}"; + const request_id = try EchoDriver.run(bc, page.frame_id, body); + + try ctx.expectSentEvent("Network.requestWillBeSent", .{ + .requestId = &request_id, + .request = .{ .method = "POST", .hasPostData = true }, + }, .{ .session_id = "SID-PDS" }); + + // The subset matcher can't assert absence; look at the event directly. + var seen = false; + for (ctx.received.items) |received| { + const method = received.object.get("method") orelse continue; + if (!std.mem.eql(u8, method.string, "Network.requestWillBeSent")) continue; + const request = received.object.get("params").?.object.get("request").?.object; + try testing.expectEqual(null, request.get("postData")); + try testing.expectEqual(null, request.get("postDataEntries")); + seen = true; + } + try testing.expect(seen); + + // Retention is independent of the inline limit. + try ctx.processMessage(.{ + .id = 2, + .method = "Network.getRequestPostData", + .params = .{ .requestId = &request_id }, + }); + try ctx.expectSentResult(.{ .postData = body }, .{ .id = 2 }); +} + test "cdp.Network: redirect hop precedes Fetch pause and carries redirectResponse" { var ctx = try testing.context(); defer ctx.deinit(); diff --git a/src/cdp/domains/page.zig b/src/cdp/domains/page.zig index b65960baf..248bc2972 100644 --- a/src/cdp/domains/page.zig +++ b/src/cdp/domains/page.zig @@ -551,7 +551,7 @@ pub fn frameCreated(bc: *CDP.BrowserContext, frame: *Frame) !void { // controlled via Network.configureDurableMessages (which we don't // support). bc.captured_requests = .empty; - bc.captured_responses = .empty; + bc.clearCapturedResponses(); } }