CDP: enforce Network.enable limits

This enforces a limit on the captures responses based on the Network.enable's
maxTotalBufferSize and maxResourceBufferSize fields. It also limits the size of
the post body we echo (e.g. in requestWillBeSent) based on the `maxPostDataSize`
field.

src/cdp/CDP.zig

@@ -39,6 +39,7 @@ const Label = @import("../browser/webapi/element/html/Label.zig");
 
 const Connection = @import("Connection.zig");
 const Incrementing = @import("id.zig").Incrementing;
+const network_domain = @import("domains/network.zig");
 const InterceptState = @import("domains/fetch.zig").InterceptState;
 
 const log = lp.log;
@@ -420,7 +421,7 @@ fn dispatchCommand(command: *Command, method: []const u8) !void {
         7 => switch (@as(u56, @bitCast(domain[0..7].*))) {
             asUint(u56, "Browser") => return @import("domains/browser.zig").processMessage(command),
             asUint(u56, "Runtime") => return @import("domains/runtime.zig").processMessage(command),
-            asUint(u56, "Network") => return @import("domains/network.zig").processMessage(command),
+            asUint(u56, "Network") => return network_domain.processMessage(command),
             asUint(u56, "Storage") => return @import("domains/storage.zig").processMessage(command),
             asUint(u56, "Console") => return @import("domains/console.zig").processMessage(command),
             else => {},
@@ -515,7 +516,9 @@ pub const BrowserContext = struct {
 
     const CapturedResponse = struct {
         must_encode: bool,
-        data: std.ArrayList(u8),
+        // null once evicted (size limit). We keep the map entry so that we
+        // can give an "evicted" error instead of "unknown"
+        data: ?std.ArrayList(u8),
     };
 
     // Key for `captured_responses` / `captured_requests`. Documents are
@@ -616,8 +619,10 @@ pub const BrowserContext = struct {
     // memory longer than they have to. In fact, the main request is only
     // ever streamed. So if CDP is the only thing that needs bodies in
     // memory for an arbitrary amount of time, then that's where we're going
-    // to store the,
-    captured_responses: std.AutoHashMapUnmanaged(CapturedKey, CapturedResponse),
+    // to store them.
+    captured_responses_size: usize = 0,
+    captured_responses: std.AutoArrayHashMapUnmanaged(CapturedKey, CapturedResponse),
+    network_limits: network_domain.BufferLimits = .{},
 
     notification: *Notification,
 
@@ -723,6 +728,7 @@ pub const BrowserContext = struct {
         // rely on those notifications to do our normal cleanup?)
 
         self.notification.unregisterAll(self);
+        self.clearCapturedResponses();
 
         // If the session has a frame, we need to clear it first. The page
         // context is always nested inside of the isolated world context,
@@ -852,7 +858,22 @@ pub const BrowserContext = struct {
         };
     }
 
-    pub fn networkEnable(self: *BrowserContext) !void {
+    pub fn networkEnable(self: *BrowserContext, limits: network_domain.BufferLimits) !void {
+        const previous = self.network_limits;
+        self.network_limits = limits;
+        if (limits.resource < previous.resource) {
+            // evict responses which are over the new small limit
+            for (self.captured_responses.values()) |*resp| {
+                const data = resp.data orelse continue;
+                if (data.items.len > limits.resource) {
+                    self.evictCapturedResponse(resp);
+                }
+            }
+        }
+        if (limits.total < previous.total) {
+            self.ensureCapturedSpace(0);
+        }
+
         try self.notification.register(.http_request_fail, self, onHttpRequestFail);
         try self.notification.register(.http_request_start, self, onHttpRequestStart);
         try self.notification.register(.http_request_done, self, onHttpRequestDone);
@@ -868,6 +889,41 @@ pub const BrowserContext = struct {
         self.notification.unregister(.http_response_data, self);
         self.notification.unregister(.http_response_header_done, self);
         self.notification.unregister(.http_request_served_from_cache, self);
+        self.clearCapturedResponses();
+    }
+
+    pub fn clearCapturedResponses(self: *BrowserContext) void {
+        const allocator = self.cdp.allocator;
+        for (self.captured_responses.values()) |*resp| {
+            if (resp.data) |*data| {
+                data.deinit(allocator);
+            }
+        }
+        self.captured_responses.deinit(allocator);
+        self.captured_responses = .empty;
+        self.captured_responses_size = 0;
+    }
+
+    fn evictCapturedResponse(self: *BrowserContext, resp: *CapturedResponse) void {
+        if (resp.data) |*data| {
+            self.captured_responses_size -= data.items.len;
+            data.deinit(self.cdp.allocator);
+            resp.data = null;
+        }
+    }
+
+    // Evict captured responses, ordered by age, until we can fit `size` more bytes
+    fn ensureCapturedSpace(self: *BrowserContext, size: usize) void {
+        for (self.captured_responses.values()) |*resp| {
+            if (self.captured_responses_size + size <= self.network_limits.total) {
+                return;
+            }
+            const data = resp.data orelse continue;
+            if (data.items.len == 0) {
+                continue;
+            }
+            self.evictCapturedResponse(resp);
+        }
     }
 
     pub fn fetchEnable(self: *BrowserContext, authRequests: bool, session_id: []const u8) !void {
@@ -1015,7 +1071,7 @@ pub const BrowserContext = struct {
             const transfer = msg.transfer;
             const key = keyFromTransfer(transfer);
             const body = transfer.req.body orelse "";
-            if (body.len == 0 or body.len > @import("domains/network.zig").max_post_data_size) {
+            if (body.len == 0 or body.len > network_domain.max_post_data_size) {
                 _ = self.captured_requests.remove(key);
             } else {
                 const owned_body = try self.frame_arena.dupe(u8, body);
@@ -1023,7 +1079,7 @@ pub const BrowserContext = struct {
             }
         }
         defer self.resetNotificationArena();
-        try @import("domains/network.zig").httpRequestStart(self.notification_arena, self, msg);
+        try network_domain.httpRequestStart(self.notification_arena, self, msg);
     }
 
     pub fn onHttpRequestIntercept(ctx: *anyopaque, msg: *const Notification.RequestIntercept) !void {
@@ -1034,7 +1090,7 @@ pub const BrowserContext = struct {
 
     pub fn onHttpRequestFail(ctx: *anyopaque, msg: *const Notification.RequestFail) !void {
         const self: *BrowserContext = @ptrCast(@alignCast(ctx));
-        return @import("domains/network.zig").httpRequestFail(self, msg);
+        return network_domain.httpRequestFail(self, msg);
     }
 
     pub fn onFrameDOMContentLoaded(ctx: *anyopaque, msg: *const Notification.FrameDOMContentLoaded) !void {
@@ -1063,14 +1119,18 @@ pub const BrowserContext = struct {
         const self: *BrowserContext = @ptrCast(@alignCast(ctx));
         defer self.resetNotificationArena();
 
-        const arena = self.frame_arena;
-
         // Prepare the captured response value.
         const key = keyFromTransfer(msg.transfer);
-        const gop = try self.captured_responses.getOrPut(arena, key);
+        const gop = try self.captured_responses.getOrPut(self.cdp.allocator, key);
         if (!gop.found_existing) {
             gop.value_ptr.* = .{
-                .data = .empty,
+                .data = blk: {
+                    const cl = msg.transfer.getContentLength() orelse break :blk .empty;
+                    if (cl > self.network_limits.resource) {
+                        break :blk null;
+                    }
+                    break :blk try std.ArrayList(u8).initCapacity(self.cdp.allocator, cl);
+                },
                 // Encode the data in base64 by default, but don't encode
                 // for well known content-type.
                 .must_encode = blk: {
@@ -1081,7 +1141,7 @@ pub const BrowserContext = struct {
                             break :blk true;
                         }
 
-                        if (std.mem.eql(u8, "UTF-8", mime.charsetString())) {
+                        if (std.ascii.eqlIgnoreCase("UTF-8", mime.charsetString())) {
                             break :blk false;
                         }
                     }
@@ -1090,22 +1150,33 @@ pub const BrowserContext = struct {
             };
         }
 
-        return @import("domains/network.zig").httpResponseHeaderDone(self.notification_arena, self, msg);
+        return network_domain.httpResponseHeaderDone(self.notification_arena, self, msg);
     }
 
     pub fn onHttpRequestDone(ctx: *anyopaque, msg: *const Notification.RequestDone) !void {
         const self: *BrowserContext = @ptrCast(@alignCast(ctx));
-        return @import("domains/network.zig").httpRequestDone(self, msg);
+        return network_domain.httpRequestDone(self, msg);
     }
 
     pub fn onHttpResponseData(ctx: *anyopaque, msg: *const Notification.ResponseData) !void {
         const self: *BrowserContext = @ptrCast(@alignCast(ctx));
-        const arena = self.frame_arena;
 
         const key = keyFromTransfer(msg.transfer);
         const resp = self.captured_responses.getPtr(key) orelse lp.assert(false, "onHttpResponseData missing captured response", .{});
 
-        return resp.data.appendSlice(arena, msg.data);
+        const data = &(resp.data orelse return);
+        const chunk = msg.data;
+        const limits = &self.network_limits;
+        if (data.items.len + chunk.len > limits.resource or chunk.len > limits.total) {
+            return self.evictCapturedResponse(resp);
+        }
+        // Can evict `resp` itself when it's the oldest, hence the re-check.
+        self.ensureCapturedSpace(chunk.len);
+        if (resp.data == null) {
+            return;
+        }
+        try data.appendSlice(self.cdp.allocator, chunk);
+        self.captured_responses_size += chunk.len;
     }
 
     pub fn onHttpRequestAuthRequired(ctx: *anyopaque, data: *const Notification.RequestAuthRequired) !void {
@@ -1116,7 +1187,7 @@ pub const BrowserContext = struct {
 
     pub fn onHttpRequestServedFromCache(ctx: *anyopaque, msg: *const Notification.RequestServedFromCache) !void {
         const self: *BrowserContext = @ptrCast(@alignCast(ctx));
-        return @import("domains/network.zig").httpServedFromCache(self, msg);
+        return network_domain.httpServedFromCache(self, msg);
     }
 
     pub fn onConsoleMessage(ctx: *anyopaque, msg: *const Notification.ConsoleMessage) !void {

src/cdp/domains/fetch.zig

@@ -209,7 +209,7 @@ pub fn requestIntercept(arena: Allocator, bc: *CDP.BrowserContext, intercept: *c
     try bc.cdp.sendEvent("Fetch.requestPaused", .{
         .requestId = &id.toInterceptId(intercept_id),
         .frameId = &id.toFrameId(transfer.req.frame_id),
-        .request = network.RequestWriter.init(arena, transfer),
+        .request = network.RequestWriter.init(arena, transfer, bc.network_limits.post_data),
         .resourceType = transfer.req.resource_type.string(),
         .networkId = &id.toRequestId(transfer), // matches the Network REQ-ID
     }, .{ .session_id = session_id });
@@ -435,7 +435,7 @@ pub fn requestAuthRequired(arena: Allocator, bc: *CDP.BrowserContext, intercept:
     try bc.cdp.sendEvent("Fetch.authRequired", .{
         .requestId = &id.toInterceptId(intercept_id),
         .frameId = &id.toFrameId(request.frame_id),
-        .request = network.RequestWriter.init(arena, transfer),
+        .request = network.RequestWriter.init(arena, transfer, bc.network_limits.post_data),
         .resourceType = request.resource_type.string(),
         .authChallenge = .{
             .origin = "", // TODO get origin, could be the proxy address for example.

src/cdp/domains/network.zig

@@ -39,6 +39,15 @@ const log = lp.log;
 const Allocator = std.mem.Allocator;
 pub const max_post_data_size = 64 * 1024;
 
+// Network.enable's buffer parameters.
+pub const BufferLimits = struct {
+    // total bytes that we'll capture before evicting older entries
+    total: usize = 200 * 1000 * 1000,
+    // max bytes-per-capture that we'll retain
+    resource: usize = 20 * 1000 * 1000,
+    post_data: usize = max_post_data_size,
+};
+
 pub fn processMessage(cmd: *CDP.Command) !void {
     const action = std.meta.stringToEnum(enum {
         enable,
@@ -103,8 +112,26 @@ fn emulateNetworkConditions(cmd: *CDP.Command) !void {
 }
 
 fn enable(cmd: *CDP.Command) !void {
+    const Params = struct {
+        maxTotalBufferSize: ?u32 = null,
+        maxResourceBufferSize: ?u32 = null,
+        maxPostDataSize: ?u32 = null,
+    };
+    const params = (try cmd.params(Params)) orelse Params{};
     const bc = cmd.browser_context orelse return error.BrowserContextNotLoaded;
-    try bc.networkEnable();
+
+    var limits: BufferLimits = .{};
+    if (params.maxTotalBufferSize) |max| {
+        limits.total = max;
+    }
+    if (params.maxResourceBufferSize) |max| {
+        limits.resource = max;
+    }
+    if (params.maxPostDataSize) |max| {
+        // 0 is Chrome's "no limit".
+        limits.post_data = if (max == 0) std.math.maxInt(usize) else max;
+    }
+    try bc.networkEnable(limits);
     return cmd.sendResult(null, .{});
 }
 
@@ -325,19 +352,22 @@ fn getResponseBody(cmd: *CDP.Command) !void {
     const key = try keyFromRequestId(params.requestId);
     const bc = cmd.browser_context orelse return error.BrowserContextNotLoaded;
     const resp = bc.captured_responses.getPtr(key) orelse return error.RequestNotFound;
+    const data = resp.data orelse {
+        return cmd.sendError(-32000, "Request content was evicted from inspector cache", .{});
+    };
 
     // must_encode trusts the declared charset; a server can declare UTF-8 and
     // still send invalid bytes.
-    if (!resp.must_encode and std.unicode.utf8ValidateSlice(resp.data.items)) {
+    if (!resp.must_encode and std.unicode.utf8ValidateSlice(data.items)) {
         return cmd.sendResult(.{
-            .body = resp.data.items,
+            .body = data.items,
             .base64Encoded = false,
         }, .{});
     }
 
-    const encoded_len = std.base64.standard.Encoder.calcSize(resp.data.items.len);
+    const encoded_len = std.base64.standard.Encoder.calcSize(data.items.len);
     const encoded = try cmd.arena.alloc(u8, encoded_len);
-    _ = std.base64.standard.Encoder.encode(encoded, resp.data.items);
+    _ = std.base64.standard.Encoder.encode(encoded, data.items);
 
     return cmd.sendResult(.{
         .body = encoded,
@@ -403,7 +433,7 @@ pub fn httpRequestStart(arena: Allocator, bc: *CDP.BrowserContext, msg: *const N
         .loaderId = &id.toLoaderId(req.loader_id),
         .type = req.resource_type.string(),
         .documentURL = frame.url,
-        .request = RequestWriter.init(arena, transfer),
+        .request = RequestWriter.init(arena, transfer, bc.network_limits.post_data),
         .initiator = .{ .type = "other" },
         .redirectResponse = if (msg.redirect_response)
             ResponseWriter.init(arena, transfer)
@@ -459,11 +489,13 @@ pub fn httpServedFromCache(bc: *CDP.BrowserContext, msg: *const Notification.Req
 pub const RequestWriter = struct {
     arena: Allocator,
     transfer: *Transfer,
+    max_post_data: usize,
 
-    pub fn init(arena: Allocator, transfer: *Transfer) RequestWriter {
+    pub fn init(arena: Allocator, transfer: *Transfer, max_post_data: usize) RequestWriter {
         return .{
             .arena = arena,
             .transfer = transfer,
+            .max_post_data = max_post_data,
         };
     }
 
@@ -500,7 +532,7 @@ pub const RequestWriter = struct {
         }
 
         if (request.body) |body| {
-            if (body.len <= max_post_data_size) {
+            if (body.len <= self.max_post_data) {
                 try jws.objectField("postData");
                 try jws.write(SafeString.wrap(body));
 
@@ -1242,6 +1274,190 @@ test "cdp.Network: POST body exposed as postData" {
     try ctx.expectSentError(-31998, "RequestNotFound", .{ .id = 3 });
 }
 
+// Drives POST /echo_body (the response echoes the body, so its size is the
+// request body's) to completion and returns the wire requestId.
+const EchoDriver = struct {
+    done: bool = false,
+    err: ?anyerror = null,
+
+    fn doneCallback(raw: *anyopaque) !void {
+        const self: *EchoDriver = @ptrCast(@alignCast(raw));
+        self.done = true;
+    }
+
+    fn errorCallback(raw: *anyopaque, err: anyerror) void {
+        const self: *EchoDriver = @ptrCast(@alignCast(raw));
+        self.err = err;
+    }
+
+    fn run(bc: *CDP.BrowserContext, frame_id: u32, body: []const u8) ![14]u8 {
+        const client = &bc.cdp.browser.http_client;
+        var request_id: [14]u8 = undefined;
+        _ = std.fmt.bufPrint(&request_id, "REQ-{d:0>10}", .{client.next_request_id +% 1}) catch unreachable;
+
+        var driver: EchoDriver = .{};
+        try client.request(.{
+            .frame_id = frame_id,
+            .loader_id = 1,
+            .method = .POST,
+            .url = "http://127.0.0.1:9582/echo_body",
+            .body = body,
+            .cookie_jar = null,
+            .cookie_origin = "http://127.0.0.1:9582/",
+            .resource_type = .fetch,
+            .notification = bc.session.notification,
+            .ctx = &driver,
+            .done_callback = doneCallback,
+            .error_callback = errorCallback,
+            .shutdown_callback = HttpClient.noopShutdown,
+        }, null);
+
+        for (0..50) |_| {
+            if (driver.done or driver.err != null) break;
+            _ = try client.tick(20);
+        }
+        try testing.expectEqual(null, driver.err);
+        try testing.expect(driver.done);
+        return request_id;
+    }
+};
+
+test "cdp.Network: enable maxResourceBufferSize evicts oversized bodies" {
+    var ctx = try testing.context();
+    defer ctx.deinit();
+
+    const bc = try ctx.loadBrowserContext(.{ .id = "BID-RBS", .session_id = "SID-RBS" });
+    const page = try bc.session.createPage();
+
+    try ctx.processMessage(.{
+        .id = 1,
+        .method = "Network.enable",
+        .params = .{ .maxResourceBufferSize = 4 },
+    });
+    try ctx.expectSentResult(null, .{ .id = 1 });
+
+    const big = try EchoDriver.run(bc, page.frame_id, "12345678");
+    const small = try EchoDriver.run(bc, page.frame_id, "123");
+
+    try ctx.processMessage(.{
+        .id = 2,
+        .method = "Network.getResponseBody",
+        .params = .{ .requestId = &big },
+    });
+    try ctx.expectSentError(-32000, "Request content was evicted from inspector cache", .{ .id = 2 });
+
+    try ctx.processMessage(.{
+        .id = 3,
+        .method = "Network.getResponseBody",
+        .params = .{ .requestId = &small },
+    });
+    try ctx.expectSentResult(.{ .body = "123", .base64Encoded = false }, .{ .id = 3 });
+    try testing.expectEqual(3, bc.captured_responses_size);
+
+    // Re-enabling with a tighter limit evicts what's already captured.
+    try ctx.processMessage(.{
+        .id = 4,
+        .method = "Network.enable",
+        .params = .{ .maxResourceBufferSize = 2 },
+    });
+    try ctx.expectSentResult(null, .{ .id = 4 });
+    try ctx.processMessage(.{
+        .id = 5,
+        .method = "Network.getResponseBody",
+        .params = .{ .requestId = &small },
+    });
+    try ctx.expectSentError(-32000, "Request content was evicted from inspector cache", .{ .id = 5 });
+    try testing.expectEqual(0, bc.captured_responses_size);
+}
+
+test "cdp.Network: enable maxTotalBufferSize evicts oldest bodies first" {
+    var ctx = try testing.context();
+    defer ctx.deinit();
+
+    const bc = try ctx.loadBrowserContext(.{ .id = "BID-TBS", .session_id = "SID-TBS" });
+    const page = try bc.session.createPage();
+
+    try ctx.processMessage(.{
+        .id = 1,
+        .method = "Network.enable",
+        .params = .{ .maxTotalBufferSize = 10 },
+    });
+    try ctx.expectSentResult(null, .{ .id = 1 });
+
+    const first = try EchoDriver.run(bc, page.frame_id, "aaaaaa");
+    const second = try EchoDriver.run(bc, page.frame_id, "bbbbbb");
+
+    try ctx.processMessage(.{
+        .id = 2,
+        .method = "Network.getResponseBody",
+        .params = .{ .requestId = &first },
+    });
+    try ctx.expectSentError(-32000, "Request content was evicted from inspector cache", .{ .id = 2 });
+
+    try ctx.processMessage(.{
+        .id = 3,
+        .method = "Network.getResponseBody",
+        .params = .{ .requestId = &second },
+    });
+    try ctx.expectSentResult(.{ .body = "bbbbbb", .base64Encoded = false }, .{ .id = 3 });
+    try testing.expectEqual(6, bc.captured_responses_size);
+
+    // Network.disable releases everything retained.
+    try ctx.processMessage(.{ .id = 4, .method = "Network.disable" });
+    try ctx.expectSentResult(null, .{ .id = 4 });
+    try testing.expectEqual(0, bc.captured_responses.count());
+    try testing.expectEqual(0, bc.captured_responses_size);
+    try ctx.processMessage(.{
+        .id = 5,
+        .method = "Network.getResponseBody",
+        .params = .{ .requestId = &second },
+    });
+    try ctx.expectSentError(-31998, "RequestNotFound", .{ .id = 5 });
+}
+
+test "cdp.Network: enable maxPostDataSize omits inline postData" {
+    var ctx = try testing.context();
+    defer ctx.deinit();
+
+    const bc = try ctx.loadBrowserContext(.{ .id = "BID-PDS", .session_id = "SID-PDS" });
+    const page = try bc.session.createPage();
+
+    try ctx.processMessage(.{
+        .id = 1,
+        .method = "Network.enable",
+        .params = .{ .maxPostDataSize = 8 },
+    });
+    try ctx.expectSentResult(null, .{ .id = 1 });
+
+    const body = "{\"source\":\"xhr\",\"pageSize\":100}";
+    const request_id = try EchoDriver.run(bc, page.frame_id, body);
+
+    try ctx.expectSentEvent("Network.requestWillBeSent", .{
+        .requestId = &request_id,
+        .request = .{ .method = "POST", .hasPostData = true },
+    }, .{ .session_id = "SID-PDS" });
+
+    // The subset matcher can't assert absence; look at the event directly.
+    var seen = false;
+    for (ctx.received.items) |received| {
+        const method = received.object.get("method") orelse continue;
+        if (!std.mem.eql(u8, method.string, "Network.requestWillBeSent")) continue;
+        const request = received.object.get("params").?.object.get("request").?.object;
+        try testing.expectEqual(null, request.get("postData"));
+        try testing.expectEqual(null, request.get("postDataEntries"));
+        seen = true;
+    }
+    try testing.expect(seen);
+
+    // Retention is independent of the inline limit.
+    try ctx.processMessage(.{
+        .id = 2,
+        .method = "Network.getRequestPostData",
+        .params = .{ .requestId = &request_id },
+    });
+    try ctx.expectSentResult(.{ .postData = body }, .{ .id = 2 });
+}
+
 test "cdp.Network: redirect hop precedes Fetch pause and carries redirectResponse" {
     var ctx = try testing.context();
     defer ctx.deinit();

src/cdp/domains/page.zig

@@ -551,7 +551,7 @@ pub fn frameCreated(bc: *CDP.BrowserContext, frame: *Frame) !void {
         // controlled via Network.configureDurableMessages (which we don't
         // support).
         bc.captured_requests = .empty;
-        bc.captured_responses = .empty;
+        bc.clearCapturedResponses();
     }
 }