Merge pull request #2760 from lightpanda-io/cdp_max_message_size

cdp: configurable max websocket and http message size

src/Config.zig

@@ -30,13 +30,6 @@ const WebBotAuthConfig = @import("network/WebBotAuth.zig").Config;
 
 const Allocator = std.mem.Allocator;
 
-pub const CDP_MAX_HTTP_REQUEST_SIZE = 4096;
-
-// max message size
-// +14 for max websocket payload overhead
-// +140 for the max control packet that might be interleaved in a message
-pub const CDP_MAX_MESSAGE_SIZE = 512 * 1024 + 14 + 140;
-
 // TCP keepalive parameters applied to accepted CDP connections.
 // Detection window ≈ IDLE + CNT * INTVL = 4 + 3*2 = 10s.
 pub const CDP_KEEPALIVE_IDLE_S: c_int = 4;
@@ -184,6 +177,9 @@ const Commands = cli.Builder(.{
             .{ .name = "timeout", .type = ?u31 },
             .{ .name = "cdp_max_connections", .type = u16, .default = 16 },
             .{ .name = "cdp_max_pending_connections", .type = u16, .default = 128 },
+            .{ .name = "cdp_max_message_size", .type = u32, .default = 1024 * 1024 },
+            // Don't widen this without growing the reader buffer in the HTTP path.
+            .{ .name = "cdp_max_http_message_size", .type = u14, .default = 4096 },
         },
         .shared_options = CommonOptions,
     },
@@ -522,6 +518,20 @@ pub fn maxPendingConnections(self: *const Config) u31 {
     };
 }
 
+pub fn cdpMaxMessageSize(self: *const Config) u32 {
+    return switch (self.mode) {
+        .serve => |opts| opts.cdp_max_message_size,
+        else => unreachable,
+    };
+}
+
+pub fn cdpMaxHTTPMessageSize(self: *const Config) u14 {
+    return switch (self.mode) {
+        .serve => |opts| opts.cdp_max_http_message_size,
+        else => unreachable,
+    };
+}
+
 pub fn storageEngine(self: *const Config) ?Storage.EngineType {
     return switch (self.mode) {
         inline .serve, .fetch, .mcp, .agent => |opts| opts.storage_engine,

src/Server.zig

@@ -313,6 +313,9 @@ test "server: buildJSONVersionResponse" {
 }
 
 test "Client: http invalid request" {
+    const filter: testing.LogFilter = .init(&.{.cdp});
+    defer filter.deinit();
+
     var c = try createTestClient();
     defer c.deinit();
 
@@ -427,8 +430,12 @@ test "Client: read invalid websocket message" {
         );
     }
 
-    // length of message is 0000 0810, i.e: 1024 * 512 + 265
-    try assertWebSocketError(1009, &.{ 129, 255, 0, 0, 0, 0, 0, 8, 1, 0, 'm', 'a', 's', 'k' });
+    {
+        const filter: testing.LogFilter = .init(&.{.cdp});
+        defer filter.deinit();
+        // length of message is 0, 0, 0, 0, 0, 16, 0, 1 i.e: 1024 * 1024 + 1
+        try assertWebSocketError(1009, &.{ 129, 255, 0, 0, 0, 0, 0, 16, 0, 1, 'm', 'a', 's', 'k' });
+    }
 
     // continuation type message must come after a normal message
     // even when not a fin frame
@@ -632,6 +639,7 @@ fn createTestClient() !TestClient {
     return .{
         .stream = stream,
         .reader = .{
+            .max_message_size = 1024,
             .allocator = testing.allocator,
             .buf = try testing.allocator.alloc(u8, 1024 * 16),
         },
@@ -641,7 +649,7 @@ fn createTestClient() !TestClient {
 const TestClient = struct {
     stream: std.net.Stream,
     buf: [1024]u8 = undefined,
-    reader: WS.Reader(false, 1024),
+    reader: WS.Reader(false),
 
     const WS = @import("network/WS.zig");
 

src/cdp/CDP.zig

@@ -111,7 +111,7 @@ pub fn init(
     try self.browser.init(app, .{ .env = .{ .with_inspector = true } }, self);
     const http_client = &self.browser.http_client;
 
-    try self.conn.init(allocator, socket, json_version_response, &http_client.inbox, &app.arena_pool);
+    try self.conn.init(app, socket, json_version_response, &http_client.inbox);
     errdefer self.conn.deinit();
 
     self.link = .{

src/cdp/Connection.zig

@@ -22,6 +22,7 @@ const builtin = @import("builtin");
 
 const CDP = @import("CDP.zig");
 
+const App = @import("../App.zig");
 const Inbox = @import("../Inbox.zig");
 const Config = @import("../Config.zig");
 const WS = @import("../network/WS.zig");
@@ -42,17 +43,17 @@ arena_pool: *ArenaPool,
 socket: posix.socket_t,
 socket_flags: usize,
 state: State = .handshaking,
-reader: WS.Reader(true, Config.CDP_MAX_MESSAGE_SIZE),
+reader: WS.Reader(true),
 send_arena: ArenaAllocator,
+max_http_message_size: usize,
 json_version_response: []const u8,
 
 pub fn init(
     self: *Connection,
-    allocator: Allocator,
+    app: *App,
     socket: posix.socket_t,
     json_version_response: []const u8,
     inbox: *Inbox,
-    arena_pool: *ArenaPool,
 ) !void {
     const socket_flags = try posix.fcntl(socket, posix.F.GETFL, 0);
     const nonblocking = @as(u32, @bitCast(posix.O{ .NONBLOCK = true }));
@@ -60,12 +61,16 @@ pub fn init(
         lp.assert(socket_flags & nonblocking == nonblocking, "Connection.init blocking", .{});
     }
 
+    const config = app.config;
+    const allocator = app.allocator;
+
     self.* = .{
         .inbox = inbox,
         .socket = socket,
-        .arena_pool = arena_pool,
+        .arena_pool = &app.arena_pool,
         .socket_flags = socket_flags,
-        .reader = try .init(allocator),
+        .max_http_message_size = config.cdpMaxHTTPMessageSize(),
+        .reader = try .init(allocator, config.cdpMaxMessageSize()),
         .send_arena = ArenaAllocator.init(allocator),
         .json_version_response = json_version_response,
     };
@@ -222,7 +227,8 @@ fn processHttpRequest(self: *Connection) !HttpResult {
     lp.assert(self.reader.pos == 0, "Connection.HTTP pos", .{ .pos = self.reader.pos });
     const request = self.reader.buf[0..self.reader.len];
 
-    if (request.len > Config.CDP_MAX_HTTP_REQUEST_SIZE) {
+    if (request.len > self.max_http_message_size) {
+        log.warn(.cdp, "CDP message too big", .{ .type = "HTTP", .len = request.len, .hint = "See the --cdp-max-http-message-size <bytes>" });
         self.sendHttpError(413, "Request too large");
         return error.RequestTooLarge;
     }

src/help.zon

@@ -35,6 +35,12 @@
     \\  --cdp-max-pending-connections <INT>
     \\          Maximum pending connections in the accept queue.
     \\          Defaults to 128.
+    \\  --cdp-max-message-size <INT>
+    \\          Maximum allowed incoming websocket message size.
+    \\          Defaults to 1048576 (1MB)
+    \\  --cdp-max-http-message-size <INT>
+    \\          Maximum allowed HTTP request size
+    \\          Defaults to 4096 (maximum allowed: 16383)
     \\  --cookie <PATH>
     \\          Path to a JSON file to load cookies from (read-only).
     \\          Defaults to no cookie loading.

src/network/WS.zig

@@ -58,7 +58,7 @@ pub const OpCode = enum(u8) {
 // WebSocket message reader. Given websocket message, acts as an iterator that
 // can return zero or more Messages. When next returns null, any incomplete
 // message will remain in reader.data
-pub fn Reader(comptime EXPECT_MASK: bool, MAX_MESSAGE_SIZE: usize) type {
+pub fn Reader(comptime EXPECT_MASK: bool) type {
     return struct {
         allocator: Allocator,
 
@@ -69,19 +69,20 @@ pub fn Reader(comptime EXPECT_MASK: bool, MAX_MESSAGE_SIZE: usize) type {
         // (any new reads must be placed after this)
         len: usize = 0,
 
-        // we add 140 to allow 1 control message (ping/pong/close) to be
-        // fragmented into a normal message.
+        max_message_size: usize,
+
         buf: []u8,
 
         fragments: ?Fragments = null,
 
         const Self = @This();
 
-        pub fn init(allocator: Allocator) !Self {
+        pub fn init(allocator: Allocator, max_message_size: usize) !Self {
             const buf = try allocator.alloc(u8, 16 * 1024);
             return .{
                 .buf = buf,
                 .allocator = allocator,
+                .max_message_size = max_message_size,
             };
         }
 
@@ -154,7 +155,8 @@ pub fn Reader(comptime EXPECT_MASK: bool, MAX_MESSAGE_SIZE: usize) type {
                     if (message_len > 125) {
                         return error.ControlTooLarge;
                     }
-                } else if (message_len > MAX_MESSAGE_SIZE) {
+                } else if (message_len > self.max_message_size) {
+                    lp.log.warn(.cdp, "CDP message too big", .{ .type = "WS", .len = message_len, .hint = "See the --cdp-max-message-size <bytes>" });
                     return error.TooLarge;
                 } else if (message_len > self.buf.len) {
                     const len = self.buf.len;
@@ -182,7 +184,9 @@ pub fn Reader(comptime EXPECT_MASK: bool, MAX_MESSAGE_SIZE: usize) type {
 
                 if (is_continuation) {
                     const fragments = &(self.fragments orelse return error.InvalidContinuation);
-                    if (fragments.message.items.len + message_len > MAX_MESSAGE_SIZE) {
+                    const full_len = fragments.message.items.len + message_len;
+                    if (full_len > self.max_message_size) {
+                        lp.log.warn(.cdp, "CDP message too big", .{ .type = "WS", .len = full_len, .hint = "See the --cdp-max-message-size <bytes>" });
                         return error.TooLarge;
                     }