diff --git a/src/browser/HttpClient.zig b/src/browser/HttpClient.zig index ecba705eb..3a7acb1ea 100644 --- a/src/browser/HttpClient.zig +++ b/src/browser/HttpClient.zig @@ -1340,7 +1340,10 @@ pub const Request = struct { credentials: ?[:0]const u8 = null, notification: *Notification, timeout_ms: u32 = 0, - skip_robots: bool = false, + + // Requests that are internal to the browser and skip various layers, + // these do not need to be deferred and do not obey robots.txt. + internal: bool = false, // When false, the caller does not guarantee that the body outlives the // transfer, and thus we'll need to dupe it. diff --git a/src/network/layer/DeferringLayer.zig b/src/network/layer/DeferringLayer.zig index d6f708224..beee867a5 100644 --- a/src/network/layer/DeferringLayer.zig +++ b/src/network/layer/DeferringLayer.zig @@ -51,6 +51,10 @@ pub fn deinit(self: *DeferringLayer) void { fn request(ptr: *anyopaque, transfer: *Transfer) anyerror!void { const self: *DeferringLayer = @ptrCast(@alignCast(ptr)); + if (transfer.req.internal) { + return self.next.request(transfer); + } + const arena = try self.network.app.arena_pool.acquire(.small, "DeferringContext"); errdefer self.network.app.arena_pool.release(arena); diff --git a/src/network/layer/RobotsLayer.zig b/src/network/layer/RobotsLayer.zig index a6f3165eb..eddccbb49 100644 --- a/src/network/layer/RobotsLayer.zig +++ b/src/network/layer/RobotsLayer.zig @@ -58,7 +58,7 @@ pub fn deinit(self: *RobotsLayer, allocator: Allocator) void { fn request(ptr: *anyopaque, transfer: *Transfer) anyerror!void { const self: *RobotsLayer = @ptrCast(@alignCast(ptr)); - if (transfer.req.skip_robots) { + if (transfer.req.internal) { return self.next.request(transfer); } @@ -133,7 +133,7 @@ fn fetchRobotsThenRequest( errdefer new_req.headers.deinit(); new_req.method = .GET; new_req.url = robots_url; - new_req.skip_robots = true; + new_req.internal = true; new_req.resource_type = .fetch; new_req.body = null; new_req.ctx = robots_ctx;