add internal flag to Request

src/browser/HttpClient.zig

@@ -1324,7 +1324,10 @@ pub const Request = struct {
     credentials: ?[:0]const u8 = null,
     notification: *Notification,
     timeout_ms: u32 = 0,
-    skip_robots: bool = false,
+
+    // Requests that are internal to the browser and skip various layers,
+    // these do not need to be deferred and do not obey robots.txt.
+    internal: bool = false,
 
     // When false, the caller does not guarantee that the body outlives the
     // transfer, and thus we'll need to dupe it.

src/network/layer/DeferringLayer.zig

@@ -50,6 +50,10 @@ pub fn deinit(self: *DeferringLayer) void {
 fn request(ptr: *anyopaque, transfer: *Transfer) anyerror!void {
     const self: *DeferringLayer = @ptrCast(@alignCast(ptr));
 
+    if (transfer.req.internal) {
+        return self.next.request(transfer);
+    }
+
     const arena = try self.network.app.arena_pool.acquire(.small, "DeferringContext");
     errdefer self.network.app.arena_pool.release(arena);
 

src/network/layer/RobotsLayer.zig

@@ -57,7 +57,7 @@ pub fn deinit(self: *RobotsLayer, allocator: Allocator) void {
 fn request(ptr: *anyopaque, transfer: *Transfer) anyerror!void {
     const self: *RobotsLayer = @ptrCast(@alignCast(ptr));
 
-    if (transfer.req.skip_robots) {
+    if (transfer.req.internal) {
         return self.next.request(transfer);
     }
 
@@ -132,7 +132,7 @@ fn fetchRobotsThenRequest(
         errdefer new_req.headers.deinit();
         new_req.method = .GET;
         new_req.url = robots_url;
-        new_req.skip_robots = true;
+        new_req.internal = true;
         new_req.resource_type = .fetch;
         new_req.body = null;
         new_req.ctx = robots_ctx;