better handling of allowed and disallowed robots cases

src/network/Robots.zig

@@ -92,7 +92,8 @@ pub const empty: Robots = .{ .rules = &.{}, .content_signals = &.{} };
 pub const RobotStore = struct {
     const RobotsEntry = union(enum) {
         present: Robots,
-        absent,
+        allowed,
+        disallowed,
     };
 
     pub const RobotsMap = std.HashMapUnmanaged([]const u8, RobotsEntry, struct {
@@ -141,7 +142,7 @@ pub const RobotStore = struct {
 
             switch (entry.value_ptr.*) {
                 .present => |*robots| robots.deinit(self.allocator),
-                .absent => {},
+                .allowed, .disallowed => {},
             }
         }
 
@@ -175,16 +176,26 @@ pub const RobotStore = struct {
         const entry = self.map.get(url) orelse return null;
         return switch (entry) {
             .present => |robots| robots.content_signals,
-            .absent => null,
+            .allowed, .disallowed => null,
         };
     }
 
-    pub fn putAbsent(self: *RobotStore, url: []const u8) !void {
+    /// This URL has no restrictions on crawling.
+    pub fn putAllowed(self: *RobotStore, url: []const u8) !void {
         self.mutex.lockUncancelable(lp.io);
         defer self.mutex.unlock(lp.io);
 
         const duped = try self.allocator.dupe(u8, url);
-        try self.map.put(self.allocator, duped, .absent);
+        try self.map.put(self.allocator, duped, .allowed);
+    }
+
+    /// This URL is fully restricted from crawling.
+    pub fn putDisallowed(self: *RobotStore, url: []const u8) !void {
+        self.mutex.lockUncancelable(lp.io);
+        defer self.mutex.unlock(lp.io);
+
+        const duped = try self.allocator.dupe(u8, url);
+        try self.map.put(self.allocator, duped, .disallowed);
     }
 };
 

src/network/RobotsGate.zig

@@ -53,7 +53,11 @@ pub fn check(self: *RobotsGate, transfer: *Transfer) !Result {
 
     if (self.network.robot_store.get(robots_url)) |robot_entry| {
         switch (robot_entry) {
-            .absent => return .allowed,
+            .allowed => return .allowed,
+            .disallowed => {
+                log.warn(.http, "blocked by robots", .{ .url = url });
+                return .blocked;
+            },
             .present => |robots| {
                 if (robots.isAllowed(URL.getPathname(url))) {
                     return .allowed;
@@ -142,7 +146,8 @@ fn flushPending(self: *RobotsGate, robots_url: []const u8) void {
         transfer.unpark();
 
         const allowed = if (robot_entry) |entry| switch (entry) {
-            .absent => true,
+            .allowed => true,
+            .disallowed => false,
             .present => |robots| robots.isAllowed(URL.getPathname(transfer.req.url)),
         } else true;
 
@@ -201,26 +206,41 @@ const RobotsContext = struct {
                     const robots: ?Robots = network.robot_store.robotsFromBytes(
                         network.config.http_headers.user_agent,
                         self.buffer.items,
-                    ) catch blk: {
-                        log.warn(.browser, "failed to parse robots", .{ .robots_url = robots_url });
-                        try network.robot_store.putAbsent(robots_url);
+                    ) catch |err| blk: {
+                        // We only return an error if an allocation or something fails.
+                        // Our parser does already leniently handle malformed input and takes whichever rules it can parse.
+                        // On this case of an allocation failure, it is our fault so we put it as disallowed.
+                        log.warn(.browser, "error while parsing robots.txt", .{ .robots_url = robots_url, .err = err });
+                        try network.robot_store.putDisallowed(robots_url);
                         break :blk null;
                     };
                     if (robots) |r| {
                         try network.robot_store.put(robots_url, r);
                     }
+                } else {
+                    // Empty robots.txt means we can short-circuit the allowed path.
+                    try network.robot_store.putAllowed(robots_url);
                 }
             },
-            404 => {
-                log.debug(.http, "robots not found", .{ .url = robots_url });
-                try network.robot_store.putAbsent(robots_url);
+            // RFC9309: Unavailable (400-499) means that we may access any resources on the server.
+            400...499 => {
+                log.debug(.http, "robots.txt unavailable", .{ .url = robots_url });
+                try network.robot_store.putAllowed(robots_url);
+            },
+            // RFC9309: Unreachable (500-599) means that we are completely disallowed.
+            500...599 => {
+                log.warn(.http, "robots.txt unreachable", .{
+                    .url = robots_url,
+                    .status = self.status,
+                });
+                try network.robot_store.putDisallowed(robots_url);
             },
             else => {
                 log.debug(.http, "unexpected status on robots", .{
                     .url = robots_url,
                     .status = self.status,
                 });
-                try network.robot_store.putAbsent(robots_url);
+                try network.robot_store.putDisallowed(robots_url);
             },
         }