Merge pull request #3494 from lightpanda-io/robot-store-leak

mem: fix robot store leak

src/network/Robots.zig

@@ -139,8 +139,17 @@ pub const RobotStore = struct {
         self.mutex.lockUncancelable(lp.io);
         defer self.mutex.unlock(lp.io);
 
-        const duped = try self.allocator.dupe(u8, url);
-        try self.map.put(self.allocator, duped, .{ .present = robots });
+        if (try self.putKey(url)) |value_ptr| {
+            // first time seeing this url, store the value
+            value_ptr.* = .{ .present = robots };
+            return;
+        }
+
+        // cannot overwrite an existing value, if it was `present`, we'd have
+        // to free the value but it might be being used.
+
+        var discarded = robots;
+        discarded.deinit(self.allocator);
     }
 
     // The returned slice is owned by the store
@@ -160,8 +169,12 @@ pub const RobotStore = struct {
         self.mutex.lockUncancelable(lp.io);
         defer self.mutex.unlock(lp.io);
 
-        const duped = try self.allocator.dupe(u8, url);
-        try self.map.put(self.allocator, duped, .allowed);
+        if (try self.putKey(url)) |value_ptr| {
+            // first time seeing this url, store the value
+            value_ptr.* = .allowed;
+        }
+        // cannot overwrite an existing value, if it was `present`, we'd have
+        // to free the value but it might be being used.
     }
 
     /// This URL is fully restricted from crawling.
@@ -169,8 +182,26 @@ pub const RobotStore = struct {
         self.mutex.lockUncancelable(lp.io);
         defer self.mutex.unlock(lp.io);
 
-        const duped = try self.allocator.dupe(u8, url);
-        try self.map.put(self.allocator, duped, .disallowed);
+        if (try self.putKey(url)) |value_ptr| {
+            // first time seeing this url, store the value
+            value_ptr.* = .disallowed;
+        }
+        // cannot overwrite an existing value, if it was `present`, we'd have
+        // to free the value but it might be being used.
+    }
+
+    // The RobotStore is shared across Browsers. Two rowsers can request the
+    // same robots URL at the same time, and they'll race here. First one wins.
+    // Caller holds the mutex.
+    fn putKey(self: *RobotStore, url: []const u8) !?*RobotsEntry {
+        const gop = try self.map.getOrPut(self.allocator, url);
+        if (gop.found_existing) {
+            // already have a value, caller should not overwrite
+            return null;
+        }
+        errdefer _ = self.map.remove(url);
+        gop.key_ptr.* = try self.allocator.dupe(u8, url);
+        return gop.value_ptr;
     }
 };
 

src/network/RobotsGate.zig

@@ -214,6 +214,7 @@ const RobotsContext = struct {
                     };
                     if (robots) |r| {
                         try network.robot_store.put(robots_url, r);
+                        // BE CAREFUL: robots can be invalidated after this call
                     }
                 } else {
                     // Empty robots.txt means we can short-circuit the allowed path.