add SqliteCache as default local impl

src/network/HttpClient.zig

@@ -905,7 +905,7 @@ fn cacheLookup(self: *Client, transfer: *Transfer) !bool {
         out.* = .{ .name = hdr.name, .value = hdr.value };
     }
 
-    const cached = cache.get(arena.allocator(), .{
+    const cached = try cache.get(arena.allocator(), .{
         .url = req.url,
         .timestamp = lp.datetime.timestamp(.real),
         .request_headers = req_headers,

src/network/Network.zig

@@ -35,7 +35,7 @@ const WebBotAuth = @import("WebBotAuth.zig");
 const CurlDebugAllocator = @import("CurlDebugAllocator.zig");
 
 const Cache = @import("cache/Cache.zig");
-const FsCache = @import("cache/FsCache.zig");
+const SqliteCache = @import("cache/SqliteCache.zig");
 
 const log = lp.log;
 const posix = std.posix;
@@ -233,9 +233,9 @@ pub fn init(allocator: Allocator, app: *App, config: *const Config) !Network {
     const cache = if (config.httpCacheDir()) |cache_dir_path|
         Cache{
             .kind = .{
-                .fs = FsCache.init(cache_dir_path) catch |e| {
+                .sqlite = SqliteCache.init(allocator, .{ .path = cache_dir_path }) catch |e| {
                     log.err(.cache, "failed to init", .{
-                        .kind = "FsCache",
+                        .kind = "SqliteCache",
                         .path = cache_dir_path,
                         .err = e,
                     });

src/network/cache/Cache.zig

@@ -19,7 +19,7 @@
 const std = @import("std");
 const lp = @import("lightpanda");
 const Http = @import("../http.zig");
-const FsCache = @import("FsCache.zig");
+const SqliteCache = @import("SqliteCache.zig");
 
 const log = lp.log;
 
@@ -28,7 +28,7 @@ const log = lp.log;
 pub const Cache = @This();
 
 kind: union(enum) {
-    fs: FsCache,
+    sqlite: SqliteCache,
 },
 
 pub fn deinit(self: *Cache) void {
@@ -37,7 +37,7 @@ pub fn deinit(self: *Cache) void {
     };
 }
 
-pub fn get(self: *Cache, arena: std.mem.Allocator, req: CacheRequest) ?CachedResponse {
+pub fn get(self: *Cache, arena: std.mem.Allocator, req: CacheRequest) !?CachedResponse {
     return switch (self.kind) {
         inline else => |*c| c.get(arena, req),
     };

src/network/cache/SqliteCache.zig

@@ -0,0 +1,833 @@
+// Copyright (C) 2023-2026  Lightpanda (Selecy SAS)
+//
+// Francis Bouvier <francis@lightpanda.io>
+// Pierre Tachoire <pierre@lightpanda.io>
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+const std = @import("std");
+const lp = @import("lightpanda");
+
+const Cache = @import("Cache.zig");
+
+const log = lp.log;
+const CacheRequest = Cache.CacheRequest;
+const RenewResponse = Cache.RenewResponse;
+const CachedMetadata = Cache.CachedMetadata;
+const CachedResponse = Cache.CachedResponse;
+
+const Http = @import("../http.zig");
+const Pool = @import("../../storage/sqlite/Pool.zig");
+const Conn = @import("../../storage/sqlite/Sqlite.zig").Conn;
+const Migration = @import("../../storage/sqlite/Sqlite.zig").Migration;
+const Migrations = @import("../../storage/sqlite/Sqlite.zig").Migrations;
+
+pub const SqliteCache = @This();
+
+allocator: std.mem.Allocator,
+pool: Pool,
+
+const cache_migrations: []const Migration = &.{
+    .{ .sql =
+    \\ create table metadata (
+    \\      url               text not null primary key,
+    \\      status            integer not null,
+    \\      stored_at         integer not null,
+    \\      age_at_store      integer not null,
+    \\      max_age           integer not null,
+    \\      must_revalidate   bool not null,
+    \\      etag              text,
+    \\      last_modified     text
+    \\ )
+    },
+    .{ .sql =
+    \\ create table body (
+    \\      url               text not null primary key,
+    \\      data              blob not null,
+    \\      foreign key (url) references metadata(url) on delete cascade
+    \\ )
+    },
+    .{ .sql =
+    \\ create table header (
+    \\      url               text not null,
+    \\      name              text not null,
+    \\      value             text not null,
+    \\      vary              bool not null,
+    \\      primary key (url, name),
+    \\      foreign key (url) references metadata(url) on delete cascade
+    \\ )
+    },
+    .{ .sql = "create index header_url on header(url)" },
+};
+
+pub const SqliteCachePath = union(enum) { path: []const u8, memory };
+
+pub fn init(allocator: std.mem.Allocator, path: SqliteCachePath) !SqliteCache {
+    var pool = switch (path) {
+        .memory => try Pool.init(allocator, ":memory:"),
+        .path => |cache_dir| blk: {
+            std.Io.Dir.cwd().createDirPath(lp.io, cache_dir) catch |e| {
+                log.err(
+                    .cache,
+                    "failed to make path",
+                    .{ .kind = "httpCacheSqlitePath", .path = cache_dir, .err = e },
+                );
+                return e;
+            };
+
+            const full_path = try std.fmt.allocPrintSentinel(
+                allocator,
+                "{s}/cache.db",
+                .{std.mem.trimEnd(u8, cache_dir, &.{'/'})},
+                0,
+            );
+            defer allocator.free(full_path);
+            break :blk try Pool.init(allocator, full_path);
+        },
+    };
+    errdefer pool.deinit(allocator);
+
+    var version: usize = 0;
+
+    {
+        const conn = try pool.acquire();
+        defer pool.release(conn);
+
+        try conn.exec("pragma journal_mode=wal", .{});
+        version = try Migrations.run(conn, cache_migrations);
+    }
+
+    for (pool.conns) |conn| {
+        try conn.exec("pragma foreign_keys=on", .{});
+    }
+
+    log.info(.cache, "sqlite cache initialized", .{ .path = path, .version = version });
+    return .{ .allocator = allocator, .pool = pool };
+}
+
+pub fn deinit(self: *SqliteCache) void {
+    self.pool.deinit(self.allocator);
+}
+
+fn loadMetadata(conn: Conn, arena: std.mem.Allocator, url: []const u8) !?CachedMetadata {
+    var entry = try conn.row(
+        \\ select status, stored_at, age_at_store,
+        \\      max_age, must_revalidate, etag, last_modified
+        \\ from metadata
+        \\ where url = $1
+    , .{url}) orelse return null;
+    defer entry.deinit();
+
+    const status: u16 = @intCast(entry.get(i64, 0));
+    const stored_at: u64 = @intCast(entry.get(i64, 1));
+    const age_at_store = entry.get(i64, 2);
+    const max_age: u64 = @intCast(entry.get(i64, 3));
+    const must_revalidate = entry.get(bool, 4);
+    const etag = if (entry.get(?[]const u8, 5)) |opt| try arena.dupe(u8, opt) else null;
+    const last_modified = if (entry.get(?[]const u8, 6)) |opt| try arena.dupe(u8, opt) else null;
+
+    var header_rows = try conn.rows(
+        "select name, value, vary from header where url = $1",
+        .{url},
+    );
+    defer header_rows.deinit();
+
+    var headers: std.ArrayList(Http.Header) = .empty;
+    var vary_headers: std.ArrayList(Http.Header) = .empty;
+    var content_type: []const u8 = "application/octet-stream";
+
+    while (try header_rows.next()) |row| {
+        const name = try arena.dupe(u8, row.get([]const u8, 0));
+        const value = try arena.dupe(u8, row.get([]const u8, 1));
+        const vary = row.get(bool, 2);
+
+        if (std.ascii.eqlIgnoreCase(name, "content-type")) {
+            content_type = value;
+        }
+
+        const h = Http.Header{ .name = name, .value = value };
+        if (vary) {
+            try vary_headers.append(arena, h);
+        } else {
+            try headers.append(arena, h);
+        }
+    }
+
+    return CachedMetadata{
+        .url = try arena.dupeZ(u8, url),
+        .content_type = content_type,
+        .status = status,
+        .stored_at = stored_at,
+        .age_at_store = @intCast(age_at_store),
+        .cache_control = .{
+            .max_age = max_age,
+            .must_revalidate = must_revalidate,
+        },
+        .headers = headers.items,
+        .vary_headers = vary_headers.items,
+        .etag = etag,
+        .last_modified = last_modified,
+    };
+}
+
+fn loadBody(conn: Conn, arena: std.mem.Allocator, url: []const u8) ![]const u8 {
+    const body_entry = try conn.row(
+        "select data from body where url = $1",
+        .{url},
+    ) orelse @panic("valid metadata must have a body");
+    defer body_entry.deinit();
+
+    return try arena.dupe(u8, body_entry.get([]const u8, 0));
+}
+
+fn insertMetadata(conn: Conn, meta: CachedMetadata, body: []const u8) !void {
+    try conn.exec(
+        \\ insert or replace into metadata
+        \\     (url, status, stored_at, age_at_store, max_age, must_revalidate, etag, last_modified)
+        \\ values ($1, $2, $3, $4, $5, $6, $7, $8)
+    , .{
+        meta.url,
+        @as(i64, @intCast(meta.status)),
+        meta.stored_at,
+        @as(i64, @intCast(meta.age_at_store)),
+        @as(i64, @intCast(meta.cache_control.max_age)),
+        meta.cache_control.must_revalidate,
+        meta.etag,
+        meta.last_modified,
+    });
+
+    try conn.exec(
+        "insert into body (url, data) values ($1, $2)",
+        .{ meta.url, body },
+    );
+
+    var lower_name: [256]u8 = undefined;
+    for (meta.headers) |h| {
+        if (h.name.len > lower_name.len) return error.HeaderNameTooLong;
+        const name = std.ascii.lowerString(lower_name[0..h.name.len], h.name);
+        try conn.exec(
+            "insert into header (url, name, value, vary) values ($1, $2, $3, false)",
+            .{ meta.url, name, h.value },
+        );
+    }
+    for (meta.vary_headers) |h| {
+        if (h.name.len > lower_name.len) return error.HeaderNameTooLong;
+        const name = std.ascii.lowerString(lower_name[0..h.name.len], h.name);
+        try conn.exec(
+            "insert into header (url, name, value, vary) values ($1, $2, $3, true)",
+            .{ meta.url, name, h.value },
+        );
+    }
+}
+
+fn updateMetadata(conn: Conn, meta: CachedMetadata) !void {
+    try conn.exec(
+        \\ update metadata
+        \\ set status = $1, stored_at = $2, age_at_store = $3, max_age = $4, must_revalidate = $5, etag = $6, last_modified = $7
+        \\ where url = $8
+    , .{
+        @as(i64, @intCast(meta.status)),
+        meta.stored_at,
+        @as(i64, @intCast(meta.age_at_store)),
+        @as(i64, @intCast(meta.cache_control.max_age)),
+        meta.cache_control.must_revalidate,
+        meta.etag,
+        meta.last_modified,
+        meta.url,
+    });
+
+    try conn.exec("delete from header where url = $1 and vary = false", .{meta.url});
+
+    var lower_name: [256]u8 = undefined;
+    for (meta.headers) |h| {
+        if (h.name.len > lower_name.len) return error.HeaderNameTooLong;
+        const name = std.ascii.lowerString(lower_name[0..h.name.len], h.name);
+        try conn.exec(
+            "insert into header (url, name, value, vary) values ($1, $2, $3, false)",
+            .{ meta.url, name, h.value },
+        );
+    }
+}
+
+pub fn get(self: *SqliteCache, arena: std.mem.Allocator, req: CacheRequest) !?CachedResponse {
+    const conn = try self.pool.acquire();
+    defer self.pool.release(conn);
+
+    try conn.begin();
+    defer conn.rollback() catch {};
+
+    const metadata = try loadMetadata(conn, arena, req.url) orelse {
+        log.debug(.cache, "miss", .{ .url = req.url, .reason = "missing" });
+        return null;
+    };
+
+    // Vary matching.
+    for (metadata.vary_headers) |vary_hdr| {
+        const incoming = for (req.request_headers) |h| {
+            if (std.ascii.eqlIgnoreCase(h.name, vary_hdr.name)) break h.value;
+        } else "";
+
+        if (!std.ascii.eqlIgnoreCase(vary_hdr.value, incoming)) {
+            log.debug(.cache, "miss", .{
+                .url = req.url,
+                .reason = "vary mismatch",
+                .header = vary_hdr.name,
+                .expected = vary_hdr.value,
+                .got = incoming,
+            });
+            return null;
+        }
+    }
+
+    const body = try loadBody(conn, arena, req.url);
+    const expired = metadata.isStale(req.timestamp);
+    log.debug(.cache, "hit", .{ .url = req.url, .expired = expired });
+
+    return .{
+        .metadata = metadata,
+        .data = .{ .buffer = body },
+        .expired = expired,
+    };
+}
+
+pub fn put(self: *SqliteCache, meta: CachedMetadata, body: []const u8) !void {
+    const conn = try self.pool.acquire();
+    defer self.pool.release(conn);
+
+    try conn.begin();
+    errdefer conn.rollback() catch {};
+
+    try insertMetadata(conn, meta, body);
+    try conn.commit();
+
+    log.debug(.cache, "put", .{ .url = meta.url, .body_len = body.len });
+}
+
+pub fn clear(self: *SqliteCache) !void {
+    const conn = try self.pool.acquire();
+    defer self.pool.release(conn);
+
+    try conn.exec("delete from metadata", .{});
+    log.debug(.cache, "clear", .{});
+}
+
+pub fn evict(self: *SqliteCache, url: []const u8) void {
+    const conn = self.pool.acquire() catch |err| {
+        log.err(.cache, "sqlite acquire", .{ .url = url, .err = err });
+        return;
+    };
+    defer self.pool.release(conn);
+
+    conn.exec("delete from metadata where url = $1", .{url}) catch |err| {
+        log.err(.cache, "delete from cache", .{ .url = url, .err = err });
+        return;
+    };
+
+    log.debug(.cache, "evict", .{ .url = url });
+}
+
+pub fn renew(self: *SqliteCache, arena: std.mem.Allocator, req: RenewResponse) !void {
+    const conn = try self.pool.acquire();
+    defer self.pool.release(conn);
+
+    try conn.begin();
+    errdefer conn.rollback() catch {};
+
+    var metadata = try loadMetadata(conn, arena, req.url) orelse {
+        log.debug(.cache, "miss", .{ .url = req.url, .reason = "missing" });
+        return error.CacheEntryNotFound;
+    };
+
+    metadata.renew(req);
+
+    try updateMetadata(conn, metadata);
+    try conn.commit();
+
+    log.debug(.cache, "renewed", .{
+        .url = req.url,
+        .timestamp = req.timestamp,
+    });
+}
+
+const testing = std.testing;
+
+fn setupCache(allocator: std.mem.Allocator) !Cache {
+    return Cache{ .kind = .{ .sqlite = try .init(allocator, .memory) } };
+}
+
+test "SqliteCache: Migrations" {
+    const allocator = testing.allocator;
+    var pool = try Pool.init(allocator, ":memory:");
+    defer pool.deinit(allocator);
+
+    const conn = try pool.acquire();
+    defer pool.release(conn);
+
+    _ = try Migrations.run(conn, cache_migrations);
+}
+
+test "SqliteCache: basic put and get" {
+    var cache = try setupCache(testing.allocator);
+    defer cache.deinit();
+
+    var arena = std.heap.ArenaAllocator.init(testing.allocator);
+    defer arena.deinit();
+
+    const now: u64 = @intCast(std.Io.Timestamp.now(testing.io, .boot).toSeconds());
+    const meta = CachedMetadata{
+        .url = "https://example.com",
+        .content_type = "text/html",
+        .status = 200,
+        .stored_at = now,
+        .age_at_store = 0,
+        .cache_control = .{ .max_age = 600, .must_revalidate = false },
+        .headers = &.{.{ .name = "Content-Type", .value = "text/html" }},
+        .vary_headers = &.{},
+    };
+
+    try cache.put(meta, "hello world");
+
+    const result = try cache.get(
+        arena.allocator(),
+        .{
+            .url = "https://example.com",
+            .timestamp = now,
+            .request_headers = &.{},
+        },
+    ) orelse return error.CacheMiss;
+
+    try testing.expectEqualStrings("hello world", result.data.buffer);
+    try testing.expectEqual(@as(u16, 200), result.metadata.status);
+    try testing.expectEqual(false, result.expired);
+    try testing.expectEqualStrings("text/html", result.metadata.content_type);
+}
+
+test "SqliteCache: get expiration" {
+    var cache = try setupCache(testing.allocator);
+    defer cache.deinit();
+
+    var arena = std.heap.ArenaAllocator.init(testing.allocator);
+    defer arena.deinit();
+
+    const now = 5000;
+    const max_age = 1000;
+
+    const meta = CachedMetadata{
+        .url = "https://example.com",
+        .content_type = "text/html",
+        .status = 200,
+        .stored_at = now,
+        .age_at_store = 900,
+        .cache_control = .{ .max_age = max_age },
+        .headers = &.{},
+        .vary_headers = &.{},
+    };
+
+    try cache.put(meta, "hello world");
+
+    // age = 50 + 900 = 950 < 1000: fresh
+    const fresh = try cache.get(
+        arena.allocator(),
+        .{
+            .url = "https://example.com",
+            .timestamp = now + 50,
+            .request_headers = &.{},
+        },
+    ) orelse return error.CacheMiss;
+    try testing.expectEqual(false, fresh.expired);
+
+    // age = 200 + 900 = 1100 >= 1000: stale
+    const stale = try cache.get(
+        arena.allocator(),
+        .{
+            .url = "https://example.com",
+            .timestamp = now + 200,
+            .request_headers = &.{},
+        },
+    ) orelse return error.CacheMiss;
+    try testing.expectEqual(true, stale.expired);
+}
+
+test "SqliteCache: put override" {
+    var cache = try setupCache(testing.allocator);
+    defer cache.deinit();
+
+    var arena = std.heap.ArenaAllocator.init(testing.allocator);
+    defer arena.deinit();
+
+    {
+        const meta = CachedMetadata{
+            .url = "https://example.com",
+            .content_type = "text/html",
+            .status = 200,
+            .stored_at = 5000,
+            .age_at_store = 0,
+            .cache_control = .{ .max_age = 1000 },
+            .headers = &.{},
+            .vary_headers = &.{},
+        };
+        try cache.put(meta, "hello world");
+
+        const result = try cache.get(
+            arena.allocator(),
+            .{
+                .url = "https://example.com",
+                .timestamp = 5000,
+                .request_headers = &.{},
+            },
+        ) orelse return error.CacheMiss;
+        try testing.expectEqualStrings("hello world", result.data.buffer);
+    }
+
+    {
+        const meta = CachedMetadata{
+            .url = "https://example.com",
+            .content_type = "text/html",
+            .status = 200,
+            .stored_at = 10000,
+            .age_at_store = 0,
+            .cache_control = .{ .max_age = 2000 },
+            .headers = &.{},
+            .vary_headers = &.{},
+        };
+        try cache.put(meta, "goodbye world");
+
+        const result = try cache.get(
+            arena.allocator(),
+            .{
+                .url = "https://example.com",
+                .timestamp = 10000,
+                .request_headers = &.{},
+            },
+        ) orelse return error.CacheMiss;
+        try testing.expectEqualStrings("goodbye world", result.data.buffer);
+    }
+}
+
+test "SqliteCache: vary hit and miss" {
+    var cache = try setupCache(testing.allocator);
+    defer cache.deinit();
+
+    var arena = std.heap.ArenaAllocator.init(testing.allocator);
+    defer arena.deinit();
+
+    const now: u64 = @intCast(std.Io.Timestamp.now(testing.io, .boot).toSeconds());
+    const meta = CachedMetadata{
+        .url = "https://example.com",
+        .content_type = "text/html",
+        .status = 200,
+        .stored_at = now,
+        .age_at_store = 0,
+        .cache_control = .{ .max_age = 600 },
+        .headers = &.{},
+        .vary_headers = &.{
+            .{ .name = "Accept-Encoding", .value = "gzip" },
+        },
+    };
+
+    try cache.put(meta, "hello world");
+
+    const hit = try cache.get(arena.allocator(), .{
+        .url = "https://example.com",
+        .timestamp = now,
+        .request_headers = &.{.{ .name = "Accept-Encoding", .value = "gzip" }},
+    }) orelse return error.CacheMiss;
+    try testing.expectEqualStrings("hello world", hit.data.buffer);
+
+    try testing.expectEqual(null, try cache.get(arena.allocator(), .{
+        .url = "https://example.com",
+        .timestamp = now,
+        .request_headers = &.{.{ .name = "Accept-Encoding", .value = "br" }},
+    }));
+
+    try testing.expectEqual(null, try cache.get(arena.allocator(), .{
+        .url = "https://example.com",
+        .timestamp = now,
+        .request_headers = &.{},
+    }));
+}
+
+test "SqliteCache: vary multiple headers" {
+    var cache = try setupCache(testing.allocator);
+    defer cache.deinit();
+
+    var arena = std.heap.ArenaAllocator.init(testing.allocator);
+    defer arena.deinit();
+
+    const now: u64 = @intCast(std.Io.Timestamp.now(testing.io, .boot).toSeconds());
+    const meta = CachedMetadata{
+        .url = "https://example.com",
+        .content_type = "text/html",
+        .status = 200,
+        .stored_at = now,
+        .age_at_store = 0,
+        .cache_control = .{ .max_age = 600 },
+        .headers = &.{},
+        .vary_headers = &.{
+            .{ .name = "Accept-Encoding", .value = "gzip" },
+            .{ .name = "Accept-Language", .value = "en" },
+        },
+    };
+
+    try cache.put(meta, "hello world");
+
+    const hit = try cache.get(arena.allocator(), .{
+        .url = "https://example.com",
+        .timestamp = now,
+        .request_headers = &.{
+            .{ .name = "Accept-Encoding", .value = "gzip" },
+            .{ .name = "Accept-Language", .value = "en" },
+        },
+    }) orelse return error.CacheMiss;
+    try testing.expectEqualStrings("hello world", hit.data.buffer);
+
+    try testing.expectEqual(null, try cache.get(arena.allocator(), .{
+        .url = "https://example.com",
+        .timestamp = now,
+        .request_headers = &.{
+            .{ .name = "Accept-Encoding", .value = "gzip" },
+            .{ .name = "Accept-Language", .value = "fr" },
+        },
+    }));
+}
+
+test "SqliteCache: clear removes all entries" {
+    var cache = try setupCache(testing.allocator);
+    defer cache.deinit();
+
+    var arena = std.heap.ArenaAllocator.init(testing.allocator);
+    defer arena.deinit();
+
+    const now: u64 = @intCast(std.Io.Timestamp.now(testing.io, .boot).toSeconds());
+    try cache.put(.{
+        .url = "https://example.com/a",
+        .content_type = "text/html",
+        .status = 200,
+        .stored_at = now,
+        .age_at_store = 0,
+        .cache_control = .{ .max_age = 600 },
+        .headers = &.{},
+        .vary_headers = &.{},
+    }, "body a");
+
+    try cache.put(.{
+        .url = "https://example.com/b",
+        .content_type = "text/html",
+        .status = 200,
+        .stored_at = now,
+        .age_at_store = 0,
+        .cache_control = .{ .max_age = 600 },
+        .headers = &.{},
+        .vary_headers = &.{},
+    }, "body b");
+
+    try testing.expect(null != try cache.get(
+        arena.allocator(),
+        .{
+            .url = "https://example.com/a",
+            .timestamp = now,
+            .request_headers = &.{},
+        },
+    ));
+    try testing.expect(null != try cache.get(
+        arena.allocator(),
+        .{
+            .url = "https://example.com/b",
+            .timestamp = now,
+            .request_headers = &.{},
+        },
+    ));
+
+    try cache.clear();
+
+    try testing.expectEqual(null, try cache.get(
+        arena.allocator(),
+        .{
+            .url = "https://example.com/a",
+            .timestamp = now,
+            .request_headers = &.{},
+        },
+    ));
+    try testing.expectEqual(null, try cache.get(
+        arena.allocator(),
+        .{
+            .url = "https://example.com/b",
+            .timestamp = now,
+            .request_headers = &.{},
+        },
+    ));
+}
+
+test "SqliteCache: put after clear works" {
+    var cache = try setupCache(testing.allocator);
+    defer cache.deinit();
+
+    var arena = std.heap.ArenaAllocator.init(testing.allocator);
+    defer arena.deinit();
+
+    const now: u64 = @intCast(std.Io.Timestamp.now(testing.io, .boot).toSeconds());
+    const meta = CachedMetadata{
+        .url = "https://example.com",
+        .content_type = "text/html",
+        .status = 200,
+        .stored_at = now,
+        .age_at_store = 0,
+        .cache_control = .{ .max_age = 600 },
+        .headers = &.{},
+        .vary_headers = &.{},
+    };
+
+    try cache.put(meta, "before clear");
+    try cache.clear();
+
+    try testing.expectEqual(null, try cache.get(
+        arena.allocator(),
+        .{
+            .url = "https://example.com",
+            .timestamp = now,
+            .request_headers = &.{},
+        },
+    ));
+
+    try cache.put(meta, "after clear");
+    const result = try cache.get(
+        arena.allocator(),
+        .{
+            .url = "https://example.com",
+            .timestamp = now,
+            .request_headers = &.{},
+        },
+    ) orelse return error.CacheMiss;
+    try testing.expectEqualStrings("after clear", result.data.buffer);
+}
+
+test "SqliteCache: evict removes entry" {
+    var cache = try setupCache(testing.allocator);
+    defer cache.deinit();
+
+    var arena = std.heap.ArenaAllocator.init(testing.allocator);
+    defer arena.deinit();
+
+    const now: u64 = @intCast(std.Io.Timestamp.now(testing.io, .boot).toSeconds());
+    const meta = CachedMetadata{
+        .url = "https://example.com",
+        .content_type = "text/html",
+        .status = 200,
+        .stored_at = now,
+        .age_at_store = 0,
+        .cache_control = .{ .max_age = 600 },
+        .headers = &.{},
+        .vary_headers = &.{},
+    };
+
+    try cache.put(meta, "hello world");
+
+    _ = try cache.get(
+        arena.allocator(),
+        .{ .url = "https://example.com", .timestamp = now, .request_headers = &.{} },
+    ) orelse return error.CacheMiss;
+
+    cache.evict("https://example.com");
+
+    try testing.expectEqual(null, try cache.get(
+        arena.allocator(),
+        .{
+            .url = "https://example.com",
+            .timestamp = now,
+            .request_headers = &.{},
+        },
+    ));
+}
+
+test "SqliteCache: renew refreshes expiry" {
+    var cache = try setupCache(testing.allocator);
+    defer cache.deinit();
+
+    var arena = std.heap.ArenaAllocator.init(testing.allocator);
+    defer arena.deinit();
+
+    const now: i64 = 5000;
+    try cache.put(.{
+        .url = "https://example.com",
+        .content_type = "text/html",
+        .status = 200,
+        .stored_at = now,
+        .age_at_store = 0,
+        .cache_control = .{ .max_age = 1000 },
+        .headers = &.{},
+        .vary_headers = &.{},
+    }, "hello world");
+
+    try cache.renew(
+        arena.allocator(),
+        .{ .url = "https://example.com", .timestamp = now + 500, .headers = &.{} },
+    );
+
+    // Clock reset to now+500, so still fresh at now+1200
+    const fresh = try cache.get(
+        arena.allocator(),
+        .{
+            .url = "https://example.com",
+            .timestamp = now + 1200,
+            .request_headers = &.{},
+        },
+    ) orelse return error.CacheMiss;
+    try testing.expectEqual(false, fresh.expired);
+
+    // Expires at now+500+1000 = now+1500
+    const stale = try cache.get(
+        arena.allocator(),
+        .{
+            .url = "https://example.com",
+            .timestamp = now + 1500,
+            .request_headers = &.{},
+        },
+    ) orelse return error.CacheMiss;
+    try testing.expectEqual(true, stale.expired);
+}
+
+test "SqliteCache: renew preserves body" {
+    var cache = try setupCache(testing.allocator);
+    defer cache.deinit();
+
+    var arena = std.heap.ArenaAllocator.init(testing.allocator);
+    defer arena.deinit();
+
+    const now: u64 = @intCast(std.Io.Timestamp.now(testing.io, .boot).toSeconds());
+    try cache.put(.{
+        .url = "https://example.com",
+        .content_type = "text/html",
+        .status = 200,
+        .stored_at = now,
+        .age_at_store = 0,
+        .cache_control = .{ .max_age = 600 },
+        .headers = &.{},
+        .vary_headers = &.{},
+    }, "original body");
+
+    try cache.renew(
+        arena.allocator(),
+        .{ .url = "https://example.com", .timestamp = now + 100, .headers = &.{} },
+    );
+
+    const result = try cache.get(
+        arena.allocator(),
+        .{
+            .url = "https://example.com",
+            .timestamp = now + 100,
+            .request_headers = &.{},
+        },
+    ) orelse return error.CacheMiss;
+    try testing.expectEqualStrings("original body", result.data.buffer);
+}

src/storage/Storage.zig

@@ -28,12 +28,10 @@ const Storage = @This();
 
 pub const EngineType = enum {
     none,
-    sqlite,
 };
 
 const Engine = union(EngineType) {
     none: Blackhole,
-    sqlite: Sqlite,
 };
 
 engine: Engine,
@@ -50,13 +48,9 @@ pub fn init(allocator: Allocator, config: *const Config) !Storage {
     };
 }
 
-fn initEngine(allocator: Allocator, engine_type: EngineType, config: *const Config) !Engine {
+fn initEngine(_: Allocator, engine_type: EngineType, _: *const Config) !Engine {
     switch (engine_type) {
         .none => return .{ .none = Blackhole{} },
-        .sqlite => {
-            const sqlite_path = config.storageSqlitePath();
-            return .{ .sqlite = try Sqlite.init(allocator, sqlite_path) };
-        },
     }
 }
 

src/storage/sqlite/Sqlite.zig

@@ -27,32 +27,60 @@ const Allocator = std.mem.Allocator;
 
 const Sqlite = @This();
 
-pool: Pool,
+pub const Migration = union(enum) {
+    sql: [:0]const u8,
+    func: struct {
+        ctx: *anyopaque,
+        func: *const fn (conn: Conn, ctx: *anyopaque) anyerror!void,
+    },
+};
 
-pub fn init(allocator: Allocator, path_: ?[:0]const u8) !Sqlite {
-    const path = path_ orelse ":memory:";
-    var pool = try Pool.init(allocator, path);
-    errdefer pool.deinit(allocator);
+pub const Migrations = struct {
+    pub fn run(conn: Conn, migrations: []const Migration) !usize {
+        try conn.exec(
+            \\create table if not exists migrations (
+            \\  id integer primary key,
+            \\  applied_at integer not null
+            \\) strict
+        , .{});
 
-    {
-        // copy by value warning! The connection HAS to be returned to the
-        // pool in this scope. If we didn't have this scope, we'd assign the
-        // pool to the return value (copy A) and then release the original
-        const conn = try pool.acquire();
-        defer pool.release(conn);
+        const current = (try conn.scalar(
+            i64,
+            "select max(id) from migrations",
+            .{},
+        )) orelse 0;
+        const start: usize = @intCast(current);
 
-        const version = try @import("migrations.zig").run(conn);
-        log.info(.storage, "storage initialized", .{ .engine = "sqlite", .version = version, .path = path });
+        if (start > migrations.len) {
+            log.err(.storage, "migrations removed", .{
+                .applied = start,
+                .defined = migrations.len,
+            });
+            return error.MigrationsRemoved;
+        }
+
+        if (start == migrations.len) {
+            return start;
+        }
+
+        try conn.begin();
+        errdefer conn.rollback() catch {};
+
+        for (migrations[start..], start..) |migration, i| {
+            switch (migration) {
+                .sql => |sql| try conn.exec(sql, .{}),
+                .func => |f| try f.func(conn, f.ctx),
+            }
+            try conn.exec(
+                "insert into migrations (id, applied_at) values ($1, $2)",
+                .{ @as(i64, @intCast(i + 1)), std.Io.Timestamp.now(lp.io, .boot).toMilliseconds() },
+            );
+        }
+
+        try conn.commit();
+        return migrations.len;
     }
-
-    return .{
-        .pool = pool,
-    };
-}
-
-pub fn deinit(self: *Sqlite, allocator: Allocator) void {
-    self.pool.deinit(allocator);
-}
+};
 
 pub const Conn = struct {
     conn: *c.sqlite3,
@@ -138,6 +166,18 @@ pub const Conn = struct {
         return .{ .stmt = stmt.?, .conn = self.conn };
     }
 
+    pub fn begin(self: Conn) !void {
+        try self.exec("begin", .{});
+    }
+
+    pub fn commit(self: Conn) !void {
+        try self.exec("commit", .{});
+    }
+
+    pub fn rollback(self: Conn) !void {
+        try self.exec("rollback", .{});
+    }
+
     pub fn busyTimeout(self: Conn, ms: c_int) !void {
         const rc = c.sqlite3_busy_timeout(self.conn, ms);
         if (rc != c.SQLITE_OK) {
@@ -568,12 +608,42 @@ test "Sqlite: exec, row and scalar" {
     }
 }
 
-test "Sqlite: Migration" {
-    var sqlite = try Sqlite.init(testing.allocator, ":memory:");
-    defer sqlite.deinit(testing.allocator);
+test "Sqlite: Migrations - basic" {
+    var conn = try Sqlite.Conn.open(":memory:");
+    defer conn.close();
 
-    const conn = try sqlite.pool.acquire();
-    defer sqlite.pool.release(conn);
+    const migrations: []const Migration = &.{
+        .{ .sql = "create table test (id integer primary key, name text)" },
+        .{ .sql = "alter table test add column email text" },
+    };
 
-    try testing.expectEqual(1, (try conn.scalar(i64, "select max(id) from migrations", .{})).?);
+    const v1 = try Migrations.run(conn, migrations);
+    try testing.expectEqual(@as(usize, 2), v1);
+
+    // idempotent - running again should return same version
+    const v2 = try Migrations.run(conn, migrations);
+    try testing.expectEqual(@as(usize, 2), v2);
+
+    // verify migrations table has correct entries
+    try testing.expectEqual(
+        @as(i64, 2),
+        (try conn.scalar(i64, "select count(*) from migrations", .{})).?,
+    );
+}
+
+test "Sqlite: Migrations - removed migration" {
+    var conn = try Sqlite.Conn.open(":memory:");
+    defer conn.close();
+
+    const m1: []const Migration = &.{
+        .{ .sql = "create table test (id integer primary key, name text)" },
+        .{ .sql = "alter table test add column email text" },
+    };
+    _ = try Migrations.run(conn, m1);
+
+    // fewer migrations than were applied
+    const m2: []const Migration = &.{
+        .{ .sql = "create table test (id integer primary key, name text)" },
+    };
+    try testing.expectError(error.MigrationsRemoved, Migrations.run(conn, m2));
 }

src/storage/sqlite/migrations.zig

@@ -1,62 +0,0 @@
-// Copyright (C) 2023-2026  Lightpanda (Selecy SAS)
-//
-// Francis Bouvier <francis@lightpanda.io>
-// Pierre Tachoire <pierre@lightpanda.io>
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as
-// published by the Free Software Foundation, either version 3 of the
-// License, or (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <https://www.gnu.org/licenses/>.
-
-const lp = @import("lightpanda");
-
-const Sqlite = @import("Sqlite.zig");
-
-const log = lp.log;
-
-pub fn run(conn: Sqlite.Conn) !i64 {
-    const version = try getVersion(conn);
-    return version;
-}
-
-fn getVersion(conn: Sqlite.Conn) !i64 {
-    const exists_sql = "select exists (select 1 from sqlite_schema where type='table' and name='migrations')";
-    if (try conn.scalar(bool, exists_sql, .{}) orelse false) {
-        if (try conn.scalar(i64, "select max(id) from migrations", .{})) |version| {
-            return version;
-        }
-
-        log.fatal(.storage, "corrupt database", .{ .engine = "sqlite", .note = "The sqlite database has an existing but empty `migrations` table" });
-        return error.CorruptDatabase;
-    }
-
-    // this pragma is one of the the few (if not only) one that's persisted, so
-    // we only have to do it the first time.
-    conn.exec("pragma journal_mode=wal", .{}) catch |err| {
-        log.fatal(.storage, "migrate", .{
-            .err = err,
-            .step = "journal_mode",
-            .sqlite = conn.lastError(),
-        });
-        return err;
-    };
-
-    const create_sql =
-        \\ create table migrations as
-        \\ select 1 as id, current_timestamp as created_at
-    ;
-    conn.exec(create_sql, .{}) catch |err| {
-        log.fatal(.storage, "migrate", .{ .err = err, .sqlite = conn.lastError(), .step = "create migrations" });
-        return err;
-    };
-
-    return 1;
-}