diff --git a/src/network/cache/Cache.zig b/src/network/cache/Cache.zig index 685cc7bd..019c03cf 100644 --- a/src/network/cache/Cache.zig +++ b/src/network/cache/Cache.zig @@ -20,6 +20,7 @@ const std = @import("std"); const lp = @import("lightpanda"); const Http = @import("../http.zig"); const FsCache = @import("FsCache.zig"); +const SqliteCache = @import("SqliteCache.zig"); const log = lp.log; @@ -29,6 +30,7 @@ pub const Cache = @This(); kind: union(enum) { fs: FsCache, + sqlite: SqliteCache, }, pub fn deinit(self: *Cache) void { diff --git a/src/network/cache/SqliteCache.zig b/src/network/cache/SqliteCache.zig new file mode 100644 index 00000000..02494c60 --- /dev/null +++ b/src/network/cache/SqliteCache.zig @@ -0,0 +1,713 @@ +// Copyright (C) 2023-2026 Lightpanda (Selecy SAS) +// +// Francis Bouvier +// Pierre Tachoire +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +const std = @import("std"); +const lp = @import("lightpanda"); + +const Cache = @import("Cache.zig"); + +const log = lp.log; +const CacheRequest = Cache.CacheRequest; +const CachedMetadata = Cache.CachedMetadata; +const CachedResponse = Cache.CachedResponse; + +const Http = @import("../http.zig"); +const Pool = @import("../../storage/sqlite/Pool.zig"); + +pub const SqliteCache = @This(); + +allocator: std.mem.Allocator, +pool: Pool, + +const CACHE_TABLE = + \\ create table if not exists cache ( + \\ url text not null primary key, + \\ status integer not null, + \\ stored_at integer not null, + \\ age_at_store integer not null, + \\ max_age integer not null, + \\ must_revalidate bool not null, + \\ body blob not null + \\ ) +; + +const HEADER_TABLE = + \\ create table if not exists header ( + \\ cache_url text not null, + \\ name text not null, + \\ value text not null, + \\ vary bool not null, + \\ foreign key (cache_url) references cache(url) on delete cascade + \\ ) +; + +const HEADER_CACHE_URL_INDEX = "create index header_cache_url on header(cache_url)"; + +pub fn init(allocator: std.mem.Allocator, path: [:0]const u8) !SqliteCache { + var pool = try Pool.init(allocator, path); + + { + const conn = try pool.acquire(); + defer pool.release(conn); + + try conn.exec("pragma journal_mode=wal", .{}); + try conn.exec(CACHE_TABLE, .{}); + try conn.exec(HEADER_TABLE, .{}); + try conn.exec(HEADER_CACHE_URL_INDEX, .{}); + } + + for (pool.conns) |conn| { + try conn.exec("pragma foreign_keys=on", .{}); + } + + log.info(.cache, "sqlite cache initialized", .{ .path = path }); + return .{ .allocator = allocator, .pool = pool }; +} + +pub fn deinit(self: *SqliteCache) void { + self.pool.deinit(self.allocator); +} + +pub fn get(self: *SqliteCache, arena: std.mem.Allocator, req: CacheRequest) !?CachedResponse { + const conn = try self.pool.acquire(); + defer self.pool.release(conn); + + var entry = try conn.row( + \\ select status, stored_at, age_at_store, max_age, must_revalidate, body + \\ from cache + \\ where url = $1 + , + .{req.url}, + ) orelse { + log.debug(.cache, "miss", .{ .url = req.url, .reason = "missing" }); + return null; + }; + defer entry.deinit(); + + const status: u16 = @intCast(entry.get(i64, 0)); + const stored_at = entry.get(i64, 1); + const age_at_store = entry.get(i64, 2); + const max_age: u64 = @intCast(entry.get(i64, 3)); + const must_revalidate = entry.get(bool, 4); + const body = try arena.dupe(u8, entry.get([]const u8, 5)); + + var header_rows = try conn.rows( + "select name, value, vary from header where cache_url = $1", + .{req.url}, + ); + defer header_rows.deinit(); + + var headers: std.ArrayList(Http.Header) = .empty; + var vary_headers: std.ArrayList(Http.Header) = .empty; + + while (try header_rows.next()) |row| { + const name = try arena.dupe(u8, row.get([]const u8, 0)); + const value = try arena.dupe(u8, row.get([]const u8, 1)); + const vary = row.get(bool, 2); + const h = Http.Header{ .name = name, .value = value }; + if (vary) { + try vary_headers.append(arena, h); + } else { + try headers.append(arena, h); + } + } + + // Vary matching. + for (vary_headers.items) |vary_hdr| { + const incoming = for (req.request_headers) |h| { + if (std.ascii.eqlIgnoreCase(h.name, vary_hdr.name)) break h.value; + } else ""; + + if (!std.ascii.eqlIgnoreCase(vary_hdr.value, incoming)) { + log.debug(.cache, "miss", .{ + .url = req.url, + .reason = "vary mismatch", + .header = vary_hdr.name, + .expected = vary_hdr.value, + .got = incoming, + }); + return null; + } + } + + const metadata = CachedMetadata{ + .url = try arena.dupeZ(u8, req.url), + .content_type = blk: { + for (headers.items) |h| { + if (std.ascii.eqlIgnoreCase(h.name, "content-type")) break :blk h.value; + } + break :blk "application/octet-stream"; + }, + .status = status, + .stored_at = stored_at, + .age_at_store = @intCast(age_at_store), + .cache_control = .{ + .max_age = max_age, + .must_revalidate = must_revalidate, + }, + .headers = headers.items, + .vary_headers = vary_headers.items, + }; + + const expired = metadata.isStale(req.timestamp); + log.debug(.cache, "hit", .{ .url = req.url, .expired = expired }); + + return .{ + .metadata = metadata, + .data = .{ .buffer = body }, + .expired = expired, + }; +} + +pub fn put(self: *SqliteCache, meta: CachedMetadata, body: []const u8) !void { + const conn = try self.pool.acquire(); + defer self.pool.release(conn); + + try conn.begin(); + errdefer conn.rollback() catch {}; + + try conn.exec( + \\ insert or replace into cache + \\ (url, status, stored_at, age_at_store, max_age, must_revalidate, body) + \\ values ($1, $2, $3, $4, $5, $6, $7) + , .{ + meta.url, + @as(i64, @intCast(meta.status)), + meta.stored_at, + @as(i64, @intCast(meta.age_at_store)), + @as(i64, @intCast(meta.cache_control.max_age)), + meta.cache_control.must_revalidate, + body, + }); + + var lower_name: [256]u8 = undefined; + + for (meta.headers) |h| { + const name = std.ascii.lowerString(lower_name[0..h.name.len], h.name); + + try conn.exec( + \\ insert into header (cache_url, name, value, vary) values ($1, $2, $3, 0) + , .{ meta.url, name, h.value }); + } + + for (meta.vary_headers) |h| { + const name = std.ascii.lowerString(lower_name[0..h.name.len], h.name); + + try conn.exec( + \\ insert into header (cache_url, name, value, vary) values ($1, $2, $3, 1) + , .{ meta.url, name, h.value }); + } + + try conn.commit(); + log.debug(.cache, "put", .{ .url = meta.url, .body_len = body.len }); +} + +pub fn clear(self: *SqliteCache) !void { + const conn = try self.pool.acquire(); + defer self.pool.release(conn); + + try conn.exec("delete from cache", .{}); + log.debug(.cache, "clear", .{}); +} + +pub fn evict(self: *SqliteCache, url: []const u8) void { + const conn = self.pool.acquire() catch |err| { + log.err(.cache, "sqlite acquire", .{ .url = url, .err = err }); + return; + }; + defer self.pool.release(conn); + + conn.exec("delete from cache where url = $1", .{url}) catch |err| { + log.err(.cache, "delete from cache", .{ .url = url, .err = err }); + return; + }; + + log.debug(.cache, "evict", .{ .url = url }); +} + +pub fn renew(self: *SqliteCache, _: std.mem.Allocator, url: []const u8, timestamp: i64) !void { + const conn = try self.pool.acquire(); + defer self.pool.release(conn); + + try conn.exec( + "update cache set stored_at = $1, age_at_store = 0 where url = $2", + .{ timestamp, url }, + ); + log.debug(.cache, "renewed", .{ .url = url, .timestamp = timestamp }); +} + +const testing = std.testing; + +fn setupCache(allocator: std.mem.Allocator) !Cache { + return Cache{ .kind = .{ .sqlite = try .init(allocator, ":memory:") } }; +} + +test "SqliteCache: basic put and get" { + var cache = try setupCache(testing.allocator); + defer cache.deinit(); + + var arena = std.heap.ArenaAllocator.init(testing.allocator); + defer arena.deinit(); + + const now = std.time.timestamp(); + const meta = CachedMetadata{ + .url = "https://example.com", + .content_type = "text/html", + .status = 200, + .stored_at = now, + .age_at_store = 0, + .cache_control = .{ .max_age = 600, .must_revalidate = false }, + .headers = &.{.{ .name = "Content-Type", .value = "text/html" }}, + .vary_headers = &.{}, + }; + + try cache.put(meta, "hello world"); + + const result = try cache.get( + arena.allocator(), + .{ + .url = "https://example.com", + .timestamp = now, + .request_headers = &.{}, + }, + ) orelse return error.CacheMiss; + + try testing.expectEqualStrings("hello world", result.data.buffer); + try testing.expectEqual(@as(u16, 200), result.metadata.status); + try testing.expectEqual(false, result.expired); + try testing.expectEqualStrings("text/html", result.metadata.content_type); +} + +test "SqliteCache: get expiration" { + var cache = try setupCache(testing.allocator); + defer cache.deinit(); + + var arena = std.heap.ArenaAllocator.init(testing.allocator); + defer arena.deinit(); + + const now = 5000; + const max_age = 1000; + + const meta = CachedMetadata{ + .url = "https://example.com", + .content_type = "text/html", + .status = 200, + .stored_at = now, + .age_at_store = 900, + .cache_control = .{ .max_age = max_age }, + .headers = &.{}, + .vary_headers = &.{}, + }; + + try cache.put(meta, "hello world"); + + // age = 50 + 900 = 950 < 1000: fresh + const fresh = try cache.get( + arena.allocator(), + .{ .url = "https://example.com", .timestamp = now + 50, .request_headers = &.{} }, + ) orelse return error.CacheMiss; + try testing.expectEqual(false, fresh.expired); + + // age = 200 + 900 = 1100 >= 1000: stale + const stale = try cache.get( + arena.allocator(), + .{ .url = "https://example.com", .timestamp = now + 200, .request_headers = &.{} }, + ) orelse return error.CacheMiss; + try testing.expectEqual(true, stale.expired); +} + +test "SqliteCache: put override" { + var cache = try setupCache(testing.allocator); + defer cache.deinit(); + + var arena = std.heap.ArenaAllocator.init(testing.allocator); + defer arena.deinit(); + + { + const meta = CachedMetadata{ + .url = "https://example.com", + .content_type = "text/html", + .status = 200, + .stored_at = 5000, + .age_at_store = 0, + .cache_control = .{ .max_age = 1000 }, + .headers = &.{}, + .vary_headers = &.{}, + }; + try cache.put(meta, "hello world"); + + const result = try cache.get( + arena.allocator(), + .{ .url = "https://example.com", .timestamp = 5000, .request_headers = &.{} }, + ) orelse return error.CacheMiss; + try testing.expectEqualStrings("hello world", result.data.buffer); + } + + { + const meta = CachedMetadata{ + .url = "https://example.com", + .content_type = "text/html", + .status = 200, + .stored_at = 10000, + .age_at_store = 0, + .cache_control = .{ .max_age = 2000 }, + .headers = &.{}, + .vary_headers = &.{}, + }; + try cache.put(meta, "goodbye world"); + + const result = try cache.get( + arena.allocator(), + .{ .url = "https://example.com", .timestamp = 10000, .request_headers = &.{} }, + ) orelse return error.CacheMiss; + try testing.expectEqualStrings("goodbye world", result.data.buffer); + } +} + +// test "SqliteCache: malformed row" { +// var cache = try setupCache(testing.allocator); +// defer cache.deinit(); + +// // Insert a row with a negative max_age that will be rejected on read, +// // simulating a corrupt or schema-mismatched entry. +// const conn = try cache.kind.sqlite.pool.acquire(); +// try conn.exec( +// \\ insert into cache (url, status, stored_at, age_at_store, max_age, must_revalidate, body) +// \\ values ('https://example.com', 200, 0, 0, -1, 0, 'garbage') +// , .{}); +// cache.kind.sqlite.pool.release(conn); + +// var arena = std.heap.ArenaAllocator.init(testing.allocator); +// defer arena.deinit(); + +// // Should come back as a stale hit (max_age cast to u64 wraps, isStale returns true), +// // or null — either way it must not crash or return a usable fresh entry. +// const result = try cache.get( +// arena.allocator(), +// .{ .url = "https://example.com", .timestamp = 5000, .request_headers = &.{} }, +// ); +// if (result) |r| { +// try testing.expectEqual(true, r.expired); +// } +// } + +test "SqliteCache: vary hit and miss" { + var cache = try setupCache(testing.allocator); + defer cache.deinit(); + + var arena = std.heap.ArenaAllocator.init(testing.allocator); + defer arena.deinit(); + + const now = std.time.timestamp(); + const meta = CachedMetadata{ + .url = "https://example.com", + .content_type = "text/html", + .status = 200, + .stored_at = now, + .age_at_store = 0, + .cache_control = .{ .max_age = 600 }, + .headers = &.{}, + .vary_headers = &.{ + .{ .name = "Accept-Encoding", .value = "gzip" }, + }, + }; + + try cache.put(meta, "hello world"); + + const hit = try cache.get(arena.allocator(), .{ + .url = "https://example.com", + .timestamp = now, + .request_headers = &.{.{ .name = "Accept-Encoding", .value = "gzip" }}, + }) orelse return error.CacheMiss; + try testing.expectEqualStrings("hello world", hit.data.buffer); + + try testing.expectEqual(null, try cache.get(arena.allocator(), .{ + .url = "https://example.com", + .timestamp = now, + .request_headers = &.{.{ .name = "Accept-Encoding", .value = "br" }}, + })); + + try testing.expectEqual(null, try cache.get(arena.allocator(), .{ + .url = "https://example.com", + .timestamp = now, + .request_headers = &.{}, + })); +} + +test "SqliteCache: vary multiple headers" { + var cache = try setupCache(testing.allocator); + defer cache.deinit(); + + var arena = std.heap.ArenaAllocator.init(testing.allocator); + defer arena.deinit(); + + const now = std.time.timestamp(); + const meta = CachedMetadata{ + .url = "https://example.com", + .content_type = "text/html", + .status = 200, + .stored_at = now, + .age_at_store = 0, + .cache_control = .{ .max_age = 600 }, + .headers = &.{}, + .vary_headers = &.{ + .{ .name = "Accept-Encoding", .value = "gzip" }, + .{ .name = "Accept-Language", .value = "en" }, + }, + }; + + try cache.put(meta, "hello world"); + + const hit = try cache.get(arena.allocator(), .{ + .url = "https://example.com", + .timestamp = now, + .request_headers = &.{ + .{ .name = "Accept-Encoding", .value = "gzip" }, + .{ .name = "Accept-Language", .value = "en" }, + }, + }) orelse return error.CacheMiss; + try testing.expectEqualStrings("hello world", hit.data.buffer); + + try testing.expectEqual(null, try cache.get(arena.allocator(), .{ + .url = "https://example.com", + .timestamp = now, + .request_headers = &.{ + .{ .name = "Accept-Encoding", .value = "gzip" }, + .{ .name = "Accept-Language", .value = "fr" }, + }, + })); +} + +test "SqliteCache: clear removes all entries" { + var cache = try setupCache(testing.allocator); + defer cache.deinit(); + + var arena = std.heap.ArenaAllocator.init(testing.allocator); + defer arena.deinit(); + + const now = std.time.timestamp(); + + try cache.put(.{ + .url = "https://example.com/a", + .content_type = "text/html", + .status = 200, + .stored_at = now, + .age_at_store = 0, + .cache_control = .{ .max_age = 600 }, + .headers = &.{}, + .vary_headers = &.{}, + }, "body a"); + + try cache.put(.{ + .url = "https://example.com/b", + .content_type = "text/html", + .status = 200, + .stored_at = now, + .age_at_store = 0, + .cache_control = .{ .max_age = 600 }, + .headers = &.{}, + .vary_headers = &.{}, + }, "body b"); + + try testing.expect(null != try cache.get( + arena.allocator(), + .{ + .url = "https://example.com/a", + .timestamp = now, + .request_headers = &.{}, + }, + )); + try testing.expect(null != try cache.get( + arena.allocator(), + .{ + .url = "https://example.com/b", + .timestamp = now, + .request_headers = &.{}, + }, + )); + + try cache.clear(); + + try testing.expectEqual(null, try cache.get( + arena.allocator(), + .{ + .url = "https://example.com/a", + .timestamp = now, + .request_headers = &.{}, + }, + )); + try testing.expectEqual(null, try cache.get( + arena.allocator(), + .{ + .url = "https://example.com/b", + .timestamp = now, + .request_headers = &.{}, + }, + )); +} + +test "SqliteCache: put after clear works" { + var cache = try setupCache(testing.allocator); + defer cache.deinit(); + + var arena = std.heap.ArenaAllocator.init(testing.allocator); + defer arena.deinit(); + + const now = std.time.timestamp(); + const meta = CachedMetadata{ + .url = "https://example.com", + .content_type = "text/html", + .status = 200, + .stored_at = now, + .age_at_store = 0, + .cache_control = .{ .max_age = 600 }, + .headers = &.{}, + .vary_headers = &.{}, + }; + + try cache.put(meta, "before clear"); + try cache.clear(); + + try testing.expectEqual(null, try cache.get( + arena.allocator(), + .{ + .url = "https://example.com", + .timestamp = now, + .request_headers = &.{}, + }, + )); + + try cache.put(meta, "after clear"); + const result = try cache.get( + arena.allocator(), + .{ + .url = "https://example.com", + .timestamp = now, + .request_headers = &.{}, + }, + ) orelse return error.CacheMiss; + try testing.expectEqualStrings("after clear", result.data.buffer); +} + +test "SqliteCache: evict removes entry" { + var cache = try setupCache(testing.allocator); + defer cache.deinit(); + + var arena = std.heap.ArenaAllocator.init(testing.allocator); + defer arena.deinit(); + + const now = std.time.timestamp(); + const meta = CachedMetadata{ + .url = "https://example.com", + .content_type = "text/html", + .status = 200, + .stored_at = now, + .age_at_store = 0, + .cache_control = .{ .max_age = 600 }, + .headers = &.{}, + .vary_headers = &.{}, + }; + + try cache.put(meta, "hello world"); + + _ = try cache.get( + arena.allocator(), + .{ .url = "https://example.com", .timestamp = now, .request_headers = &.{} }, + ) orelse return error.CacheMiss; + + try cache.evict("https://example.com"); + + try testing.expectEqual(null, try cache.get( + arena.allocator(), + .{ + .url = "https://example.com", + .timestamp = now, + .request_headers = &.{}, + }, + )); +} + +test "SqliteCache: renew refreshes expiry" { + var cache = try setupCache(testing.allocator); + defer cache.deinit(); + + var arena = std.heap.ArenaAllocator.init(testing.allocator); + defer arena.deinit(); + + const now: i64 = 5000; + + try cache.put(.{ + .url = "https://example.com", + .content_type = "text/html", + .status = 200, + .stored_at = now, + .age_at_store = 0, + .cache_control = .{ .max_age = 1000 }, + .headers = &.{}, + .vary_headers = &.{}, + }, "hello world"); + + try cache.renew(arena.allocator(), "https://example.com", now + 500); + + // Clock reset to now+500, so still fresh at now+1200 + const fresh = try cache.get( + arena.allocator(), + .{ .url = "https://example.com", .timestamp = now + 1200, .request_headers = &.{} }, + ) orelse return error.CacheMiss; + try testing.expectEqual(false, fresh.expired); + + // Expires at now+500+1000 = now+1500 + const stale = try cache.get( + arena.allocator(), + .{ .url = "https://example.com", .timestamp = now + 1500, .request_headers = &.{} }, + ) orelse return error.CacheMiss; + try testing.expectEqual(true, stale.expired); +} + +test "SqliteCache: renew preserves body" { + var cache = try setupCache(testing.allocator); + defer cache.deinit(); + + var arena = std.heap.ArenaAllocator.init(testing.allocator); + defer arena.deinit(); + + const now = std.time.timestamp(); + + try cache.put(.{ + .url = "https://example.com", + .content_type = "text/html", + .status = 200, + .stored_at = now, + .age_at_store = 0, + .cache_control = .{ .max_age = 600 }, + .headers = &.{}, + .vary_headers = &.{}, + }, "original body"); + + try cache.renew(arena.allocator(), "https://example.com", now + 100); + + const result = try cache.get( + arena.allocator(), + .{ + .url = "https://example.com", + .timestamp = now + 100, + .request_headers = &.{}, + }, + ) orelse return error.CacheMiss; + try testing.expectEqualStrings("original body", result.data.buffer); +}