From 0f2957cda2a3d2bfb8f61ec0b0b34295e877ff47 Mon Sep 17 00:00:00 2001 From: Muki Kiboigo Date: Wed, 15 Jul 2026 06:09:15 -0700 Subject: [PATCH] add SqliteCache as default local impl --- src/network/HttpClient.zig | 2 +- src/network/Network.zig | 6 +- src/network/cache/Cache.zig | 6 +- src/network/cache/SqliteCache.zig | 833 ++++++++++++++++++++++++++++++ src/storage/Storage.zig | 8 +- src/storage/sqlite/Sqlite.zig | 126 ++++- src/storage/sqlite/migrations.zig | 62 --- 7 files changed, 939 insertions(+), 104 deletions(-) create mode 100644 src/network/cache/SqliteCache.zig delete mode 100644 src/storage/sqlite/migrations.zig diff --git a/src/network/HttpClient.zig b/src/network/HttpClient.zig index 57da9a0f6..89d020e7b 100644 --- a/src/network/HttpClient.zig +++ b/src/network/HttpClient.zig @@ -905,7 +905,7 @@ fn cacheLookup(self: *Client, transfer: *Transfer) !bool { out.* = .{ .name = hdr.name, .value = hdr.value }; } - const cached = cache.get(arena.allocator(), .{ + const cached = try cache.get(arena.allocator(), .{ .url = req.url, .timestamp = lp.datetime.timestamp(.real), .request_headers = req_headers, diff --git a/src/network/Network.zig b/src/network/Network.zig index 153011b9c..ceaf93f0d 100644 --- a/src/network/Network.zig +++ b/src/network/Network.zig @@ -35,7 +35,7 @@ const WebBotAuth = @import("WebBotAuth.zig"); const CurlDebugAllocator = @import("CurlDebugAllocator.zig"); const Cache = @import("cache/Cache.zig"); -const FsCache = @import("cache/FsCache.zig"); +const SqliteCache = @import("cache/SqliteCache.zig"); const log = lp.log; const posix = std.posix; @@ -233,9 +233,9 @@ pub fn init(allocator: Allocator, app: *App, config: *const Config) !Network { const cache = if (config.httpCacheDir()) |cache_dir_path| Cache{ .kind = .{ - .fs = FsCache.init(cache_dir_path) catch |e| { + .sqlite = SqliteCache.init(allocator, .{ .path = cache_dir_path }) catch |e| { log.err(.cache, "failed to init", .{ - .kind = "FsCache", + .kind = "SqliteCache", .path = cache_dir_path, .err = e, }); diff --git a/src/network/cache/Cache.zig b/src/network/cache/Cache.zig index ebe95ca19..159984f08 100644 --- a/src/network/cache/Cache.zig +++ b/src/network/cache/Cache.zig @@ -19,7 +19,7 @@ const std = @import("std"); const lp = @import("lightpanda"); const Http = @import("../http.zig"); -const FsCache = @import("FsCache.zig"); +const SqliteCache = @import("SqliteCache.zig"); const log = lp.log; @@ -28,7 +28,7 @@ const log = lp.log; pub const Cache = @This(); kind: union(enum) { - fs: FsCache, + sqlite: SqliteCache, }, pub fn deinit(self: *Cache) void { @@ -37,7 +37,7 @@ pub fn deinit(self: *Cache) void { }; } -pub fn get(self: *Cache, arena: std.mem.Allocator, req: CacheRequest) ?CachedResponse { +pub fn get(self: *Cache, arena: std.mem.Allocator, req: CacheRequest) !?CachedResponse { return switch (self.kind) { inline else => |*c| c.get(arena, req), }; diff --git a/src/network/cache/SqliteCache.zig b/src/network/cache/SqliteCache.zig new file mode 100644 index 000000000..c9cfbc9e5 --- /dev/null +++ b/src/network/cache/SqliteCache.zig @@ -0,0 +1,833 @@ +// Copyright (C) 2023-2026 Lightpanda (Selecy SAS) +// +// Francis Bouvier +// Pierre Tachoire +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +const std = @import("std"); +const lp = @import("lightpanda"); + +const Cache = @import("Cache.zig"); + +const log = lp.log; +const CacheRequest = Cache.CacheRequest; +const RenewResponse = Cache.RenewResponse; +const CachedMetadata = Cache.CachedMetadata; +const CachedResponse = Cache.CachedResponse; + +const Http = @import("../http.zig"); +const Pool = @import("../../storage/sqlite/Pool.zig"); +const Conn = @import("../../storage/sqlite/Sqlite.zig").Conn; +const Migration = @import("../../storage/sqlite/Sqlite.zig").Migration; +const Migrations = @import("../../storage/sqlite/Sqlite.zig").Migrations; + +pub const SqliteCache = @This(); + +allocator: std.mem.Allocator, +pool: Pool, + +const cache_migrations: []const Migration = &.{ + .{ .sql = + \\ create table metadata ( + \\ url text not null primary key, + \\ status integer not null, + \\ stored_at integer not null, + \\ age_at_store integer not null, + \\ max_age integer not null, + \\ must_revalidate bool not null, + \\ etag text, + \\ last_modified text + \\ ) + }, + .{ .sql = + \\ create table body ( + \\ url text not null primary key, + \\ data blob not null, + \\ foreign key (url) references metadata(url) on delete cascade + \\ ) + }, + .{ .sql = + \\ create table header ( + \\ url text not null, + \\ name text not null, + \\ value text not null, + \\ vary bool not null, + \\ primary key (url, name), + \\ foreign key (url) references metadata(url) on delete cascade + \\ ) + }, + .{ .sql = "create index header_url on header(url)" }, +}; + +pub const SqliteCachePath = union(enum) { path: []const u8, memory }; + +pub fn init(allocator: std.mem.Allocator, path: SqliteCachePath) !SqliteCache { + var pool = switch (path) { + .memory => try Pool.init(allocator, ":memory:"), + .path => |cache_dir| blk: { + std.Io.Dir.cwd().createDirPath(lp.io, cache_dir) catch |e| { + log.err( + .cache, + "failed to make path", + .{ .kind = "httpCacheSqlitePath", .path = cache_dir, .err = e }, + ); + return e; + }; + + const full_path = try std.fmt.allocPrintSentinel( + allocator, + "{s}/cache.db", + .{std.mem.trimEnd(u8, cache_dir, &.{'/'})}, + 0, + ); + defer allocator.free(full_path); + break :blk try Pool.init(allocator, full_path); + }, + }; + errdefer pool.deinit(allocator); + + var version: usize = 0; + + { + const conn = try pool.acquire(); + defer pool.release(conn); + + try conn.exec("pragma journal_mode=wal", .{}); + version = try Migrations.run(conn, cache_migrations); + } + + for (pool.conns) |conn| { + try conn.exec("pragma foreign_keys=on", .{}); + } + + log.info(.cache, "sqlite cache initialized", .{ .path = path, .version = version }); + return .{ .allocator = allocator, .pool = pool }; +} + +pub fn deinit(self: *SqliteCache) void { + self.pool.deinit(self.allocator); +} + +fn loadMetadata(conn: Conn, arena: std.mem.Allocator, url: []const u8) !?CachedMetadata { + var entry = try conn.row( + \\ select status, stored_at, age_at_store, + \\ max_age, must_revalidate, etag, last_modified + \\ from metadata + \\ where url = $1 + , .{url}) orelse return null; + defer entry.deinit(); + + const status: u16 = @intCast(entry.get(i64, 0)); + const stored_at: u64 = @intCast(entry.get(i64, 1)); + const age_at_store = entry.get(i64, 2); + const max_age: u64 = @intCast(entry.get(i64, 3)); + const must_revalidate = entry.get(bool, 4); + const etag = if (entry.get(?[]const u8, 5)) |opt| try arena.dupe(u8, opt) else null; + const last_modified = if (entry.get(?[]const u8, 6)) |opt| try arena.dupe(u8, opt) else null; + + var header_rows = try conn.rows( + "select name, value, vary from header where url = $1", + .{url}, + ); + defer header_rows.deinit(); + + var headers: std.ArrayList(Http.Header) = .empty; + var vary_headers: std.ArrayList(Http.Header) = .empty; + var content_type: []const u8 = "application/octet-stream"; + + while (try header_rows.next()) |row| { + const name = try arena.dupe(u8, row.get([]const u8, 0)); + const value = try arena.dupe(u8, row.get([]const u8, 1)); + const vary = row.get(bool, 2); + + if (std.ascii.eqlIgnoreCase(name, "content-type")) { + content_type = value; + } + + const h = Http.Header{ .name = name, .value = value }; + if (vary) { + try vary_headers.append(arena, h); + } else { + try headers.append(arena, h); + } + } + + return CachedMetadata{ + .url = try arena.dupeZ(u8, url), + .content_type = content_type, + .status = status, + .stored_at = stored_at, + .age_at_store = @intCast(age_at_store), + .cache_control = .{ + .max_age = max_age, + .must_revalidate = must_revalidate, + }, + .headers = headers.items, + .vary_headers = vary_headers.items, + .etag = etag, + .last_modified = last_modified, + }; +} + +fn loadBody(conn: Conn, arena: std.mem.Allocator, url: []const u8) ![]const u8 { + const body_entry = try conn.row( + "select data from body where url = $1", + .{url}, + ) orelse @panic("valid metadata must have a body"); + defer body_entry.deinit(); + + return try arena.dupe(u8, body_entry.get([]const u8, 0)); +} + +fn insertMetadata(conn: Conn, meta: CachedMetadata, body: []const u8) !void { + try conn.exec( + \\ insert or replace into metadata + \\ (url, status, stored_at, age_at_store, max_age, must_revalidate, etag, last_modified) + \\ values ($1, $2, $3, $4, $5, $6, $7, $8) + , .{ + meta.url, + @as(i64, @intCast(meta.status)), + meta.stored_at, + @as(i64, @intCast(meta.age_at_store)), + @as(i64, @intCast(meta.cache_control.max_age)), + meta.cache_control.must_revalidate, + meta.etag, + meta.last_modified, + }); + + try conn.exec( + "insert into body (url, data) values ($1, $2)", + .{ meta.url, body }, + ); + + var lower_name: [256]u8 = undefined; + for (meta.headers) |h| { + if (h.name.len > lower_name.len) return error.HeaderNameTooLong; + const name = std.ascii.lowerString(lower_name[0..h.name.len], h.name); + try conn.exec( + "insert into header (url, name, value, vary) values ($1, $2, $3, false)", + .{ meta.url, name, h.value }, + ); + } + for (meta.vary_headers) |h| { + if (h.name.len > lower_name.len) return error.HeaderNameTooLong; + const name = std.ascii.lowerString(lower_name[0..h.name.len], h.name); + try conn.exec( + "insert into header (url, name, value, vary) values ($1, $2, $3, true)", + .{ meta.url, name, h.value }, + ); + } +} + +fn updateMetadata(conn: Conn, meta: CachedMetadata) !void { + try conn.exec( + \\ update metadata + \\ set status = $1, stored_at = $2, age_at_store = $3, max_age = $4, must_revalidate = $5, etag = $6, last_modified = $7 + \\ where url = $8 + , .{ + @as(i64, @intCast(meta.status)), + meta.stored_at, + @as(i64, @intCast(meta.age_at_store)), + @as(i64, @intCast(meta.cache_control.max_age)), + meta.cache_control.must_revalidate, + meta.etag, + meta.last_modified, + meta.url, + }); + + try conn.exec("delete from header where url = $1 and vary = false", .{meta.url}); + + var lower_name: [256]u8 = undefined; + for (meta.headers) |h| { + if (h.name.len > lower_name.len) return error.HeaderNameTooLong; + const name = std.ascii.lowerString(lower_name[0..h.name.len], h.name); + try conn.exec( + "insert into header (url, name, value, vary) values ($1, $2, $3, false)", + .{ meta.url, name, h.value }, + ); + } +} + +pub fn get(self: *SqliteCache, arena: std.mem.Allocator, req: CacheRequest) !?CachedResponse { + const conn = try self.pool.acquire(); + defer self.pool.release(conn); + + try conn.begin(); + defer conn.rollback() catch {}; + + const metadata = try loadMetadata(conn, arena, req.url) orelse { + log.debug(.cache, "miss", .{ .url = req.url, .reason = "missing" }); + return null; + }; + + // Vary matching. + for (metadata.vary_headers) |vary_hdr| { + const incoming = for (req.request_headers) |h| { + if (std.ascii.eqlIgnoreCase(h.name, vary_hdr.name)) break h.value; + } else ""; + + if (!std.ascii.eqlIgnoreCase(vary_hdr.value, incoming)) { + log.debug(.cache, "miss", .{ + .url = req.url, + .reason = "vary mismatch", + .header = vary_hdr.name, + .expected = vary_hdr.value, + .got = incoming, + }); + return null; + } + } + + const body = try loadBody(conn, arena, req.url); + const expired = metadata.isStale(req.timestamp); + log.debug(.cache, "hit", .{ .url = req.url, .expired = expired }); + + return .{ + .metadata = metadata, + .data = .{ .buffer = body }, + .expired = expired, + }; +} + +pub fn put(self: *SqliteCache, meta: CachedMetadata, body: []const u8) !void { + const conn = try self.pool.acquire(); + defer self.pool.release(conn); + + try conn.begin(); + errdefer conn.rollback() catch {}; + + try insertMetadata(conn, meta, body); + try conn.commit(); + + log.debug(.cache, "put", .{ .url = meta.url, .body_len = body.len }); +} + +pub fn clear(self: *SqliteCache) !void { + const conn = try self.pool.acquire(); + defer self.pool.release(conn); + + try conn.exec("delete from metadata", .{}); + log.debug(.cache, "clear", .{}); +} + +pub fn evict(self: *SqliteCache, url: []const u8) void { + const conn = self.pool.acquire() catch |err| { + log.err(.cache, "sqlite acquire", .{ .url = url, .err = err }); + return; + }; + defer self.pool.release(conn); + + conn.exec("delete from metadata where url = $1", .{url}) catch |err| { + log.err(.cache, "delete from cache", .{ .url = url, .err = err }); + return; + }; + + log.debug(.cache, "evict", .{ .url = url }); +} + +pub fn renew(self: *SqliteCache, arena: std.mem.Allocator, req: RenewResponse) !void { + const conn = try self.pool.acquire(); + defer self.pool.release(conn); + + try conn.begin(); + errdefer conn.rollback() catch {}; + + var metadata = try loadMetadata(conn, arena, req.url) orelse { + log.debug(.cache, "miss", .{ .url = req.url, .reason = "missing" }); + return error.CacheEntryNotFound; + }; + + metadata.renew(req); + + try updateMetadata(conn, metadata); + try conn.commit(); + + log.debug(.cache, "renewed", .{ + .url = req.url, + .timestamp = req.timestamp, + }); +} + +const testing = std.testing; + +fn setupCache(allocator: std.mem.Allocator) !Cache { + return Cache{ .kind = .{ .sqlite = try .init(allocator, .memory) } }; +} + +test "SqliteCache: Migrations" { + const allocator = testing.allocator; + var pool = try Pool.init(allocator, ":memory:"); + defer pool.deinit(allocator); + + const conn = try pool.acquire(); + defer pool.release(conn); + + _ = try Migrations.run(conn, cache_migrations); +} + +test "SqliteCache: basic put and get" { + var cache = try setupCache(testing.allocator); + defer cache.deinit(); + + var arena = std.heap.ArenaAllocator.init(testing.allocator); + defer arena.deinit(); + + const now: u64 = @intCast(std.Io.Timestamp.now(testing.io, .boot).toSeconds()); + const meta = CachedMetadata{ + .url = "https://example.com", + .content_type = "text/html", + .status = 200, + .stored_at = now, + .age_at_store = 0, + .cache_control = .{ .max_age = 600, .must_revalidate = false }, + .headers = &.{.{ .name = "Content-Type", .value = "text/html" }}, + .vary_headers = &.{}, + }; + + try cache.put(meta, "hello world"); + + const result = try cache.get( + arena.allocator(), + .{ + .url = "https://example.com", + .timestamp = now, + .request_headers = &.{}, + }, + ) orelse return error.CacheMiss; + + try testing.expectEqualStrings("hello world", result.data.buffer); + try testing.expectEqual(@as(u16, 200), result.metadata.status); + try testing.expectEqual(false, result.expired); + try testing.expectEqualStrings("text/html", result.metadata.content_type); +} + +test "SqliteCache: get expiration" { + var cache = try setupCache(testing.allocator); + defer cache.deinit(); + + var arena = std.heap.ArenaAllocator.init(testing.allocator); + defer arena.deinit(); + + const now = 5000; + const max_age = 1000; + + const meta = CachedMetadata{ + .url = "https://example.com", + .content_type = "text/html", + .status = 200, + .stored_at = now, + .age_at_store = 900, + .cache_control = .{ .max_age = max_age }, + .headers = &.{}, + .vary_headers = &.{}, + }; + + try cache.put(meta, "hello world"); + + // age = 50 + 900 = 950 < 1000: fresh + const fresh = try cache.get( + arena.allocator(), + .{ + .url = "https://example.com", + .timestamp = now + 50, + .request_headers = &.{}, + }, + ) orelse return error.CacheMiss; + try testing.expectEqual(false, fresh.expired); + + // age = 200 + 900 = 1100 >= 1000: stale + const stale = try cache.get( + arena.allocator(), + .{ + .url = "https://example.com", + .timestamp = now + 200, + .request_headers = &.{}, + }, + ) orelse return error.CacheMiss; + try testing.expectEqual(true, stale.expired); +} + +test "SqliteCache: put override" { + var cache = try setupCache(testing.allocator); + defer cache.deinit(); + + var arena = std.heap.ArenaAllocator.init(testing.allocator); + defer arena.deinit(); + + { + const meta = CachedMetadata{ + .url = "https://example.com", + .content_type = "text/html", + .status = 200, + .stored_at = 5000, + .age_at_store = 0, + .cache_control = .{ .max_age = 1000 }, + .headers = &.{}, + .vary_headers = &.{}, + }; + try cache.put(meta, "hello world"); + + const result = try cache.get( + arena.allocator(), + .{ + .url = "https://example.com", + .timestamp = 5000, + .request_headers = &.{}, + }, + ) orelse return error.CacheMiss; + try testing.expectEqualStrings("hello world", result.data.buffer); + } + + { + const meta = CachedMetadata{ + .url = "https://example.com", + .content_type = "text/html", + .status = 200, + .stored_at = 10000, + .age_at_store = 0, + .cache_control = .{ .max_age = 2000 }, + .headers = &.{}, + .vary_headers = &.{}, + }; + try cache.put(meta, "goodbye world"); + + const result = try cache.get( + arena.allocator(), + .{ + .url = "https://example.com", + .timestamp = 10000, + .request_headers = &.{}, + }, + ) orelse return error.CacheMiss; + try testing.expectEqualStrings("goodbye world", result.data.buffer); + } +} + +test "SqliteCache: vary hit and miss" { + var cache = try setupCache(testing.allocator); + defer cache.deinit(); + + var arena = std.heap.ArenaAllocator.init(testing.allocator); + defer arena.deinit(); + + const now: u64 = @intCast(std.Io.Timestamp.now(testing.io, .boot).toSeconds()); + const meta = CachedMetadata{ + .url = "https://example.com", + .content_type = "text/html", + .status = 200, + .stored_at = now, + .age_at_store = 0, + .cache_control = .{ .max_age = 600 }, + .headers = &.{}, + .vary_headers = &.{ + .{ .name = "Accept-Encoding", .value = "gzip" }, + }, + }; + + try cache.put(meta, "hello world"); + + const hit = try cache.get(arena.allocator(), .{ + .url = "https://example.com", + .timestamp = now, + .request_headers = &.{.{ .name = "Accept-Encoding", .value = "gzip" }}, + }) orelse return error.CacheMiss; + try testing.expectEqualStrings("hello world", hit.data.buffer); + + try testing.expectEqual(null, try cache.get(arena.allocator(), .{ + .url = "https://example.com", + .timestamp = now, + .request_headers = &.{.{ .name = "Accept-Encoding", .value = "br" }}, + })); + + try testing.expectEqual(null, try cache.get(arena.allocator(), .{ + .url = "https://example.com", + .timestamp = now, + .request_headers = &.{}, + })); +} + +test "SqliteCache: vary multiple headers" { + var cache = try setupCache(testing.allocator); + defer cache.deinit(); + + var arena = std.heap.ArenaAllocator.init(testing.allocator); + defer arena.deinit(); + + const now: u64 = @intCast(std.Io.Timestamp.now(testing.io, .boot).toSeconds()); + const meta = CachedMetadata{ + .url = "https://example.com", + .content_type = "text/html", + .status = 200, + .stored_at = now, + .age_at_store = 0, + .cache_control = .{ .max_age = 600 }, + .headers = &.{}, + .vary_headers = &.{ + .{ .name = "Accept-Encoding", .value = "gzip" }, + .{ .name = "Accept-Language", .value = "en" }, + }, + }; + + try cache.put(meta, "hello world"); + + const hit = try cache.get(arena.allocator(), .{ + .url = "https://example.com", + .timestamp = now, + .request_headers = &.{ + .{ .name = "Accept-Encoding", .value = "gzip" }, + .{ .name = "Accept-Language", .value = "en" }, + }, + }) orelse return error.CacheMiss; + try testing.expectEqualStrings("hello world", hit.data.buffer); + + try testing.expectEqual(null, try cache.get(arena.allocator(), .{ + .url = "https://example.com", + .timestamp = now, + .request_headers = &.{ + .{ .name = "Accept-Encoding", .value = "gzip" }, + .{ .name = "Accept-Language", .value = "fr" }, + }, + })); +} + +test "SqliteCache: clear removes all entries" { + var cache = try setupCache(testing.allocator); + defer cache.deinit(); + + var arena = std.heap.ArenaAllocator.init(testing.allocator); + defer arena.deinit(); + + const now: u64 = @intCast(std.Io.Timestamp.now(testing.io, .boot).toSeconds()); + try cache.put(.{ + .url = "https://example.com/a", + .content_type = "text/html", + .status = 200, + .stored_at = now, + .age_at_store = 0, + .cache_control = .{ .max_age = 600 }, + .headers = &.{}, + .vary_headers = &.{}, + }, "body a"); + + try cache.put(.{ + .url = "https://example.com/b", + .content_type = "text/html", + .status = 200, + .stored_at = now, + .age_at_store = 0, + .cache_control = .{ .max_age = 600 }, + .headers = &.{}, + .vary_headers = &.{}, + }, "body b"); + + try testing.expect(null != try cache.get( + arena.allocator(), + .{ + .url = "https://example.com/a", + .timestamp = now, + .request_headers = &.{}, + }, + )); + try testing.expect(null != try cache.get( + arena.allocator(), + .{ + .url = "https://example.com/b", + .timestamp = now, + .request_headers = &.{}, + }, + )); + + try cache.clear(); + + try testing.expectEqual(null, try cache.get( + arena.allocator(), + .{ + .url = "https://example.com/a", + .timestamp = now, + .request_headers = &.{}, + }, + )); + try testing.expectEqual(null, try cache.get( + arena.allocator(), + .{ + .url = "https://example.com/b", + .timestamp = now, + .request_headers = &.{}, + }, + )); +} + +test "SqliteCache: put after clear works" { + var cache = try setupCache(testing.allocator); + defer cache.deinit(); + + var arena = std.heap.ArenaAllocator.init(testing.allocator); + defer arena.deinit(); + + const now: u64 = @intCast(std.Io.Timestamp.now(testing.io, .boot).toSeconds()); + const meta = CachedMetadata{ + .url = "https://example.com", + .content_type = "text/html", + .status = 200, + .stored_at = now, + .age_at_store = 0, + .cache_control = .{ .max_age = 600 }, + .headers = &.{}, + .vary_headers = &.{}, + }; + + try cache.put(meta, "before clear"); + try cache.clear(); + + try testing.expectEqual(null, try cache.get( + arena.allocator(), + .{ + .url = "https://example.com", + .timestamp = now, + .request_headers = &.{}, + }, + )); + + try cache.put(meta, "after clear"); + const result = try cache.get( + arena.allocator(), + .{ + .url = "https://example.com", + .timestamp = now, + .request_headers = &.{}, + }, + ) orelse return error.CacheMiss; + try testing.expectEqualStrings("after clear", result.data.buffer); +} + +test "SqliteCache: evict removes entry" { + var cache = try setupCache(testing.allocator); + defer cache.deinit(); + + var arena = std.heap.ArenaAllocator.init(testing.allocator); + defer arena.deinit(); + + const now: u64 = @intCast(std.Io.Timestamp.now(testing.io, .boot).toSeconds()); + const meta = CachedMetadata{ + .url = "https://example.com", + .content_type = "text/html", + .status = 200, + .stored_at = now, + .age_at_store = 0, + .cache_control = .{ .max_age = 600 }, + .headers = &.{}, + .vary_headers = &.{}, + }; + + try cache.put(meta, "hello world"); + + _ = try cache.get( + arena.allocator(), + .{ .url = "https://example.com", .timestamp = now, .request_headers = &.{} }, + ) orelse return error.CacheMiss; + + cache.evict("https://example.com"); + + try testing.expectEqual(null, try cache.get( + arena.allocator(), + .{ + .url = "https://example.com", + .timestamp = now, + .request_headers = &.{}, + }, + )); +} + +test "SqliteCache: renew refreshes expiry" { + var cache = try setupCache(testing.allocator); + defer cache.deinit(); + + var arena = std.heap.ArenaAllocator.init(testing.allocator); + defer arena.deinit(); + + const now: i64 = 5000; + try cache.put(.{ + .url = "https://example.com", + .content_type = "text/html", + .status = 200, + .stored_at = now, + .age_at_store = 0, + .cache_control = .{ .max_age = 1000 }, + .headers = &.{}, + .vary_headers = &.{}, + }, "hello world"); + + try cache.renew( + arena.allocator(), + .{ .url = "https://example.com", .timestamp = now + 500, .headers = &.{} }, + ); + + // Clock reset to now+500, so still fresh at now+1200 + const fresh = try cache.get( + arena.allocator(), + .{ + .url = "https://example.com", + .timestamp = now + 1200, + .request_headers = &.{}, + }, + ) orelse return error.CacheMiss; + try testing.expectEqual(false, fresh.expired); + + // Expires at now+500+1000 = now+1500 + const stale = try cache.get( + arena.allocator(), + .{ + .url = "https://example.com", + .timestamp = now + 1500, + .request_headers = &.{}, + }, + ) orelse return error.CacheMiss; + try testing.expectEqual(true, stale.expired); +} + +test "SqliteCache: renew preserves body" { + var cache = try setupCache(testing.allocator); + defer cache.deinit(); + + var arena = std.heap.ArenaAllocator.init(testing.allocator); + defer arena.deinit(); + + const now: u64 = @intCast(std.Io.Timestamp.now(testing.io, .boot).toSeconds()); + try cache.put(.{ + .url = "https://example.com", + .content_type = "text/html", + .status = 200, + .stored_at = now, + .age_at_store = 0, + .cache_control = .{ .max_age = 600 }, + .headers = &.{}, + .vary_headers = &.{}, + }, "original body"); + + try cache.renew( + arena.allocator(), + .{ .url = "https://example.com", .timestamp = now + 100, .headers = &.{} }, + ); + + const result = try cache.get( + arena.allocator(), + .{ + .url = "https://example.com", + .timestamp = now + 100, + .request_headers = &.{}, + }, + ) orelse return error.CacheMiss; + try testing.expectEqualStrings("original body", result.data.buffer); +} diff --git a/src/storage/Storage.zig b/src/storage/Storage.zig index 54cefdaf7..54ac21f3d 100644 --- a/src/storage/Storage.zig +++ b/src/storage/Storage.zig @@ -28,12 +28,10 @@ const Storage = @This(); pub const EngineType = enum { none, - sqlite, }; const Engine = union(EngineType) { none: Blackhole, - sqlite: Sqlite, }; engine: Engine, @@ -50,13 +48,9 @@ pub fn init(allocator: Allocator, config: *const Config) !Storage { }; } -fn initEngine(allocator: Allocator, engine_type: EngineType, config: *const Config) !Engine { +fn initEngine(_: Allocator, engine_type: EngineType, _: *const Config) !Engine { switch (engine_type) { .none => return .{ .none = Blackhole{} }, - .sqlite => { - const sqlite_path = config.storageSqlitePath(); - return .{ .sqlite = try Sqlite.init(allocator, sqlite_path) }; - }, } } diff --git a/src/storage/sqlite/Sqlite.zig b/src/storage/sqlite/Sqlite.zig index 6a8098fac..f06d5f63b 100644 --- a/src/storage/sqlite/Sqlite.zig +++ b/src/storage/sqlite/Sqlite.zig @@ -27,32 +27,60 @@ const Allocator = std.mem.Allocator; const Sqlite = @This(); -pool: Pool, +pub const Migration = union(enum) { + sql: [:0]const u8, + func: struct { + ctx: *anyopaque, + func: *const fn (conn: Conn, ctx: *anyopaque) anyerror!void, + }, +}; -pub fn init(allocator: Allocator, path_: ?[:0]const u8) !Sqlite { - const path = path_ orelse ":memory:"; - var pool = try Pool.init(allocator, path); - errdefer pool.deinit(allocator); +pub const Migrations = struct { + pub fn run(conn: Conn, migrations: []const Migration) !usize { + try conn.exec( + \\create table if not exists migrations ( + \\ id integer primary key, + \\ applied_at integer not null + \\) strict + , .{}); - { - // copy by value warning! The connection HAS to be returned to the - // pool in this scope. If we didn't have this scope, we'd assign the - // pool to the return value (copy A) and then release the original - const conn = try pool.acquire(); - defer pool.release(conn); + const current = (try conn.scalar( + i64, + "select max(id) from migrations", + .{}, + )) orelse 0; + const start: usize = @intCast(current); - const version = try @import("migrations.zig").run(conn); - log.info(.storage, "storage initialized", .{ .engine = "sqlite", .version = version, .path = path }); + if (start > migrations.len) { + log.err(.storage, "migrations removed", .{ + .applied = start, + .defined = migrations.len, + }); + return error.MigrationsRemoved; + } + + if (start == migrations.len) { + return start; + } + + try conn.begin(); + errdefer conn.rollback() catch {}; + + for (migrations[start..], start..) |migration, i| { + switch (migration) { + .sql => |sql| try conn.exec(sql, .{}), + .func => |f| try f.func(conn, f.ctx), + } + try conn.exec( + "insert into migrations (id, applied_at) values ($1, $2)", + .{ @as(i64, @intCast(i + 1)), std.Io.Timestamp.now(lp.io, .boot).toMilliseconds() }, + ); + } + + try conn.commit(); + return migrations.len; } - - return .{ - .pool = pool, - }; -} - -pub fn deinit(self: *Sqlite, allocator: Allocator) void { - self.pool.deinit(allocator); -} +}; pub const Conn = struct { conn: *c.sqlite3, @@ -138,6 +166,18 @@ pub const Conn = struct { return .{ .stmt = stmt.?, .conn = self.conn }; } + pub fn begin(self: Conn) !void { + try self.exec("begin", .{}); + } + + pub fn commit(self: Conn) !void { + try self.exec("commit", .{}); + } + + pub fn rollback(self: Conn) !void { + try self.exec("rollback", .{}); + } + pub fn busyTimeout(self: Conn, ms: c_int) !void { const rc = c.sqlite3_busy_timeout(self.conn, ms); if (rc != c.SQLITE_OK) { @@ -568,12 +608,42 @@ test "Sqlite: exec, row and scalar" { } } -test "Sqlite: Migration" { - var sqlite = try Sqlite.init(testing.allocator, ":memory:"); - defer sqlite.deinit(testing.allocator); +test "Sqlite: Migrations - basic" { + var conn = try Sqlite.Conn.open(":memory:"); + defer conn.close(); - const conn = try sqlite.pool.acquire(); - defer sqlite.pool.release(conn); + const migrations: []const Migration = &.{ + .{ .sql = "create table test (id integer primary key, name text)" }, + .{ .sql = "alter table test add column email text" }, + }; - try testing.expectEqual(1, (try conn.scalar(i64, "select max(id) from migrations", .{})).?); + const v1 = try Migrations.run(conn, migrations); + try testing.expectEqual(@as(usize, 2), v1); + + // idempotent - running again should return same version + const v2 = try Migrations.run(conn, migrations); + try testing.expectEqual(@as(usize, 2), v2); + + // verify migrations table has correct entries + try testing.expectEqual( + @as(i64, 2), + (try conn.scalar(i64, "select count(*) from migrations", .{})).?, + ); +} + +test "Sqlite: Migrations - removed migration" { + var conn = try Sqlite.Conn.open(":memory:"); + defer conn.close(); + + const m1: []const Migration = &.{ + .{ .sql = "create table test (id integer primary key, name text)" }, + .{ .sql = "alter table test add column email text" }, + }; + _ = try Migrations.run(conn, m1); + + // fewer migrations than were applied + const m2: []const Migration = &.{ + .{ .sql = "create table test (id integer primary key, name text)" }, + }; + try testing.expectError(error.MigrationsRemoved, Migrations.run(conn, m2)); } diff --git a/src/storage/sqlite/migrations.zig b/src/storage/sqlite/migrations.zig deleted file mode 100644 index e431c443f..000000000 --- a/src/storage/sqlite/migrations.zig +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright (C) 2023-2026 Lightpanda (Selecy SAS) -// -// Francis Bouvier -// Pierre Tachoire -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as -// published by the Free Software Foundation, either version 3 of the -// License, or (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -const lp = @import("lightpanda"); - -const Sqlite = @import("Sqlite.zig"); - -const log = lp.log; - -pub fn run(conn: Sqlite.Conn) !i64 { - const version = try getVersion(conn); - return version; -} - -fn getVersion(conn: Sqlite.Conn) !i64 { - const exists_sql = "select exists (select 1 from sqlite_schema where type='table' and name='migrations')"; - if (try conn.scalar(bool, exists_sql, .{}) orelse false) { - if (try conn.scalar(i64, "select max(id) from migrations", .{})) |version| { - return version; - } - - log.fatal(.storage, "corrupt database", .{ .engine = "sqlite", .note = "The sqlite database has an existing but empty `migrations` table" }); - return error.CorruptDatabase; - } - - // this pragma is one of the the few (if not only) one that's persisted, so - // we only have to do it the first time. - conn.exec("pragma journal_mode=wal", .{}) catch |err| { - log.fatal(.storage, "migrate", .{ - .err = err, - .step = "journal_mode", - .sqlite = conn.lastError(), - }); - return err; - }; - - const create_sql = - \\ create table migrations as - \\ select 1 as id, current_timestamp as created_at - ; - conn.exec(create_sql, .{}) catch |err| { - log.fatal(.storage, "migrate", .{ .err = err, .sqlite = conn.lastError(), .step = "create migrations" }); - return err; - }; - - return 1; -}