diff --git a/src/network/HttpClient.zig b/src/network/HttpClient.zig index 49b463743..a42283c79 100644 --- a/src/network/HttpClient.zig +++ b/src/network/HttpClient.zig @@ -944,13 +944,13 @@ fn cacheLookup(self: *Client, transfer: *Transfer) !bool { .revalidate => |cached| { log.debug(.cache, "revalidate cache entry", .{ .url = req.url, - .etag = cached.metadata.etag, - .last_modified = cached.metadata.last_modified, + .etag = cached.etag, + .last_modified = cached.last_modified, }); - if (cached.metadata.etag) |etag| { + if (cached.etag) |etag| { try req.headers.add(try std.fmt.allocPrintSentinel(arena, "If-None-Match: {s}", .{etag}, 0)); } - if (cached.metadata.last_modified) |lm| { + if (cached.last_modified) |lm| { try req.headers.add(try std.fmt.allocPrintSentinel(arena, "If-Modified-Since: {s}", .{lm}, 0)); } transfer._cache_intent = .{ .revalidate = cached }; @@ -1024,7 +1024,7 @@ fn cacheStore(self: *Client, transfer: *Transfer) void { const headers = transfer.res.headers; const vary = findHeader(headers, "vary"); - const maybe_cm = Cache.tryCache( + const maybe_req = Cache.tryCache( arena, std.Io.Clock.now(.real, lp.io).toSeconds(), transfer._cache_key, @@ -1041,7 +1041,7 @@ fn cacheStore(self: *Client, transfer: *Transfer) void { log.warn(.http, "cache eligibility", .{ .err = err }); return; }; - var metadata = maybe_cm orelse return; + var req = maybe_req orelse return; var vary_headers: std.ArrayList(http.Header) = .empty; if (vary) |vary_str| { @@ -1061,13 +1061,13 @@ fn cacheStore(self: *Client, transfer: *Transfer) void { } } - metadata.headers = headers; - metadata.vary_headers = vary_headers.items; + req.headers = headers; + req.vary_headers = vary_headers.items; if (comptime IS_DEBUG) { - log.debug(.browser, "http cache", .{ .key = transfer._cache_key, .metadata = metadata }); + log.debug(.browser, "http cache", .{ .key = transfer._cache_key, .put = req }); } - cache.put(metadata, transfer.res.buffer.items) catch |err| { + cache.put(req, transfer.res.buffer.items) catch |err| { log.warn(.http, "cache put failed", .{ .err = err }); }; } @@ -2366,8 +2366,8 @@ pub const Transfer = struct { .buffer => |b| b, }; - self.setResponseHead(cached.metadata.status, cached.metadata.content_type); - self.res.headers = cached.metadata.headers; + self.setResponseHead(cached.status, cached.content_type); + self.res.headers = cached.headers; self._from_cache = true; self._cdp_content_length = body.len; try self.bufferEvents(body); diff --git a/src/network/cache/Cache.zig b/src/network/cache/Cache.zig index 5f7472892..82b60f5a6 100644 --- a/src/network/cache/Cache.zig +++ b/src/network/cache/Cache.zig @@ -37,15 +37,15 @@ pub fn deinit(self: *Cache) void { }; } -pub fn get(self: *Cache, arena: std.mem.Allocator, req: CacheRequest) !CacheGetResult { +pub fn get(self: *Cache, arena: std.mem.Allocator, req: CacheGetRequest) !CacheGetResult { return switch (self.kind) { inline else => |*c| c.get(arena, req), }; } -pub fn put(self: *Cache, metadata: CachedMetadata, body: []const u8) !void { +pub fn put(self: *Cache, req: CachePutRequest, body: []const u8) !void { return switch (self.kind) { - inline else => |*c| c.put(metadata, body), + inline else => |*c| c.put(req, body), }; } @@ -133,7 +133,13 @@ pub const CacheControl = struct { } }; -pub const CachedMetadata = struct { +pub const CacheGetRequest = struct { + url: []const u8, + timestamp: i64, + request_headers: []const Http.Header, +}; + +pub const CachePutRequest = struct { url: [:0]const u8, content_type: []const u8, @@ -151,7 +157,7 @@ pub const CachedMetadata = struct { etag: ?[]const u8 = null, last_modified: ?[]const u8 = null, - pub fn format(self: CachedMetadata, writer: *std.Io.Writer) !void { + pub fn format(self: CachePutRequest, writer: *std.Io.Writer) !void { try writer.print("url={s} | status={d} | content_type={s} | max_age={d} | etag={s} | last-modified={s} | vary=[", .{ self.url, self.status, @@ -172,42 +178,6 @@ pub const CachedMetadata = struct { } try writer.print("]", .{}); } - - pub fn isStale(self: CachedMetadata, timestamp: i64) bool { - if (self.cache_control.must_revalidate) return true; - const age = (timestamp - self.stored_at) + @as(i64, @intCast(self.age_at_store)); - return age >= @as(i64, @intCast(self.cache_control.max_age)); - } - - pub fn hasValidators(self: CachedMetadata) bool { - return self.etag != null or self.last_modified != null; - } - - pub fn renew(self: *CachedMetadata, req: RenewResponse) void { - self.stored_at = req.timestamp; - self.age_at_store = 0; - - for (req.headers) |h| { - const name = h.name; - const value = h.value; - - if (std.ascii.eqlIgnoreCase("Age", name)) { - self.age_at_store = parseDeltaSeconds(value) orelse 0; - } else if (std.ascii.eqlIgnoreCase("Cache-Control", name)) { - self.cache_control = CacheControl.parse(value) orelse continue; - } else if (std.ascii.eqlIgnoreCase("ETag", name)) { - self.etag = value; - } else if (std.ascii.eqlIgnoreCase("Last-Modified", name)) { - self.last_modified = value; - } - } - } -}; - -pub const CacheRequest = struct { - url: []const u8, - timestamp: i64, - request_headers: []const Http.Header, }; pub const RenewResponse = struct { @@ -245,13 +215,20 @@ pub const CacheGetResult = union(enum) { }; pub const CachedResponse = struct { - metadata: CachedMetadata, + status: u16, + content_type: []const u8, + etag: ?[]const u8, + last_modified: ?[]const u8, + headers: []const Http.Header, data: CachedData, pub fn format(self: *const CachedResponse, writer: *std.Io.Writer) !void { - try writer.print("metadata=(", .{}); - try self.metadata.format(writer); - try writer.print("), data=", .{}); + try writer.print("status={d} | content_type={s} | etag={s} | last-modified={s} | ", .{ + self.status, + self.content_type, + self.etag orelse "null", + self.last_modified orelse "null", + }); try self.data.format(writer); } }; @@ -269,7 +246,7 @@ pub fn tryCache( last_modified: ?[]const u8, has_set_cookie: bool, has_authorization: bool, -) !?CachedMetadata { +) !?CachePutRequest { if (status != 200) { log.debug(.cache, "no store", .{ .url = url, .code = status, .reason = "status" }); return null; @@ -364,122 +341,6 @@ test "Cache: CacheControl.parse" { try testing.expectEqual(max_delta_seconds, CacheControl.parse("s-maxage=9999999999999999999").?.max_age); } -test "Cache: CachedMetadata.renew updates timestamp and age" { - var meta = CachedMetadata{ - .url = "https://example.com", - .content_type = "text/html", - .status = 200, - .stored_at = 1000, - .age_at_store = 50, - .cache_control = .{ .max_age = 600 }, - .headers = &.{}, - .vary_headers = &.{}, - }; - - meta.renew(.{ .url = "https://example.com", .timestamp = 2000, .headers = &.{} }); - - try testing.expectEqual(2000, meta.stored_at); - try testing.expectEqual(0, meta.age_at_store); -} - -test "Cache: CachedMetadata.renew updates age from Age header" { - var meta = CachedMetadata{ - .url = "https://example.com", - .content_type = "text/html", - .status = 200, - .stored_at = 1000, - .age_at_store = 0, - .cache_control = .{ .max_age = 600 }, - .headers = &.{}, - .vary_headers = &.{}, - }; - - meta.renew(.{ - .url = "https://example.com", - .timestamp = 2000, - .headers = &.{.{ .name = "Age", .value = "42" }}, - }); - - try testing.expectEqual(42, meta.age_at_store); - - meta.renew(.{ - .url = "https://example.com", - .timestamp = 2000, - .headers = &.{.{ .name = "Age", .value = "18446744073709551615" }}, - }); - - try testing.expectEqual(max_delta_seconds, meta.age_at_store); -} - -test "Cache: CachedMetadata.renew updates cache_control" { - var meta = CachedMetadata{ - .url = "https://example.com", - .content_type = "text/html", - .status = 200, - .stored_at = 1000, - .age_at_store = 0, - .cache_control = .{ .max_age = 600 }, - .headers = &.{}, - .vary_headers = &.{}, - }; - - meta.renew(.{ - .url = "https://example.com", - .timestamp = 2000, - .headers = &.{.{ .name = "Cache-Control", .value = "max-age=1200" }}, - }); - - try testing.expectEqual(1200, meta.cache_control.max_age); -} - -test "Cache: CachedMetadata.renew preserves cache_control on invalid header" { - var meta = CachedMetadata{ - .url = "https://example.com", - .content_type = "text/html", - .status = 200, - .stored_at = 1000, - .age_at_store = 0, - .cache_control = .{ .max_age = 600 }, - .headers = &.{}, - .vary_headers = &.{}, - }; - - meta.renew(.{ - .url = "https://example.com", - .timestamp = 2000, - .headers = &.{.{ .name = "Cache-Control", .value = "no-store" }}, - }); - - try testing.expectEqual(600, meta.cache_control.max_age); -} - -test "Cache: CachedMetadata.renew updates etag and last_modified" { - var meta = CachedMetadata{ - .url = "https://example.com", - .content_type = "text/html", - .status = 200, - .stored_at = 1000, - .age_at_store = 0, - .cache_control = .{ .max_age = 600 }, - .headers = &.{}, - .vary_headers = &.{}, - .etag = "\"old-etag\"", - .last_modified = "Mon, 01 Jan 2024 00:00:00 GMT", - }; - - meta.renew(.{ - .url = "https://example.com", - .timestamp = 2000, - .headers = &.{ - .{ .name = "ETag", .value = "\"new-etag\"" }, - .{ .name = "Last-Modified", .value = "Tue, 02 Jan 2024 00:00:00 GMT" }, - }, - }); - - try testing.expectEqualSlices(u8, "\"new-etag\"", meta.etag.?); - try testing.expectEqualSlices(u8, "Tue, 02 Jan 2024 00:00:00 GMT", meta.last_modified.?); -} - test "Cache: tryCache heuristic when no cache-control" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); diff --git a/src/network/cache/SqliteCache.zig b/src/network/cache/SqliteCache.zig index cf25bb0c7..43b8d2d71 100644 --- a/src/network/cache/SqliteCache.zig +++ b/src/network/cache/SqliteCache.zig @@ -22,9 +22,9 @@ const lp = @import("lightpanda"); const Cache = @import("Cache.zig"); const log = lp.log; -const CacheRequest = Cache.CacheRequest; +const CacheGetRequest = Cache.CacheGetRequest; const RenewResponse = Cache.RenewResponse; -const CachedMetadata = Cache.CachedMetadata; +const CachePutRequest = Cache.CachePutRequest; const CacheGetResult = Cache.CacheGetResult; const CachedResponse = Cache.CachedResponse; const CacheControl = Cache.CacheControl; @@ -44,7 +44,7 @@ pool: Pool, const cache_migrations: []const Migration = &.{ .{ .sql = - \\ create table metadata ( + \\ create table cache ( \\ url text not null primary key, \\ status integer not null, \\ stored_at integer not null, @@ -52,14 +52,8 @@ const cache_migrations: []const Migration = &.{ \\ max_age integer not null, \\ must_revalidate integer not null, \\ etag text, - \\ last_modified text - \\ ) strict - }, - .{ .sql = - \\ create table body ( - \\ url text not null primary key, - \\ data blob not null, - \\ foreign key (url) references metadata(url) on delete cascade + \\ last_modified text, + \\ body blob not null \\ ) strict }, .{ .sql = @@ -68,7 +62,7 @@ const cache_migrations: []const Migration = &.{ \\ name text not null, \\ value blob not null, \\ vary integer not null, - \\ foreign key (url) references metadata(url) on delete cascade + \\ foreign key (url) references cache(url) on delete cascade \\ ) strict }, .{ .sql = "create index header_url on header(url)" }, @@ -123,7 +117,7 @@ pub fn deinit(self: *SqliteCache) void { self.pool.deinit(self.allocator); } -pub fn get(self: *SqliteCache, arena: std.mem.Allocator, req: CacheRequest) !CacheGetResult { +pub fn get(self: *SqliteCache, arena: std.mem.Allocator, req: CacheGetRequest) !CacheGetResult { const conn = try self.pool.acquire(); defer self.pool.release(conn); @@ -132,8 +126,8 @@ pub fn get(self: *SqliteCache, arena: std.mem.Allocator, req: CacheRequest) !Cac var entry = try conn.row( \\ select status, stored_at, age_at_store, - \\ max_age, must_revalidate, etag, last_modified - \\ from metadata where url = $1 + \\ max_age, must_revalidate, etag, last_modified, body + \\ from cache where url = $1 , .{req.url}) orelse { log.debug(.cache, "miss", .{ .url = req.url, .reason = "missing" }); return .miss; @@ -147,6 +141,7 @@ pub fn get(self: *SqliteCache, arena: std.mem.Allocator, req: CacheRequest) !Cac const must_revalidate = entry.get(bool, 4); const raw_etag = entry.get(?[]const u8, 5); const raw_last_modified = entry.get(?[]const u8, 6); + const raw_body = entry.get(Blob, 7); const expired = must_revalidate or blk: { const age = (req.timestamp - stored_at) + age_at_store; @@ -161,37 +156,29 @@ pub fn get(self: *SqliteCache, arena: std.mem.Allocator, req: CacheRequest) !Cac return .stale; } - var vary_headers: std.ArrayList(Http.Header) = .empty; - { - var vary_rows = try conn.rows( - "select name, value from header where url = $1 and vary = true", - .{req.url}, - ); - defer vary_rows.deinit(); + var vary_rows = try conn.rows( + "select name, value from header where url = $1 and vary = true", + .{req.url}, + ); + defer vary_rows.deinit(); - while (try vary_rows.next()) |row| { - const name = row.get([]const u8, 0); - const value = row.get(Blob, 1).data; + while (try vary_rows.next()) |row| { + const name = row.get([]const u8, 0); + const value = row.get(Blob, 1).data; - const incoming = for (req.request_headers) |rh| { - if (std.ascii.eqlIgnoreCase(rh.name, name)) break rh.value; - } else ""; + const incoming = for (req.request_headers) |rh| { + if (std.ascii.eqlIgnoreCase(rh.name, name)) break rh.value; + } else ""; - if (!std.ascii.eqlIgnoreCase(value, incoming)) { - log.debug(.cache, "miss", .{ - .url = req.url, - .reason = "vary mismatch", - .header = name, - .expected = value, - .got = incoming, - }); - return .miss; - } - - try vary_headers.append(arena, .{ - .name = try arena.dupe(u8, name), - .value = try arena.dupe(u8, value), + if (!std.ascii.eqlIgnoreCase(value, incoming)) { + log.debug(.cache, "miss", .{ + .url = req.url, + .reason = "vary mismatch", + .header = name, + .expected = value, + .got = incoming, }); + return .miss; } } @@ -211,94 +198,72 @@ pub fn get(self: *SqliteCache, arena: std.mem.Allocator, req: CacheRequest) !Cac try headers.append(arena, .{ .name = name, .value = value }); } - const metadata = CachedMetadata{ - .url = try arena.dupeZ(u8, req.url), - .content_type = content_type, + log.debug(.cache, "hit", .{ .url = req.url, .expired = expired }); + + const resp: CachedResponse = .{ .status = status, - .stored_at = stored_at, - .age_at_store = @intCast(age_at_store), - .cache_control = .{ - .max_age = max_age, - .must_revalidate = must_revalidate, - }, + .content_type = content_type, .headers = headers.items, - .vary_headers = vary_headers.items, .etag = if (raw_etag) |v| try arena.dupe(u8, v) else null, .last_modified = if (raw_last_modified) |v| try arena.dupe(u8, v) else null, + .data = .{ .buffer = try arena.dupe(u8, raw_body.data) }, }; - - var body_entry = try conn.row( - "select data from body where url = $1", - .{req.url}, - ) orelse { - log.debug(.cache, "miss", .{ .url = req.url, .reason = "missing body" }); - return .stale; - }; - defer body_entry.deinit(); - const body = try arena.dupe(u8, body_entry.get(Blob, 0).data); - - log.debug(.cache, "hit", .{ .url = req.url, .expired = expired }); - const resp: CachedResponse = .{ .metadata = metadata, .data = .{ .buffer = body } }; return if (expired) .{ .revalidate = resp } else .{ .hit = resp }; } -pub fn put(self: *SqliteCache, meta: CachedMetadata, body: []const u8) !void { +pub fn put(self: *SqliteCache, req: CachePutRequest, body: []const u8) !void { const conn = try self.pool.acquire(); defer self.pool.release(conn); try conn.begin(.immediate); defer conn.rollback() catch {}; - try conn.exec("delete from metadata where url = $1", .{meta.url}); + try conn.exec("delete from cache where url = $1", .{req.url}); try conn.exec( - \\ insert into metadata - \\ (url, status, stored_at, age_at_store, max_age, must_revalidate, etag, last_modified) - \\ values ($1, $2, $3, $4, $5, $6, $7, $8) + \\ insert into cache + \\ (url, status, stored_at, age_at_store, max_age, must_revalidate, etag, last_modified, body) + \\ values ($1, $2, $3, $4, $5, $6, $7, $8, $9) , .{ - meta.url, - @as(i64, @intCast(meta.status)), - meta.stored_at, - @as(i64, @intCast(meta.age_at_store)), - @as(i64, @intCast(meta.cache_control.max_age)), - meta.cache_control.must_revalidate, - meta.etag, - meta.last_modified, + req.url, + @as(i64, @intCast(req.status)), + req.stored_at, + @as(i64, @intCast(req.age_at_store)), + @as(i64, @intCast(req.cache_control.max_age)), + req.cache_control.must_revalidate, + req.etag, + req.last_modified, + Blob{ .data = body }, }); - try conn.exec( - "insert into body (url, data) values ($1, $2)", - .{ meta.url, Blob{ .data = body } }, - ); - var lower_name: [256]u8 = undefined; - for (meta.headers) |h| { + for (req.headers) |h| { if (h.name.len > lower_name.len) return error.HeaderNameTooLong; const name = std.ascii.lowerString(lower_name[0..h.name.len], h.name); try conn.exec( "insert into header (url, name, value, vary) values ($1, $2, $3, false)", - .{ meta.url, name, Blob{ .data = h.value } }, + .{ req.url, name, Blob{ .data = h.value } }, ); } - for (meta.vary_headers) |h| { + for (req.vary_headers) |h| { if (h.name.len > lower_name.len) return error.HeaderNameTooLong; const name = std.ascii.lowerString(lower_name[0..h.name.len], h.name); try conn.exec( "insert into header (url, name, value, vary) values ($1, $2, $3, true)", - .{ meta.url, name, Blob{ .data = h.value } }, + .{ req.url, name, Blob{ .data = h.value } }, ); } try conn.commit(); - log.debug(.cache, "put", .{ .url = meta.url, .body_len = body.len }); + log.debug(.cache, "put", .{ .url = req.url, .body_len = body.len }); } pub fn clear(self: *SqliteCache) !void { const conn = try self.pool.acquire(); defer self.pool.release(conn); - try conn.exec("delete from metadata", .{}); + try conn.exec("delete from cache", .{}); log.debug(.cache, "clear", .{}); } @@ -309,7 +274,7 @@ pub fn evict(self: *SqliteCache, url: []const u8) void { }; defer self.pool.release(conn); - conn.exec("delete from metadata where url = $1", .{url}) catch |err| { + conn.exec("delete from cache where url = $1", .{url}) catch |err| { log.err(.cache, "delete from cache", .{ .url = url, .err = err }); return; }; @@ -342,7 +307,7 @@ pub fn renew(self: *SqliteCache, _: std.mem.Allocator, req: RenewResponse) !void } try conn.exec( - \\ update metadata + \\ update cache \\ set stored_at = $1, \\ age_at_store = $2, \\ max_age = coalesce($3, max_age), @@ -412,7 +377,7 @@ test "SqliteCache: basic put and get" { defer arena.deinit(); const now = std.Io.Clock.now(.boot, lp.io).toSeconds(); - const meta = CachedMetadata{ + const meta = CachePutRequest{ .url = "https://example.com", .content_type = "text/html", .status = 200, @@ -436,8 +401,8 @@ test "SqliteCache: basic put and get" { try testing.expect(result == .hit); try testing.expectEqualStrings("hello world", result.hit.data.buffer); - try testing.expectEqual(@as(u16, 200), result.hit.metadata.status); - try testing.expectEqualStrings("text/html", result.hit.metadata.content_type); + try testing.expectEqual(@as(u16, 200), result.hit.status); + try testing.expectEqualStrings("text/html", result.hit.content_type); } test "SqliteCache: get expiration" { @@ -450,7 +415,7 @@ test "SqliteCache: get expiration" { const now = 5000; const max_age = 1000; - const meta = CachedMetadata{ + const meta = CachePutRequest{ .url = "https://example.com", .content_type = "text/html", .status = 200, @@ -497,7 +462,7 @@ test "SqliteCache: get expiration (without validators)" { const now = 5000; const max_age = 1000; - const meta = CachedMetadata{ + const meta = CachePutRequest{ .url = "https://example.com", .content_type = "text/html", .status = 200, @@ -540,7 +505,7 @@ test "SqliteCache: put override" { defer arena.deinit(); { - const meta = CachedMetadata{ + const meta = CachePutRequest{ .url = "https://example.com", .content_type = "text/html", .status = 200, @@ -565,7 +530,7 @@ test "SqliteCache: put override" { } { - const meta = CachedMetadata{ + const meta = CachePutRequest{ .url = "https://example.com", .content_type = "text/html", .status = 200, @@ -598,7 +563,7 @@ test "SqliteCache: vary hit and miss" { defer arena.deinit(); const now = std.Io.Clock.now(.boot, testing.io).toSeconds(); - const meta = CachedMetadata{ + const meta = CachePutRequest{ .url = "https://example.com", .content_type = "text/html", .status = 200, @@ -644,7 +609,7 @@ test "SqliteCache: vary multiple headers" { defer arena.deinit(); const now = std.Io.Clock.now(.boot, testing.io).toSeconds(); - const meta = CachedMetadata{ + const meta = CachePutRequest{ .url = "https://example.com", .content_type = "text/html", .status = 200, @@ -757,7 +722,7 @@ test "SqliteCache: put after clear works" { defer arena.deinit(); const now = std.Io.Clock.now(.boot, testing.io).toSeconds(); - const meta = CachedMetadata{ + const meta = CachePutRequest{ .url = "https://example.com", .content_type = "text/html", .status = 200, @@ -801,7 +766,7 @@ test "SqliteCache: evict removes entry" { defer arena.deinit(); const now = std.Io.Clock.now(.boot, testing.io).toSeconds(); - const meta = CachedMetadata{ + const meta = CachePutRequest{ .url = "https://example.com", .content_type = "text/html", .status = 200,