From decedca6e7bfb1cfdf944f8a610a8470c1a7b268 Mon Sep 17 00:00:00 2001 From: Karl Seguin Date: Fri, 21 Aug 2026 17:42:58 +0800 Subject: [PATCH] chore: Network cleanup Network has accumulated a bit of bagged. It knows a lot about certificates, it knows a lot of the cache. I have plans to expand Network, and wanted to clean it up. 1 - certificate logic moved to dedicated Certificates.zig 2 - Removed CurlDebugAllocator - this makes setup easier, to Updater can just init libcurl directly 3 - Change Updater to be a single function 4 - Cache initialization is don in the Cache 5 - ?Cache => Cache{.kind = .noop} --- src/Updater.zig | 82 +++++------- src/browser/Browser.zig | 3 - src/cdp/domains/network.zig | 8 +- src/cdp/testing.zig | 2 +- src/lightpanda.zig | 8 +- src/network/Certificates.zig | 139 ++++++++++++++++++++ src/network/CurlDebugAllocator.zig | 134 -------------------- src/network/HttpClient.zig | 22 ++-- src/network/Network.zig | 197 +++-------------------------- src/network/cache/Cache.zig | 42 ++++++ src/network/http.zig | 14 +- src/telemetry/lightpanda.zig | 2 +- 12 files changed, 257 insertions(+), 396 deletions(-) create mode 100644 src/network/Certificates.zig delete mode 100644 src/network/CurlDebugAllocator.zig diff --git a/src/Updater.zig b/src/Updater.zig index 837c8e42d..2c753a6a0 100644 --- a/src/Updater.zig +++ b/src/Updater.zig @@ -19,70 +19,31 @@ const std = @import("std"); const lp = @import("lightpanda"); -const Network = @import("network/Network.zig"); const http = @import("network/http.zig"); +const Certificates = @import("network/Certificates.zig"); const libcurl = @import("sys/libcurl.zig"); -const crypto = @import("sys/libcrypto.zig"); -const Config = @import("Config.zig"); const Allocator = std.mem.Allocator; -/// Sole purpose of this client is to do updates; hence, its very minimal. -const Updater = @This(); -x509_store: *crypto.X509_STORE, -config: *const Config, - -/// Initializes the update client; meant to be used as singleton. -pub fn init(allocator: Allocator, config: *const Config) !Updater { - Network.globalInit(allocator); - errdefer Network.globalDeinit(); - const x509_store = try Network.prepareX509Store(allocator, config); - - return .{ - .x509_store = x509_store, - .config = config, - }; -} - -pub fn deinit(self: *Updater) void { - Network.globalDeinit(); - crypto.X509_STORE_free(self.x509_store); -} - /// Sends running Lightpanda version to remote to get update information. /// Outputs directly to given `Writer`. -pub fn inform(self: *Updater, writer: *std.Io.Writer) !void { - var conn = try http.Connection.init(self.x509_store, self.config, null); +pub fn inform(allocator: Allocator, config: *const lp.Config, writer: *std.Io.Writer) !void { + const certificates = try Certificates.init(allocator, config); + defer certificates.deinit(); + + libcurl.curl_global_init(.{ .ssl = true }, null) catch |err| { + lp.assert(false, "curl global init", .{ .err = err }); + }; + defer libcurl.curl_global_cleanup(); + + var conn = try http.Connection.init(certificates, config, null); defer conn.deinit(); - const url = std.fmt.comptimePrint("https://telemetry.lightpanda.io/v/{s}", .{lp.build_config.version}); - // Prepare the request. - try conn.setURL(url); + try conn.setURL("https://telemetry.lightpanda.io/v/" ++ lp.build_config.version); try conn.setGetMode(); try conn.setFollowLocation(true); // Wraps everything needed to receive bytes. - const ReceiverContext = struct { - writer: *std.Io.Writer, - err: std.Io.Writer.Error!void = {}, - - /// curl -> writer. - fn drain( - buffer: [*]const u8, - buf_count: usize, - buf_len: usize, - raw_ctx: *anyopaque, - ) callconv(.c) usize { - const ctx: *@This() = @ptrCast(@alignCast(raw_ctx)); - const chunk = buffer[0 .. buf_count * buf_len]; - ctx.writer.writeAll(chunk) catch |err| { - ctx.err = err; - return 0; - }; - - return chunk.len; - } - }; // Set receiver context. var ctx = ReceiverContext{ .writer = writer }; @@ -97,3 +58,22 @@ pub fn inform(self: *Updater, writer: *std.Io.Writer) !void { _ = status_int; return writer.flush(); } + +const ReceiverContext = struct { + writer: *std.Io.Writer, + err: std.Io.Writer.Error!void = {}, + + fn drain(buffer: [*]const u8, buf_count: usize, buf_len: usize, ctx: *anyopaque) callconv(.c) usize { + // libcurl only ever sends 1 buffer + std.debug.assert(buf_count == 1); + + const self: *ReceiverContext = @ptrCast(@alignCast(ctx)); + const chunk = buffer[0..buf_len]; + self.writer.writeAll(chunk) catch |err| { + self.err = err; + return 0; + }; + + return buf_len; + } +}; diff --git a/src/browser/Browser.zig b/src/browser/Browser.zig index f4cab1c65..3302a48bd 100644 --- a/src/browser/Browser.zig +++ b/src/browser/Browser.zig @@ -153,9 +153,6 @@ pub fn deinit(self: *Browser) void { // fire — only now is it safe to free the pool backing their parameters. self.fc_identity_pool.deinit(allocator); self.page_pool.deinit(allocator); - if (self.http_client.cache) |cache| { - cache.maintenance(lp.datetime.timestamp(.real)); - } self.http_client.deinit(); self.clearPermissions(); self.permissions.deinit(allocator); diff --git a/src/cdp/domains/network.zig b/src/cdp/domains/network.zig index 99bf6cb37..c5fd99de7 100644 --- a/src/cdp/domains/network.zig +++ b/src/cdp/domains/network.zig @@ -216,14 +216,14 @@ fn clearBrowserCache(cmd: *CDP.Command) !void { // Chrome accepts that and clears the jar; reject only on truly malformed JSON. const bc = cmd.browser_context orelse return error.BrowserContextNotLoaded; const network = bc.cdp.browser.http_client.network; - if (network.cache) |*c| try c.clear(); + try network.cache.clear(); return cmd.sendResult(null, .{}); } fn canClearBrowserCache(cmd: *CDP.Command) !void { const bc = cmd.browser_context orelse return error.BrowserContextNotLoaded; const network = bc.cdp.browser.http_client.network; - return cmd.sendResult(.{ .result = network.cache != null }, .{}); + return cmd.sendResult(.{ .result = network.cache.active() != null }, .{}); } fn clearBrowserCookies(cmd: *CDP.Command) !void { @@ -1030,7 +1030,7 @@ test "cdp.Network: setCacheDisabled" { .params = .{ .cacheDisabled = true }, }); try ctx.expectSentResult(null, .{ .id = 1 }); - try testing.expect(client.cache == null); + try testing.expectEqual(null, client.cache.active()); } test "cdp.Network: configured CDP ignores setCacheDisabled" { @@ -1041,7 +1041,7 @@ test "cdp.Network: configured CDP ignores setCacheDisabled" { var cache: Cache = undefined; const client = &ctx.cdp().browser.http_client; client.cache = &cache; - defer client.cache = null; + defer client.cache = &testing.base.test_app.network.cache; try ctx.processMessage(.{ .id = 1, diff --git a/src/cdp/testing.zig b/src/cdp/testing.zig index 8133f32bb..fd9edb2fe 100644 --- a/src/cdp/testing.zig +++ b/src/cdp/testing.zig @@ -20,7 +20,7 @@ const std = @import("std"); const CDP = @import("CDP.zig"); -const base = @import("../testing.zig"); +pub const base = @import("../testing.zig"); const json = std.json; const posix = std.posix; diff --git a/src/lightpanda.zig b/src/lightpanda.zig index f6fcd03c8..a8337ab2a 100644 --- a/src/lightpanda.zig +++ b/src/lightpanda.zig @@ -60,8 +60,6 @@ pub const build_config = @import("build_config"); pub const crash_handler = @import("crash_handler.zig"); pub const core_dump = @import("core_dump.zig"); -pub const Updater = @import("Updater.zig"); - pub var metrics = @import("Metrics.zig"){}; pub const IS_TEST = @import("builtin").is_test; @@ -395,14 +393,10 @@ fn dumpContent(app: *App, mode: Config.DumpFormat, dump_opts: dump.Opts, frame: } pub fn checkVersion(allocator: std.mem.Allocator, config: *const Config) !void { - var client = try Updater.init(allocator, config); - defer client.deinit(); - const stdout = std.Io.File.stdout(); var buf: [4096]u8 = undefined; var writer = stdout.writer(io, &buf); - const w = &writer.interface; - try client.inform(w); + try @import("Updater.zig").inform(allocator, config, &writer.interface); } // Writes a single page's result object. Framing (the enclosing array and any diff --git a/src/network/Certificates.zig b/src/network/Certificates.zig new file mode 100644 index 000000000..b85658cc2 --- /dev/null +++ b/src/network/Certificates.zig @@ -0,0 +1,139 @@ +// Copyright (C) 2023-2026 Lightpanda (Selecy SAS) +// +// Francis Bouvier +// Pierre Tachoire +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +const std = @import("std"); +const lp = @import("lightpanda"); + +const crypto = @import("../sys/libcrypto.zig"); + +const log = lp.log; +const Config = lp.Config; +const Allocator = std.mem.Allocator; + +const Certificates = @This(); + +store: *crypto.X509_STORE, + +pub fn init(allocator: Allocator, config: *const Config) !Certificates { + const store = blk: { + const custom_store = config.customCertStore(); + if (config.tlsVerifyHost() == false) { + if (custom_store != null) { + log.warn(.app, "custom CA ignored", .{ + .arg = "--ca-cert, --ca-path", + .reason = "TLS verification disabled", + }); + } + break :blk crypto.X509_STORE_new() orelse return error.FailedToCreateX509Store; + } + break :blk custom_store orelse try storeFromSystemCA(allocator); + }; + + return .{ .store = store }; +} + +pub fn deinit(self: Certificates) void { + crypto.X509_STORE_free(self.store); +} + +fn storeFromSystemCA(allocator: Allocator) !*crypto.X509_STORE { + const store = crypto.X509_STORE_new() orelse return error.FailedToCreateX509Store; + errdefer crypto.X509_STORE_free(store); + + var count: usize = 0; + defer { + if (count == 0) { + log.warn(.app, "No certificates loaded", .{}); + } + } + + switch (comptime @import("builtin").os.tag) { + .linux, .openbsd, .netbsd, .freebsd => blk: { + // Iterate over known directories; this may or may not succeed. + const cwd = std.Io.Dir.cwd(); + inline for ([_][]const u8{ + "/etc/ssl/certs", // Debian/Ubuntu/Gentoo/Alpine, SUSE + "/etc/pki/tls/certs", // Fedora/RHEL + }) |dir_path| { + count += try loadFromDirectory(allocator, store, cwd, dir_path); + if (count > 0) break :blk; + } + + // Iterate over known files. + inline for ([_][*:0]const u8{ + "/etc/ssl/certs/ca-certificates.crt", // Debian/Ubuntu/Gentoo + "/etc/pki/tls/certs/ca-bundle.crt", // Fedora/RHEL 6 + "/etc/ssl/ca-bundle.pem", // OpenSUSE + "/etc/pki/tls/cacert.pem", // OpenELEC + "/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem", // CentOS/RHEL 7 + "/etc/ssl/cert.pem", // Alpine, *BSD + }) |file| { + if (crypto.X509_STORE_load_locations(store, file, null) == 1) { + count += 1; + break :blk; + } + } + }, + else => { + // Prefer stdlib's cert scanner. + var bundle: std.crypto.Certificate.Bundle = .empty; + try bundle.rescan(allocator, lp.io, std.Io.Clock.now(.real, lp.io)); + defer bundle.deinit(allocator); + + const bytes = bundle.bytes.items; + var it = bundle.map.valueIterator(); + while (it.next()) |index| { + // d2i_X509 reads the cert's own DER length header to find its end and + // advances `ptr` past it, so we just hand it the rest of the buffer. + var ptr: [*]const u8 = bytes.ptr + index.*; + const x509 = crypto.d2i_X509(null, &ptr, @intCast(bytes.len - index.*)) orelse { + log.warn(.app, "Skipping unparseable system cert", .{}); + continue; + }; + defer crypto.X509_free(x509); // add_cert takes its own ref; drop ours. + + const result = crypto.X509_STORE_add_cert(store, x509); + if (result != 1) { + log.warn(.app, "Failed to add X509 cert to store", .{}); + } + count += 1; + } + }, + } + + return store; +} + +fn loadFromDirectory(allocator: Allocator, store: *crypto.X509_STORE, cwd: std.Io.Dir, dir_path: []const u8) !usize { + var count: usize = 0; + var dir = cwd.openDir(lp.io, dir_path, .{ .iterate = true }) catch return count; + defer dir.close(lp.io); + + var it = dir.iterate(); + while (it.next(lp.io) catch return count) |entry| { + if (entry.kind != .file and entry.kind != .sym_link) continue; + + const path = try std.fs.path.joinZ(allocator, &.{ dir_path, entry.name }); + defer allocator.free(path); + + if (crypto.X509_STORE_load_locations(store, path, null) == 1) { + count += 1; + } + } + return count; +} diff --git a/src/network/CurlDebugAllocator.zig b/src/network/CurlDebugAllocator.zig deleted file mode 100644 index 7fc700e54..000000000 --- a/src/network/CurlDebugAllocator.zig +++ /dev/null @@ -1,134 +0,0 @@ -// Copyright (C) 2023-2026 Lightpanda (Selecy SAS) -// -// Francis Bouvier -// Pierre Tachoire -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as -// published by the Free Software Foundation, either version 3 of the -// License, or (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -const std = @import("std"); -const lp = @import("lightpanda"); - -const libcurl = @import("../sys/libcurl.zig"); - -const Allocator = std.mem.Allocator; - -const CurlDebugAllocator = @This(); - -// C11 requires malloc to return memory aligned to max_align_t (16 bytes on x86_64). -// We match this guarantee since libcurl expects malloc-compatible alignment. -const alignment = 16; - -var instance: ?CurlDebugAllocator = null; - -allocator: Allocator, - -pub fn init(allocator: Allocator) void { - lp.assert(instance == null, "Initialization of curl must happen only once", .{}); - instance = .{ .allocator = allocator }; -} - -pub fn interface() libcurl.CurlAllocator { - return .{ - .free = free, - .strdup = strdup, - .malloc = malloc, - .calloc = calloc, - .realloc = realloc, - }; -} - -fn _allocBlock(size: usize) ?*Block { - const slice = instance.?.allocator.alignedAlloc(u8, .fromByteUnits(alignment), Block.fullsize(size)) catch return null; - const block: *Block = @ptrCast(@alignCast(slice.ptr)); - block.size = size; - return block; -} - -fn _freeBlock(header: *Block) void { - instance.?.allocator.free(header.slice()); -} - -fn malloc(size: usize) ?*anyopaque { - const block = _allocBlock(size) orelse return null; - return @ptrCast(block.data()); -} - -fn calloc(nmemb: usize, size: usize) ?*anyopaque { - const total = nmemb * size; - const block = _allocBlock(total) orelse return null; - const ptr = block.data(); - @memset(ptr[0..total], 0); // for historical reasons, calloc zeroes memory, but malloc does not. - return @ptrCast(ptr); -} - -fn realloc(ptr: ?*anyopaque, size: usize) ?*anyopaque { - const p = ptr orelse return malloc(size); - const block = Block.fromPtr(p); - - const old_size = block.size; - if (size == old_size) return ptr; - - if (instance.?.allocator.resize(block.slice(), alignment + size)) { - block.size = size; - return ptr; - } - - const copy_size = @min(old_size, size); - const new_block = _allocBlock(size) orelse return null; - @memcpy(new_block.data()[0..copy_size], block.data()[0..copy_size]); - _freeBlock(block); - return @ptrCast(new_block.data()); -} - -fn free(ptr: ?*anyopaque) void { - const p = ptr orelse return; - _freeBlock(Block.fromPtr(p)); -} - -fn strdup(str: [*:0]const u8) ?[*:0]u8 { - const len = std.mem.len(str); - const header = _allocBlock(len + 1) orelse return null; - const ptr = header.data(); - @memcpy(ptr[0..len], str[0..len]); - ptr[len] = 0; - return ptr[0..len :0]; -} - -const Block = extern struct { - size: usize = 0, - _padding: [alignment - @sizeOf(usize)]u8 = .{0} ** (alignment - @sizeOf(usize)), - - inline fn fullsize(bytes: usize) usize { - return alignment + bytes; - } - - inline fn fromPtr(ptr: *anyopaque) *Block { - const raw: [*]u8 = @ptrCast(ptr); - return @ptrCast(@alignCast(raw - @sizeOf(Block))); - } - - inline fn data(self: *Block) [*]u8 { - const ptr: [*]u8 = @ptrCast(self); - return ptr + @sizeOf(Block); - } - - inline fn slice(self: *Block) []align(alignment) u8 { - const base: [*]align(alignment) u8 = @ptrCast(@alignCast(self)); - return base[0 .. alignment + self.size]; - } -}; - -comptime { - std.debug.assert(@sizeOf(Block) == alignment); -} diff --git a/src/network/HttpClient.zig b/src/network/HttpClient.zig index 041b00f64..06e437184 100644 --- a/src/network/HttpClient.zig +++ b/src/network/HttpClient.zig @@ -181,8 +181,7 @@ blocking_requests: std.AutoHashMapUnmanaged(u32, u32) = .empty, // heuristics add this in. intercepted: usize = 0, -// null or referencing network.cache -cache: ?*Cache, +cache: *Cache, // Cached config decisions, resolved once at init. serve_mode: bool, @@ -218,7 +217,7 @@ pub fn init(self: *Client, allocator: Allocator, network: *Network, cdp: ?*CDP) .allocator = allocator, .cdp = cdp, .inbox = .{}, - .cache = if (network.cache) |*c| c else null, + .cache = &network.cache, .use_proxy = http_proxy != null, .http_proxy = http_proxy, @@ -265,6 +264,7 @@ pub fn deinit(self: *Client) void { self.blocking_requests.deinit(self.allocator); self.transfers.deinit(self.allocator); self.inbox.deinit(); + self.cache.maintenance(lp.datetime.timestamp(.real)); } // Look up a live transfer by its id. Returns null if the transfer has been @@ -317,9 +317,9 @@ pub fn obeyRobots(self: *Client, enable: bool) !void { pub fn disableCache(self: *Client, disable: bool) void { if (disable) { - self.cache = null; + self.cache = &Cache.noop; } else { - self.cache = if (self.network.cache) |*c| c else null; + self.cache = &self.network.cache; } } @@ -984,7 +984,7 @@ fn findHeader(headers: []const http.Header, name: []const u8) ?[]const u8 { // the response; on an expired-with-validators entry the request becomes a // conditional revalidation. fn cacheLookup(self: *Client, transfer: *Transfer) !bool { - const cache = self.cache orelse return false; + const cache = self.cache.active() orelse return false; const req = &transfer.req; if (req.method != .GET or req.streaming or req.skip_cache) { @@ -1067,7 +1067,7 @@ fn cacheRevalidated(self: *Client, transfer: *Transfer) !bool { return false; } // could have been disabled in-between - const cache = self.cache orelse return false; + const cache = self.cache.active() orelse return false; const stale = transfer._cache_intent.revalidate; transfer._cache_intent = .none; @@ -1103,7 +1103,7 @@ fn cacheStore(self: *Client, transfer: *Transfer) void { transfer._cache_intent = .none; // could have been disabled while waiting of the response - const cache = self.cache orelse return; + const cache = self.cache.active() orelse return; const arena = transfer.arena; const rh = &(transfer.res.header orelse return); @@ -3582,7 +3582,7 @@ fn initTestClient(client: *Client, pool: *ArenaPool) void { client.intercepted = 0; client.http_active = 0; client.ws_active = 0; - client.cache = null; + client.cache = &Cache.noop; client.serve_mode = false; client.obey_robots = false; client.robots = .{ @@ -3951,7 +3951,7 @@ test "HttpClient: fulfillIntercepted follows a 3xx redirect" { // Only network.config (httpMaxRedirects, which ignores its config), // network.cache and the (empty) connection pool are read on this path. var net: Network = undefined; - net.cache = null; + net.cache = Cache.noop; net.adblocker = null; // An empty pool makes processTransfer queue the re-issued request // instead of putting it on the wire — the queue IS the capture. @@ -4309,7 +4309,7 @@ test "HttpClient: throttled navigations wait for their per-host slot" { defer pool.deinit(); var net: Network = undefined; - net.cache = null; + net.cache = Cache.noop; net.adblocker = null; net.web_bot_auth = null; // An empty pool makes processTransfer queue a started transfer instead diff --git a/src/network/Network.zig b/src/network/Network.zig index 4087f2fda..2c532d010 100644 --- a/src/network/Network.zig +++ b/src/network/Network.zig @@ -26,17 +26,15 @@ const Config = @import("../Config.zig"); const CDP = @import("../cdp/CDP.zig"); const sys_net = @import("../sys/net.zig"); const libcurl = @import("../sys/libcurl.zig"); -const crypto = @import("../sys/libcrypto.zig"); const http = @import("http.zig"); const IpFilter = @import("IpFilter.zig"); const RobotStore = @import("Robots.zig").RobotStore; const WebBotAuth = @import("WebBotAuth.zig"); const RateLimiter = @import("RateLimiter.zig"); -const CurlDebugAllocator = @import("CurlDebugAllocator.zig"); +const Certificates = @import("Certificates.zig"); const Cache = @import("cache/Cache.zig"); -const SqliteCache = @import("cache/SqliteCache.zig"); const AdBlocker = @import("adblock/AdBlocker.zig"); const log = lp.log; @@ -86,13 +84,12 @@ const PSEUDO_POLLFDS = 2; allocator: Allocator, app: *App, -cache: ?Cache, +cache: Cache, config: *const Config, -/// Holds certificate bundle. -x509_store: *crypto.X509_STORE, robot_store: RobotStore, web_bot_auth: ?WebBotAuth, rate_limiter: ?RateLimiter, +certificates: Certificates, /// Hostname dictionaries built from `--adblock-lists`. Parsed once here and /// never mutated afterwards, so every HttpClient can share this one copy. adblocker: ?AdBlocker, @@ -143,31 +140,11 @@ cdp_start: usize, /// Optional IP filter for blocking requests to private/internal networks (--block-private-networks). ip_filter: ?*IpFilter = null, -/// Calling `init` also calls this function; only marked public for situations -/// networking is needed without `App`. -pub fn globalInit(allocator: Allocator) void { - // Only route curl's own allocations through our allocator in Debug, so the - // leak detector sees them. In Release it'd just wrap c_allocator (curl's - // default malloc anyway) at the cost of a per-allocation header. - const curl_allocator = comptime if (lp.IS_DEBUG) CurlDebugAllocator.interface() else null; - if (comptime lp.IS_DEBUG) { - CurlDebugAllocator.init(allocator); - } - - libcurl.curl_global_init(.{ .ssl = true }, curl_allocator) catch |err| { +pub fn init(allocator: Allocator, app: *App, config: *const Config) !Network { + libcurl.curl_global_init(.{ .ssl = true }, null) catch |err| { lp.assert(false, "curl global init", .{ .err = err }); }; -} - -/// Calling `deinit` also calls this function; only marked public for situations -/// networking is needed without `App`. -pub fn globalDeinit() void { - libcurl.curl_global_cleanup(); -} - -pub fn init(allocator: Allocator, app: *App, config: *const Config) !Network { - globalInit(allocator); - errdefer globalDeinit(); + errdefer libcurl.curl_global_cleanup(); const pipe = try sys_net.pipe2(.{ .NONBLOCK = true, .CLOEXEC = true }); @@ -186,22 +163,8 @@ pub fn init(allocator: Allocator, app: *App, config: *const Config) !Network { @memset(pollfds, .{ .fd = -1, .events = 0, .revents = 0 }); pollfds[0] = .{ .fd = pipe[0], .events = posix.POLL.IN, .revents = 0 }; - const x509_store = blk: { - if (config.tlsVerifyHost()) { - break :blk try prepareX509Store(allocator, config); - } - // Verification is off, so the store is never consulted — but still - // take ownership of a user-supplied one so the flags compose and - // nothing leaks. - if (config.customCertStore()) |store| { - log.warn(.app, "custom CA ignored", .{ .arg = "--ca-cert, --ca-path", .reason = "TLS verification disabled" }); - break :blk store; - } - break :blk crypto.X509_STORE_new() orelse { - return error.FailedToCreateX509Store; - }; - }; - errdefer crypto.X509_STORE_free(x509_store); + const certificates = try Certificates.init(allocator, config); + errdefer certificates.deinit(); // IP filter for blocking requests to private/internal networks. const block_private = config.blockPrivateNetworks(); @@ -227,7 +190,7 @@ pub fn init(allocator: Allocator, app: *App, config: *const Config) !Network { var available: DoublyLinkedList = .{}; for (0..count) |i| { - connections[i] = try http.Connection.init(x509_store, config, ip_filter); + connections[i] = try http.Connection.init(certificates, config, ip_filter); available.append(&connections[i].node); } @@ -240,30 +203,14 @@ pub fn init(allocator: Allocator, app: *App, config: *const Config) !Network { var adblocker = try AdBlocker.fromConfig(allocator, config); errdefer if (adblocker) |*blocker| blocker.deinit(); - const cache = if (config.httpCacheDir()) |cache_dir_path| - Cache{ - .kind = .{ - .sqlite = SqliteCache.init( - allocator, - .{ .path = cache_dir_path }, - config.httpCacheEntryLimit(), - ) catch |e| { - log.err(.cache, "failed to init", .{ - .kind = "SqliteCache", - .path = cache_dir_path, - .err = e, - }); - return e; - }, - }, - } - else - null; + var cache = try Cache.init(allocator, config); + errdefer cache.deinit(); return .{ - .allocator = allocator, + .app = app, .config = config, - .x509_store = x509_store, + .allocator = allocator, + .certificates = certificates, .pollfds = pollfds, .wakeup_pipe = pipe, @@ -273,8 +220,6 @@ pub fn init(allocator: Allocator, app: *App, config: *const Config) !Network { .available = available, .connections = connections, - .app = app, - .cache = cache, .robot_store = RobotStore.init(allocator), .web_bot_auth = web_bot_auth, @@ -299,7 +244,7 @@ pub fn deinit(self: *Network) void { self.allocator.free(self.pollfds); self.allocator.free(self.cdp_poll_snapshot); - crypto.X509_STORE_free(self.x509_store); + self.certificates.deinit(); for (self.connections) |*conn| { conn.deinit(); @@ -318,14 +263,14 @@ pub fn deinit(self: *Network) void { if (self.adblocker) |*blocker| blocker.deinit(); - if (self.cache) |*cache| cache.deinit(); + self.cache.deinit(); if (self.ip_filter) |f| { f.deinit(self.allocator); self.allocator.destroy(f); } - globalDeinit(); + libcurl.curl_global_cleanup(); } pub fn bind( @@ -720,7 +665,7 @@ pub fn releaseConnection(self: *Network, conn: *http.Connection) void { self.ws_count -= 1; }, else => { - conn.reset(self.config, self.x509_store, self.ip_filter) catch |err| { + conn.reset(self.config, self.certificates, self.ip_filter) catch |err| { lp.assert(false, "couldn't reset curl easy", .{ .err = err }); }; self.conn_mutex.lockUncancelable(lp.io); @@ -745,7 +690,7 @@ pub fn newConnection(self: *Network) ?*http.Connection { }; // don't do this under lock - conn.* = http.Connection.init(self.x509_store, self.config, self.ip_filter) catch { + conn.* = http.Connection.init(self.certificates, self.config, self.ip_filter) catch { self.ws_mutex.lockUncancelable(lp.io); defer self.ws_mutex.unlock(lp.io); self.ws_pool.destroy(conn); @@ -757,110 +702,6 @@ pub fn newConnection(self: *Network) ?*http.Connection { return conn; } -/// NEVER give full ownership of store to `SSL_CTX`, always rely on ref counting. -/// Allocations made through passed `allocator` are freed before this function returns. -pub fn prepareX509Store(allocator: Allocator, config: *const Config) !*crypto.X509_STORE { - // A user-supplied store replaces system trust entirely. - if (config.customCertStore()) |store| { - return store; - } - return storeFromSystemCA(allocator); -} - -/// Creates an X509_STORE from system root CA. -fn storeFromSystemCA(allocator: Allocator) !*crypto.X509_STORE { - const store = crypto.X509_STORE_new() orelse return error.FailedToCreateX509Store; - errdefer crypto.X509_STORE_free(store); - - var count: usize = 0; - defer { - if (count == 0) { - log.warn(.app, "No certificates loaded", .{}); - } - } - - switch (comptime builtin.os.tag) { - .linux, .openbsd, .netbsd, .freebsd => blk: { - // Iterate over known directories; this may or may not succeed. - const cwd = std.Io.Dir.cwd(); - inline for ([_][]const u8{ - "/etc/ssl/certs", // Debian/Ubuntu/Gentoo/Alpine, SUSE - "/etc/pki/tls/certs", // Fedora/RHEL - }) |dir_path| { - count += try loadFromDirectory(allocator, store, cwd, dir_path); - if (count > 0) break :blk; - } - - // Iterate over known files. - inline for ([_][*:0]const u8{ - "/etc/ssl/certs/ca-certificates.crt", // Debian/Ubuntu/Gentoo - "/etc/pki/tls/certs/ca-bundle.crt", // Fedora/RHEL 6 - "/etc/ssl/ca-bundle.pem", // OpenSUSE - "/etc/pki/tls/cacert.pem", // OpenELEC - "/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem", // CentOS/RHEL 7 - "/etc/ssl/cert.pem", // Alpine, *BSD - }) |file| { - if (crypto.X509_STORE_load_locations(store, file, null) == 1) { - count += 1; - break :blk; - } - } - }, - else => { - // Prefer stdlib's cert scanner. - var bundle: std.crypto.Certificate.Bundle = .empty; - try bundle.rescan(allocator, lp.io, std.Io.Clock.now(.real, lp.io)); - defer bundle.deinit(allocator); - - const bytes = bundle.bytes.items; - var it = bundle.map.valueIterator(); - while (it.next()) |index| { - // d2i_X509 reads the cert's own DER length header to find its end and - // advances `ptr` past it, so we just hand it the rest of the buffer. - var ptr: [*]const u8 = bytes.ptr + index.*; - const x509 = crypto.d2i_X509(null, &ptr, @intCast(bytes.len - index.*)) orelse { - log.warn(.app, "Skipping unparseable system cert", .{}); - continue; - }; - defer crypto.X509_free(x509); // add_cert takes its own ref; drop ours. - - const result = crypto.X509_STORE_add_cert(store, x509); - if (result != 1) { - log.warn(.app, "Failed to add X509 cert to store", .{}); - } - count += 1; - } - }, - } - - return store; -} - -/// Loads certificates from given `path`; returning how many CA loaded. -fn loadFromDirectory( - allocator: Allocator, - store: *crypto.X509_STORE, - cwd: std.Io.Dir, - dir_path: []const u8, -) Allocator.Error!usize { - var count: usize = 0; - var dir = cwd.openDir(lp.io, dir_path, .{ .iterate = true }) catch return count; - defer dir.close(lp.io); - - var it = dir.iterate(); - while (it.next(lp.io) catch return count) |entry| { - if (entry.kind != .file and entry.kind != .sym_link) continue; - - const path = try std.fs.path.joinZ(allocator, &.{ dir_path, entry.name }); - defer allocator.free(path); - - if (crypto.X509_STORE_load_locations(store, path, null) == 1) { - count += 1; - } - } - return count; -} - pub fn HostHashMap(comptime V: type) type { return std.HashMapUnmanaged([]const u8, V, HostContext, 80); } diff --git a/src/network/cache/Cache.zig b/src/network/cache/Cache.zig index 71fd819c7..8b0565881 100644 --- a/src/network/cache/Cache.zig +++ b/src/network/cache/Cache.zig @@ -18,9 +18,13 @@ const std = @import("std"); const lp = @import("lightpanda"); + const Http = @import("../http.zig"); + const SqliteCache = @import("SqliteCache.zig"); +pub var noop: Cache = .{ .kind = .noop }; + const log = lp.log; /// A browser-wide cache for resources across the network. @@ -28,47 +32,85 @@ const log = lp.log; pub const Cache = @This(); kind: union(enum) { + noop: void, sqlite: SqliteCache, }, +pub fn init(allocator: std.mem.Allocator, config: *const lp.Config) !Cache { + const cache_path = config.httpCacheDir() orelse { + return .{ .kind = .noop }; + }; + + const sqlite = SqliteCache.init( + allocator, + .{ .path = cache_path }, + config.httpCacheEntryLimit(), + ) catch |err| { + log.err(.cache, "failed to init", .{ + .kind = "SqliteCache", + .path = cache_path, + .err = err, + }); + return err; + }; + + return .{ + .kind = .{ .sqlite = sqlite }, + }; +} + pub fn deinit(self: *Cache) void { return switch (self.kind) { + .noop => {}, inline else => |*c| c.deinit(), }; } +pub fn active(self: *Cache) ?*Cache { + return switch (self.kind) { + .noop => null, + inline else => self, + }; +} + pub fn get(self: *Cache, arena: std.mem.Allocator, req: CacheGetRequest) !CacheGetResult { return switch (self.kind) { + .noop => .miss, inline else => |*c| c.get(arena, req), }; } pub fn put(self: *Cache, req: CachePutRequest, body: []const u8) !void { return switch (self.kind) { + .noop => {}, inline else => |*c| c.put(req, body), }; } pub fn evict(self: *Cache, url: []const u8) void { return switch (self.kind) { + .noop => {}, inline else => |*c| c.evict(url), }; } pub fn renew(self: *Cache, arena: std.mem.Allocator, req: RenewResponse) !void { return switch (self.kind) { + .noop => {}, inline else => |*c| c.renew(arena, req), }; } pub fn clear(self: *Cache) !void { return switch (self.kind) { + .noop => {}, inline else => |*c| c.clear(), }; } pub fn maintenance(self: *Cache, now: u64) void { return switch (self.kind) { + .noop => {}, inline else => |*c| c.maintenance(now), }; } diff --git a/src/network/http.zig b/src/network/http.zig index 11c601bbe..2442b5806 100644 --- a/src/network/http.zig +++ b/src/network/http.zig @@ -17,7 +17,7 @@ // along with this program. If not, see . const std = @import("std"); -const posix = std.posix; +const lp = @import("lightpanda"); const Config = @import("../Config.zig"); const sys_net = @import("../sys/net.zig"); @@ -25,8 +25,10 @@ const libcurl = @import("../sys/libcurl.zig"); const crypto = @import("../sys/libcrypto.zig"); const IpFilter = @import("IpFilter.zig"); +const Certificates = @import("Certificates.zig"); -const log = @import("lightpanda").log; +const log = lp.log; +const posix = std.posix; pub const ENABLE_DEBUG = false; @@ -280,7 +282,7 @@ pub const Connection = struct { }; pub fn init( - x509_store: *crypto.X509_STORE, + certificates: Certificates, config: *const Config, ip_filter: ?*const IpFilter, ) !Connection { @@ -289,7 +291,7 @@ pub const Connection = struct { var self = Connection{ ._easy = easy, .transport = .none }; errdefer self.deinit(); - try self.reset(config, x509_store, ip_filter); + try self.reset(config, certificates, ip_filter); return self; } @@ -443,7 +445,7 @@ pub const Connection = struct { pub fn reset( self: *Connection, config: *const Config, - x509_store: *crypto.X509_STORE, + certificates: Certificates, ip_filter: ?*const IpFilter, ) !void { libcurl.curl_easy_reset(self._easy); @@ -487,7 +489,7 @@ pub const Connection = struct { } }).wrap); // Pass our store to CURLOPT_SSL_CTX_FUNCTION. - try libcurl.curl_easy_setopt(self._easy, .ssl_ctx_data, x509_store); + try libcurl.curl_easy_setopt(self._easy, .ssl_ctx_data, certificates.store); } else { try libcurl.curl_easy_setopt(self._easy, .ssl_verify_host, false); try libcurl.curl_easy_setopt(self._easy, .ssl_verify_peer, false); diff --git a/src/telemetry/lightpanda.zig b/src/telemetry/lightpanda.zig index 4201ae22e..133dc1fcb 100644 --- a/src/telemetry/lightpanda.zig +++ b/src/telemetry/lightpanda.zig @@ -132,7 +132,7 @@ pub fn send(self: *LightPanda, raw_event: telemetry.Event) !void { fn run(self: *LightPanda) void { // The connection is created, owned, and torn down entirely on this thread; // the network thread never sees it (Transport == .none). - var conn = http.Connection.init(self.network.x509_store, self.network.config, self.network.ip_filter) catch |err| { + var conn = http.Connection.init(self.network.certificates, self.network.config, self.network.ip_filter) catch |err| { // Essentially OOM — the process is already in trouble. The thread // handle stays set so send() won't respawn; events drop at the cap. log.warn(.telemetry, "connection init", .{ .err = err });