mirror/browser
1
0 Fork 0
mirror of https://github.com/lightpanda-io/browser.git synced 2026-09-18 09:54:36 -04:00
Code Issues Packages Projects Releases Wiki Activity

chore: Network cleanup

Browse Source 
Network has accumulated a bit of bagged. It knows a lot about certificates, it
knows a lot of the cache. I have plans to expand Network, and wanted to clean it
up.

1 - certificate logic moved to dedicated Certificates.zig
2 - Removed CurlDebugAllocator
    - this makes setup easier, to Updater can just init libcurl directly
3 - Change Updater to be a single function
4 - Cache initialization is don in the Cache
5 - ?Cache => Cache{.kind = .noop}
This commit is contained in:
Karl Seguin committed 2026-08-22 07:04:58 +08:00
1 parent 9f4b39d723
commit decedca6e7
12 files changed
+257 -396
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/Updater.zig
+31 -51
View File
@@ -19,70 +19,31 @@
const std = @import("std");
const lp = @import("lightpanda");
const Network = @import("network/Network.zig");
const http = @import("network/http.zig");
const Certificates = @import("network/Certificates.zig");
const libcurl = @import("sys/libcurl.zig");
const crypto = @import("sys/libcrypto.zig");
const Config = @import("Config.zig");
const Allocator = std.mem.Allocator;
/// Sole purpose of this client is to do updates; hence, its very minimal.
const Updater = @This();
x509_store: *crypto.X509_STORE,
config: *const Config,
/// Initializes the update client; meant to be used as singleton.
pub fn init(allocator: Allocator, config: *const Config) !Updater {
Network.globalInit(allocator);
errdefer Network.globalDeinit();
const x509_store = try Network.prepareX509Store(allocator, config);
return .{
.x509_store = x509_store,
.config = config,
};
}
pub fn deinit(self: *Updater) void {
Network.globalDeinit();
crypto.X509_STORE_free(self.x509_store);
}
/// Sends running Lightpanda version to remote to get update information.
/// Outputs directly to given `Writer`.
pub fn inform(self: *Updater, writer: *std.Io.Writer) !void {
var conn = try http.Connection.init(self.x509_store, self.config, null);
pub fn inform(allocator: Allocator, config: *const lp.Config, writer: *std.Io.Writer) !void {
const certificates = try Certificates.init(allocator, config);
defer certificates.deinit();
libcurl.curl_global_init(.{ .ssl = true }, null) catch |err| {
lp.assert(false, "curl global init", .{ .err = err });
};
defer libcurl.curl_global_cleanup();
var conn = try http.Connection.init(certificates, config, null);
defer conn.deinit();
const url = std.fmt.comptimePrint("https://telemetry.lightpanda.io/v/{s}", .{lp.build_config.version});
// Prepare the request.
try conn.setURL(url);
try conn.setURL("https://telemetry.lightpanda.io/v/" ++ lp.build_config.version);
try conn.setGetMode();
try conn.setFollowLocation(true);
// Wraps everything needed to receive bytes.
const ReceiverContext = struct {
writer: *std.Io.Writer,
err: std.Io.Writer.Error!void = {},
/// curl -> writer.
fn drain(
buffer: [*]const u8,
buf_count: usize,
buf_len: usize,
raw_ctx: *anyopaque,
) callconv(.c) usize {
const ctx: *@This() = @ptrCast(@alignCast(raw_ctx));
const chunk = buffer[0 .. buf_count * buf_len];
ctx.writer.writeAll(chunk) catch |err| {
ctx.err = err;
return 0;
};
return chunk.len;
}
};
// Set receiver context.
var ctx = ReceiverContext{ .writer = writer };
@@ -97,3 +58,22 @@ pub fn inform(self: *Updater, writer: *std.Io.Writer) !void {
_ = status_int;
return writer.flush();
}
const ReceiverContext = struct {
writer: *std.Io.Writer,
err: std.Io.Writer.Error!void = {},
fn drain(buffer: [*]const u8, buf_count: usize, buf_len: usize, ctx: *anyopaque) callconv(.c) usize {
// libcurl only ever sends 1 buffer
std.debug.assert(buf_count == 1);
const self: *ReceiverContext = @ptrCast(@alignCast(ctx));
const chunk = buffer[0..buf_len];
self.writer.writeAll(chunk) catch |err| {
self.err = err;
return 0;
};
return buf_len;
}
};
src/browser/Browser.zig
-3
View File
@@ -153,9 +153,6 @@ pub fn deinit(self: *Browser) void {
// fire — only now is it safe to free the pool backing their parameters.
self.fc_identity_pool.deinit(allocator);
self.page_pool.deinit(allocator);
if (self.http_client.cache) |cache| {
cache.maintenance(lp.datetime.timestamp(.real));
}
self.http_client.deinit();
self.clearPermissions();
self.permissions.deinit(allocator);
src/cdp/domains/network.zig
+4 -4
View File
@@ -216,14 +216,14 @@ fn clearBrowserCache(cmd: *CDP.Command) !void {
// Chrome accepts that and clears the jar; reject only on truly malformed JSON.
const bc = cmd.browser_context orelse return error.BrowserContextNotLoaded;
const network = bc.cdp.browser.http_client.network;
if (network.cache) |*c| try c.clear();
try network.cache.clear();
return cmd.sendResult(null, .{});
}
fn canClearBrowserCache(cmd: *CDP.Command) !void {
const bc = cmd.browser_context orelse return error.BrowserContextNotLoaded;
const network = bc.cdp.browser.http_client.network;
return cmd.sendResult(.{ .result = network.cache != null }, .{});
return cmd.sendResult(.{ .result = network.cache.active() != null }, .{});
}
fn clearBrowserCookies(cmd: *CDP.Command) !void {
@@ -1030,7 +1030,7 @@ test "cdp.Network: setCacheDisabled" {
.params = .{ .cacheDisabled = true },
});
try ctx.expectSentResult(null, .{ .id = 1 });
try testing.expect(client.cache == null);
try testing.expectEqual(null, client.cache.active());
}
test "cdp.Network: configured CDP ignores setCacheDisabled" {
@@ -1041,7 +1041,7 @@ test "cdp.Network: configured CDP ignores setCacheDisabled" {
var cache: Cache = undefined;
const client = &ctx.cdp().browser.http_client;
client.cache = &cache;
defer client.cache = null;
defer client.cache = &testing.base.test_app.network.cache;
try ctx.processMessage(.{
.id = 1,
src/cdp/testing.zig
+1 -1
View File
@@ -20,7 +20,7 @@ const std = @import("std");
const CDP = @import("CDP.zig");
const base = @import("../testing.zig");
pub const base = @import("../testing.zig");
const json = std.json;
const posix = std.posix;
src/lightpanda.zig
+1 -7
View File
@@ -60,8 +60,6 @@ pub const build_config = @import("build_config");
pub const crash_handler = @import("crash_handler.zig");
pub const core_dump = @import("core_dump.zig");
pub const Updater = @import("Updater.zig");
pub var metrics = @import("Metrics.zig"){};
pub const IS_TEST = @import("builtin").is_test;
@@ -395,14 +393,10 @@ fn dumpContent(app: *App, mode: Config.DumpFormat, dump_opts: dump.Opts, frame:
}
pub fn checkVersion(allocator: std.mem.Allocator, config: *const Config) !void {
var client = try Updater.init(allocator, config);
defer client.deinit();
const stdout = std.Io.File.stdout();
var buf: [4096]u8 = undefined;
var writer = stdout.writer(io, &buf);
const w = &writer.interface;
try client.inform(w);
try @import("Updater.zig").inform(allocator, config, &writer.interface);
}
// Writes a single page's result object. Framing (the enclosing array and any
src/network/Certificates.zig
+139
View File
@@ -0,0 +1,139 @@
// Copyright (C) 2023-2026 Lightpanda (Selecy SAS)
//
// Francis Bouvier <francis@lightpanda.io>
// Pierre Tachoire <pierre@lightpanda.io>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
const std = @import("std");
const lp = @import("lightpanda");
const crypto = @import("../sys/libcrypto.zig");
const log = lp.log;
const Config = lp.Config;
const Allocator = std.mem.Allocator;
const Certificates = @This();
store: *crypto.X509_STORE,
pub fn init(allocator: Allocator, config: *const Config) !Certificates {
const store = blk: {
const custom_store = config.customCertStore();
if (config.tlsVerifyHost() == false) {
if (custom_store != null) {
log.warn(.app, "custom CA ignored", .{
.arg = "--ca-cert, --ca-path",
.reason = "TLS verification disabled",
});
}
break :blk crypto.X509_STORE_new() orelse return error.FailedToCreateX509Store;
}
break :blk custom_store orelse try storeFromSystemCA(allocator);
};
return .{ .store = store };
}
pub fn deinit(self: Certificates) void {
crypto.X509_STORE_free(self.store);
}
fn storeFromSystemCA(allocator: Allocator) !*crypto.X509_STORE {
const store = crypto.X509_STORE_new() orelse return error.FailedToCreateX509Store;
errdefer crypto.X509_STORE_free(store);
var count: usize = 0;
defer {
if (count == 0) {
log.warn(.app, "No certificates loaded", .{});
}
}
switch (comptime @import("builtin").os.tag) {
.linux, .openbsd, .netbsd, .freebsd => blk: {
// Iterate over known directories; this may or may not succeed.
const cwd = std.Io.Dir.cwd();
inline for ([_][]const u8{
"/etc/ssl/certs", // Debian/Ubuntu/Gentoo/Alpine, SUSE
"/etc/pki/tls/certs", // Fedora/RHEL
}) |dir_path| {
count += try loadFromDirectory(allocator, store, cwd, dir_path);
if (count > 0) break :blk;
}
// Iterate over known files.
inline for ([_][*:0]const u8{
"/etc/ssl/certs/ca-certificates.crt", // Debian/Ubuntu/Gentoo
"/etc/pki/tls/certs/ca-bundle.crt", // Fedora/RHEL 6
"/etc/ssl/ca-bundle.pem", // OpenSUSE
"/etc/pki/tls/cacert.pem", // OpenELEC
"/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem", // CentOS/RHEL 7
"/etc/ssl/cert.pem", // Alpine, *BSD
}) |file| {
if (crypto.X509_STORE_load_locations(store, file, null) == 1) {
count += 1;
break :blk;
}
}
},
else => {
// Prefer stdlib's cert scanner.
var bundle: std.crypto.Certificate.Bundle = .empty;
try bundle.rescan(allocator, lp.io, std.Io.Clock.now(.real, lp.io));
defer bundle.deinit(allocator);
const bytes = bundle.bytes.items;
var it = bundle.map.valueIterator();
while (it.next()) |index| {
// d2i_X509 reads the cert's own DER length header to find its end and
// advances `ptr` past it, so we just hand it the rest of the buffer.
var ptr: [*]const u8 = bytes.ptr + index.*;
const x509 = crypto.d2i_X509(null, &ptr, @intCast(bytes.len - index.*)) orelse {
log.warn(.app, "Skipping unparseable system cert", .{});
continue;
};
defer crypto.X509_free(x509); // add_cert takes its own ref; drop ours.
const result = crypto.X509_STORE_add_cert(store, x509);
if (result != 1) {
log.warn(.app, "Failed to add X509 cert to store", .{});
}
count += 1;
}
},
}
return store;
}
fn loadFromDirectory(allocator: Allocator, store: *crypto.X509_STORE, cwd: std.Io.Dir, dir_path: []const u8) !usize {
var count: usize = 0;
var dir = cwd.openDir(lp.io, dir_path, .{ .iterate = true }) catch return count;
defer dir.close(lp.io);
var it = dir.iterate();
while (it.next(lp.io) catch return count) |entry| {
if (entry.kind != .file and entry.kind != .sym_link) continue;
const path = try std.fs.path.joinZ(allocator, &.{ dir_path, entry.name });
defer allocator.free(path);
if (crypto.X509_STORE_load_locations(store, path, null) == 1) {
count += 1;
}
}
return count;
}
src/network/CurlDebugAllocator.zig
-134
View File
@@ -1,134 +0,0 @@
// Copyright (C) 2023-2026 Lightpanda (Selecy SAS)
//
// Francis Bouvier <francis@lightpanda.io>
// Pierre Tachoire <pierre@lightpanda.io>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
const std = @import("std");
const lp = @import("lightpanda");
const libcurl = @import("../sys/libcurl.zig");
const Allocator = std.mem.Allocator;
const CurlDebugAllocator = @This();
// C11 requires malloc to return memory aligned to max_align_t (16 bytes on x86_64).
// We match this guarantee since libcurl expects malloc-compatible alignment.
const alignment = 16;
var instance: ?CurlDebugAllocator = null;
allocator: Allocator,
pub fn init(allocator: Allocator) void {
lp.assert(instance == null, "Initialization of curl must happen only once", .{});
instance = .{ .allocator = allocator };
}
pub fn interface() libcurl.CurlAllocator {
return .{
.free = free,
.strdup = strdup,
.malloc = malloc,
.calloc = calloc,
.realloc = realloc,
};
}
fn _allocBlock(size: usize) ?*Block {
const slice = instance.?.allocator.alignedAlloc(u8, .fromByteUnits(alignment), Block.fullsize(size)) catch return null;
const block: *Block = @ptrCast(@alignCast(slice.ptr));
block.size = size;
return block;
}
fn _freeBlock(header: *Block) void {
instance.?.allocator.free(header.slice());
}
fn malloc(size: usize) ?*anyopaque {
const block = _allocBlock(size) orelse return null;
return @ptrCast(block.data());
}
fn calloc(nmemb: usize, size: usize) ?*anyopaque {
const total = nmemb * size;
const block = _allocBlock(total) orelse return null;
const ptr = block.data();
@memset(ptr[0..total], 0); // for historical reasons, calloc zeroes memory, but malloc does not.
return @ptrCast(ptr);
}
fn realloc(ptr: ?*anyopaque, size: usize) ?*anyopaque {
const p = ptr orelse return malloc(size);
const block = Block.fromPtr(p);
const old_size = block.size;
if (size == old_size) return ptr;
if (instance.?.allocator.resize(block.slice(), alignment + size)) {
block.size = size;
return ptr;
}
const copy_size = @min(old_size, size);
const new_block = _allocBlock(size) orelse return null;
@memcpy(new_block.data()[0..copy_size], block.data()[0..copy_size]);
_freeBlock(block);
return @ptrCast(new_block.data());
}
fn free(ptr: ?*anyopaque) void {
const p = ptr orelse return;
_freeBlock(Block.fromPtr(p));
}
fn strdup(str: [*:0]const u8) ?[*:0]u8 {
const len = std.mem.len(str);
const header = _allocBlock(len + 1) orelse return null;
const ptr = header.data();
@memcpy(ptr[0..len], str[0..len]);
ptr[len] = 0;
return ptr[0..len :0];
}
const Block = extern struct {
size: usize = 0,
_padding: [alignment - @sizeOf(usize)]u8 = .{0} ** (alignment - @sizeOf(usize)),
inline fn fullsize(bytes: usize) usize {
return alignment + bytes;
}
inline fn fromPtr(ptr: *anyopaque) *Block {
const raw: [*]u8 = @ptrCast(ptr);
return @ptrCast(@alignCast(raw - @sizeOf(Block)));
}
inline fn data(self: *Block) [*]u8 {
const ptr: [*]u8 = @ptrCast(self);
return ptr + @sizeOf(Block);
}
inline fn slice(self: *Block) []align(alignment) u8 {
const base: [*]align(alignment) u8 = @ptrCast(@alignCast(self));
return base[0 .. alignment + self.size];
}
};
comptime {
std.debug.assert(@sizeOf(Block) == alignment);
}
src/network/HttpClient.zig
+11 -11
View File
@@ -181,8 +181,7 @@ blocking_requests: std.AutoHashMapUnmanaged(u32, u32) = .empty,
// heuristics add this in.
intercepted: usize = 0,
// null or referencing network.cache
cache: ?*Cache,
cache: *Cache,
// Cached config decisions, resolved once at init.
serve_mode: bool,
@@ -218,7 +217,7 @@ pub fn init(self: *Client, allocator: Allocator, network: *Network, cdp: ?*CDP)
.allocator = allocator,
.cdp = cdp,
.inbox = .{},
.cache = if (network.cache) |*c| c else null,
.cache = &network.cache,
.use_proxy = http_proxy != null,
.http_proxy = http_proxy,
@@ -265,6 +264,7 @@ pub fn deinit(self: *Client) void {
self.blocking_requests.deinit(self.allocator);
self.transfers.deinit(self.allocator);
self.inbox.deinit();
self.cache.maintenance(lp.datetime.timestamp(.real));
}
// Look up a live transfer by its id. Returns null if the transfer has been
@@ -317,9 +317,9 @@ pub fn obeyRobots(self: *Client, enable: bool) !void {
pub fn disableCache(self: *Client, disable: bool) void {
if (disable) {
self.cache = null;
self.cache = &Cache.noop;
} else {
self.cache = if (self.network.cache) |*c| c else null;
self.cache = &self.network.cache;
}
}
@@ -984,7 +984,7 @@ fn findHeader(headers: []const http.Header, name: []const u8) ?[]const u8 {
// the response; on an expired-with-validators entry the request becomes a
// conditional revalidation.
fn cacheLookup(self: *Client, transfer: *Transfer) !bool {
const cache = self.cache orelse return false;
const cache = self.cache.active() orelse return false;
const req = &transfer.req;
if (req.method != .GET or req.streaming or req.skip_cache) {
@@ -1067,7 +1067,7 @@ fn cacheRevalidated(self: *Client, transfer: *Transfer) !bool {
return false;
}
// could have been disabled in-between
const cache = self.cache orelse return false;
const cache = self.cache.active() orelse return false;
const stale = transfer._cache_intent.revalidate;
transfer._cache_intent = .none;
@@ -1103,7 +1103,7 @@ fn cacheStore(self: *Client, transfer: *Transfer) void {
transfer._cache_intent = .none;
// could have been disabled while waiting of the response
const cache = self.cache orelse return;
const cache = self.cache.active() orelse return;
const arena = transfer.arena;
const rh = &(transfer.res.header orelse return);
@@ -3582,7 +3582,7 @@ fn initTestClient(client: *Client, pool: *ArenaPool) void {
client.intercepted = 0;
client.http_active = 0;
client.ws_active = 0;
client.cache = null;
client.cache = &Cache.noop;
client.serve_mode = false;
client.obey_robots = false;
client.robots = .{
@@ -3951,7 +3951,7 @@ test "HttpClient: fulfillIntercepted follows a 3xx redirect" {
// Only network.config (httpMaxRedirects, which ignores its config),
// network.cache and the (empty) connection pool are read on this path.
var net: Network = undefined;
net.cache = null;
net.cache = Cache.noop;
net.adblocker = null;
// An empty pool makes processTransfer queue the re-issued request
// instead of putting it on the wire — the queue IS the capture.
@@ -4309,7 +4309,7 @@ test "HttpClient: throttled navigations wait for their per-host slot" {
defer pool.deinit();
var net: Network = undefined;
net.cache = null;
net.cache = Cache.noop;
net.adblocker = null;
net.web_bot_auth = null;
// An empty pool makes processTransfer queue a started transfer instead
src/network/Network.zig
+19 -178
View File
@@ -26,17 +26,15 @@ const Config = @import("../Config.zig");
const CDP = @import("../cdp/CDP.zig");
const sys_net = @import("../sys/net.zig");
const libcurl = @import("../sys/libcurl.zig");
const crypto = @import("../sys/libcrypto.zig");
const http = @import("http.zig");
const IpFilter = @import("IpFilter.zig");
const RobotStore = @import("Robots.zig").RobotStore;
const WebBotAuth = @import("WebBotAuth.zig");
const RateLimiter = @import("RateLimiter.zig");
const CurlDebugAllocator = @import("CurlDebugAllocator.zig");
const Certificates = @import("Certificates.zig");
const Cache = @import("cache/Cache.zig");
const SqliteCache = @import("cache/SqliteCache.zig");
const AdBlocker = @import("adblock/AdBlocker.zig");
const log = lp.log;
@@ -86,13 +84,12 @@ const PSEUDO_POLLFDS = 2;
allocator: Allocator,
app: *App,
cache: ?Cache,
cache: Cache,
config: *const Config,
/// Holds certificate bundle.
x509_store: *crypto.X509_STORE,
robot_store: RobotStore,
web_bot_auth: ?WebBotAuth,
rate_limiter: ?RateLimiter,
certificates: Certificates,
/// Hostname dictionaries built from `--adblock-lists`. Parsed once here and
/// never mutated afterwards, so every HttpClient can share this one copy.
adblocker: ?AdBlocker,
@@ -143,31 +140,11 @@ cdp_start: usize,
/// Optional IP filter for blocking requests to private/internal networks (--block-private-networks).
ip_filter: ?*IpFilter = null,
/// Calling `init` also calls this function; only marked public for situations
/// networking is needed without `App`.
pub fn globalInit(allocator: Allocator) void {
// Only route curl's own allocations through our allocator in Debug, so the
// leak detector sees them. In Release it'd just wrap c_allocator (curl's
// default malloc anyway) at the cost of a per-allocation header.
const curl_allocator = comptime if (lp.IS_DEBUG) CurlDebugAllocator.interface() else null;
if (comptime lp.IS_DEBUG) {
CurlDebugAllocator.init(allocator);
}
libcurl.curl_global_init(.{ .ssl = true }, curl_allocator) catch |err| {
pub fn init(allocator: Allocator, app: *App, config: *const Config) !Network {
libcurl.curl_global_init(.{ .ssl = true }, null) catch |err| {
lp.assert(false, "curl global init", .{ .err = err });
};
}
/// Calling `deinit` also calls this function; only marked public for situations
/// networking is needed without `App`.
pub fn globalDeinit() void {
libcurl.curl_global_cleanup();
}
pub fn init(allocator: Allocator, app: *App, config: *const Config) !Network {
globalInit(allocator);
errdefer globalDeinit();
errdefer libcurl.curl_global_cleanup();
const pipe = try sys_net.pipe2(.{ .NONBLOCK = true, .CLOEXEC = true });
@@ -186,22 +163,8 @@ pub fn init(allocator: Allocator, app: *App, config: *const Config) !Network {
@memset(pollfds, .{ .fd = -1, .events = 0, .revents = 0 });
pollfds[0] = .{ .fd = pipe[0], .events = posix.POLL.IN, .revents = 0 };
const x509_store = blk: {
if (config.tlsVerifyHost()) {
break :blk try prepareX509Store(allocator, config);
}
// Verification is off, so the store is never consulted — but still
// take ownership of a user-supplied one so the flags compose and
// nothing leaks.
if (config.customCertStore()) |store| {
log.warn(.app, "custom CA ignored", .{ .arg = "--ca-cert, --ca-path", .reason = "TLS verification disabled" });
break :blk store;
}
break :blk crypto.X509_STORE_new() orelse {
return error.FailedToCreateX509Store;
};
};
errdefer crypto.X509_STORE_free(x509_store);
const certificates = try Certificates.init(allocator, config);
errdefer certificates.deinit();
// IP filter for blocking requests to private/internal networks.
const block_private = config.blockPrivateNetworks();
@@ -227,7 +190,7 @@ pub fn init(allocator: Allocator, app: *App, config: *const Config) !Network {
var available: DoublyLinkedList = .{};
for (0..count) |i| {
connections[i] = try http.Connection.init(x509_store, config, ip_filter);
connections[i] = try http.Connection.init(certificates, config, ip_filter);
available.append(&connections[i].node);
}
@@ -240,30 +203,14 @@ pub fn init(allocator: Allocator, app: *App, config: *const Config) !Network {
var adblocker = try AdBlocker.fromConfig(allocator, config);
errdefer if (adblocker) |*blocker| blocker.deinit();
const cache = if (config.httpCacheDir()) |cache_dir_path|
Cache{
.kind = .{
.sqlite = SqliteCache.init(
allocator,
.{ .path = cache_dir_path },
config.httpCacheEntryLimit(),
) catch |e| {
log.err(.cache, "failed to init", .{
.kind = "SqliteCache",
.path = cache_dir_path,
.err = e,
});
return e;
},
},
}
else
null;
var cache = try Cache.init(allocator, config);
errdefer cache.deinit();
return .{
.allocator = allocator,
.app = app,
.config = config,
.x509_store = x509_store,
.allocator = allocator,
.certificates = certificates,
.pollfds = pollfds,
.wakeup_pipe = pipe,
@@ -273,8 +220,6 @@ pub fn init(allocator: Allocator, app: *App, config: *const Config) !Network {
.available = available,
.connections = connections,
.app = app,
.cache = cache,
.robot_store = RobotStore.init(allocator),
.web_bot_auth = web_bot_auth,
@@ -299,7 +244,7 @@ pub fn deinit(self: *Network) void {
self.allocator.free(self.pollfds);
self.allocator.free(self.cdp_poll_snapshot);
crypto.X509_STORE_free(self.x509_store);
self.certificates.deinit();
for (self.connections) |*conn| {
conn.deinit();
@@ -318,14 +263,14 @@ pub fn deinit(self: *Network) void {
if (self.adblocker) |*blocker| blocker.deinit();
if (self.cache) |*cache| cache.deinit();
self.cache.deinit();
if (self.ip_filter) |f| {
f.deinit(self.allocator);
self.allocator.destroy(f);
}
globalDeinit();
libcurl.curl_global_cleanup();
}
pub fn bind(
@@ -720,7 +665,7 @@ pub fn releaseConnection(self: *Network, conn: *http.Connection) void {
self.ws_count -= 1;
},
else => {
conn.reset(self.config, self.x509_store, self.ip_filter) catch |err| {
conn.reset(self.config, self.certificates, self.ip_filter) catch |err| {
lp.assert(false, "couldn't reset curl easy", .{ .err = err });
};
self.conn_mutex.lockUncancelable(lp.io);
@@ -745,7 +690,7 @@ pub fn newConnection(self: *Network) ?*http.Connection {
};
// don't do this under lock
conn.* = http.Connection.init(self.x509_store, self.config, self.ip_filter) catch {
conn.* = http.Connection.init(self.certificates, self.config, self.ip_filter) catch {
self.ws_mutex.lockUncancelable(lp.io);
defer self.ws_mutex.unlock(lp.io);
self.ws_pool.destroy(conn);
@@ -757,110 +702,6 @@ pub fn newConnection(self: *Network) ?*http.Connection {
return conn;
}
/// NEVER give full ownership of store to `SSL_CTX`, always rely on ref counting.
/// Allocations made through passed `allocator` are freed before this function returns.
pub fn prepareX509Store(allocator: Allocator, config: *const Config) !*crypto.X509_STORE {
// A user-supplied store replaces system trust entirely.
if (config.customCertStore()) |store| {
return store;
}
return storeFromSystemCA(allocator);
}
/// Creates an X509_STORE from system root CA.
fn storeFromSystemCA(allocator: Allocator) !*crypto.X509_STORE {
const store = crypto.X509_STORE_new() orelse return error.FailedToCreateX509Store;
errdefer crypto.X509_STORE_free(store);
var count: usize = 0;
defer {
if (count == 0) {
log.warn(.app, "No certificates loaded", .{});
}
}
switch (comptime builtin.os.tag) {
.linux, .openbsd, .netbsd, .freebsd => blk: {
// Iterate over known directories; this may or may not succeed.
const cwd = std.Io.Dir.cwd();
inline for ([_][]const u8{
"/etc/ssl/certs", // Debian/Ubuntu/Gentoo/Alpine, SUSE
"/etc/pki/tls/certs", // Fedora/RHEL
}) |dir_path| {
count += try loadFromDirectory(allocator, store, cwd, dir_path);
if (count > 0) break :blk;
}
// Iterate over known files.
inline for ([_][*:0]const u8{
"/etc/ssl/certs/ca-certificates.crt", // Debian/Ubuntu/Gentoo
"/etc/pki/tls/certs/ca-bundle.crt", // Fedora/RHEL 6
"/etc/ssl/ca-bundle.pem", // OpenSUSE
"/etc/pki/tls/cacert.pem", // OpenELEC
"/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem", // CentOS/RHEL 7
"/etc/ssl/cert.pem", // Alpine, *BSD
}) |file| {
if (crypto.X509_STORE_load_locations(store, file, null) == 1) {
count += 1;
break :blk;
}
}
},
else => {
// Prefer stdlib's cert scanner.
var bundle: std.crypto.Certificate.Bundle = .empty;
try bundle.rescan(allocator, lp.io, std.Io.Clock.now(.real, lp.io));
defer bundle.deinit(allocator);
const bytes = bundle.bytes.items;
var it = bundle.map.valueIterator();
while (it.next()) |index| {
// d2i_X509 reads the cert's own DER length header to find its end and
// advances `ptr` past it, so we just hand it the rest of the buffer.
var ptr: [*]const u8 = bytes.ptr + index.*;
const x509 = crypto.d2i_X509(null, &ptr, @intCast(bytes.len - index.*)) orelse {
log.warn(.app, "Skipping unparseable system cert", .{});
continue;
};
defer crypto.X509_free(x509); // add_cert takes its own ref; drop ours.
const result = crypto.X509_STORE_add_cert(store, x509);
if (result != 1) {
log.warn(.app, "Failed to add X509 cert to store", .{});
}
count += 1;
}
},
}
return store;
}
/// Loads certificates from given `path`; returning how many CA loaded.
fn loadFromDirectory(
allocator: Allocator,
store: *crypto.X509_STORE,
cwd: std.Io.Dir,
dir_path: []const u8,
) Allocator.Error!usize {
var count: usize = 0;
var dir = cwd.openDir(lp.io, dir_path, .{ .iterate = true }) catch return count;
defer dir.close(lp.io);
var it = dir.iterate();
while (it.next(lp.io) catch return count) |entry| {
if (entry.kind != .file and entry.kind != .sym_link) continue;
const path = try std.fs.path.joinZ(allocator, &.{ dir_path, entry.name });
defer allocator.free(path);
if (crypto.X509_STORE_load_locations(store, path, null) == 1) {
count += 1;
}
}
return count;
}
pub fn HostHashMap(comptime V: type) type {
return std.HashMapUnmanaged([]const u8, V, HostContext, 80);
}
src/network/cache/Cache.zig
Vendored
+42
View File
@@ -18,9 +18,13 @@
const std = @import("std");
const lp = @import("lightpanda");
const Http = @import("../http.zig");
const SqliteCache = @import("SqliteCache.zig");
pub var noop: Cache = .{ .kind = .noop };
const log = lp.log;
/// A browser-wide cache for resources across the network.
@@ -28,47 +32,85 @@ const log = lp.log;
pub const Cache = @This();
kind: union(enum) {
noop: void,
sqlite: SqliteCache,
},
pub fn init(allocator: std.mem.Allocator, config: *const lp.Config) !Cache {
const cache_path = config.httpCacheDir() orelse {
return .{ .kind = .noop };
};
const sqlite = SqliteCache.init(
allocator,
.{ .path = cache_path },
config.httpCacheEntryLimit(),
) catch |err| {
log.err(.cache, "failed to init", .{
.kind = "SqliteCache",
.path = cache_path,
.err = err,
});
return err;
};
return .{
.kind = .{ .sqlite = sqlite },
};
}
pub fn deinit(self: *Cache) void {
return switch (self.kind) {
.noop => {},
inline else => |*c| c.deinit(),
};
}
pub fn active(self: *Cache) ?*Cache {
return switch (self.kind) {
.noop => null,
inline else => self,
};
}
pub fn get(self: *Cache, arena: std.mem.Allocator, req: CacheGetRequest) !CacheGetResult {
return switch (self.kind) {
.noop => .miss,
inline else => |*c| c.get(arena, req),
};
}
pub fn put(self: *Cache, req: CachePutRequest, body: []const u8) !void {
return switch (self.kind) {
.noop => {},
inline else => |*c| c.put(req, body),
};
}
pub fn evict(self: *Cache, url: []const u8) void {
return switch (self.kind) {
.noop => {},
inline else => |*c| c.evict(url),
};
}
pub fn renew(self: *Cache, arena: std.mem.Allocator, req: RenewResponse) !void {
return switch (self.kind) {
.noop => {},
inline else => |*c| c.renew(arena, req),
};
}
pub fn clear(self: *Cache) !void {
return switch (self.kind) {
.noop => {},
inline else => |*c| c.clear(),
};
}
pub fn maintenance(self: *Cache, now: u64) void {
return switch (self.kind) {
.noop => {},
inline else => |*c| c.maintenance(now),
};
}
src/network/http.zig
+8 -6
View File
@@ -17,7 +17,7 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
const std = @import("std");
const posix = std.posix;
const lp = @import("lightpanda");
const Config = @import("../Config.zig");
const sys_net = @import("../sys/net.zig");
@@ -25,8 +25,10 @@ const libcurl = @import("../sys/libcurl.zig");
const crypto = @import("../sys/libcrypto.zig");
const IpFilter = @import("IpFilter.zig");
const Certificates = @import("Certificates.zig");
const log = @import("lightpanda").log;
const log = lp.log;
const posix = std.posix;
pub const ENABLE_DEBUG = false;
@@ -280,7 +282,7 @@ pub const Connection = struct {
};
pub fn init(
x509_store: *crypto.X509_STORE,
certificates: Certificates,
config: *const Config,
ip_filter: ?*const IpFilter,
) !Connection {
@@ -289,7 +291,7 @@ pub const Connection = struct {
var self = Connection{ ._easy = easy, .transport = .none };
errdefer self.deinit();
try self.reset(config, x509_store, ip_filter);
try self.reset(config, certificates, ip_filter);
return self;
}
@@ -443,7 +445,7 @@ pub const Connection = struct {
pub fn reset(
self: *Connection,
config: *const Config,
x509_store: *crypto.X509_STORE,
certificates: Certificates,
ip_filter: ?*const IpFilter,
) !void {
libcurl.curl_easy_reset(self._easy);
@@ -487,7 +489,7 @@ pub const Connection = struct {
}
}).wrap);
// Pass our store to CURLOPT_SSL_CTX_FUNCTION.
try libcurl.curl_easy_setopt(self._easy, .ssl_ctx_data, x509_store);
try libcurl.curl_easy_setopt(self._easy, .ssl_ctx_data, certificates.store);
} else {
try libcurl.curl_easy_setopt(self._easy, .ssl_verify_host, false);
try libcurl.curl_easy_setopt(self._easy, .ssl_verify_peer, false);
src/telemetry/lightpanda.zig
+1 -1
View File
@@ -132,7 +132,7 @@ pub fn send(self: *LightPanda, raw_event: telemetry.Event) !void {
fn run(self: *LightPanda) void {
// The connection is created, owned, and torn down entirely on this thread;
// the network thread never sees it (Transport == .none).
var conn = http.Connection.init(self.network.x509_store, self.network.config, self.network.ip_filter) catch |err| {
var conn = http.Connection.init(self.network.certificates, self.network.config, self.network.ip_filter) catch |err| {
// Essentially OOM — the process is already in trouble. The thread
// handle stays set so send() won't respawn; events drop at the cap.
log.warn(.telemetry, "connection init", .{ .err = err });