ops: Add prometheus metrics

Adds a /metrics endpoint in serve mode. Can be disabled via --disable-metrics.

In fetch, dumps the metrics if --metrics is specified (defaults to false).

None of the metrics being collected are on a hot path, so they're just always
collected using atomic operations (i.e. no `if (!enabled) return;`). Because
the operations are cheap and infrequent enough not to matter.

The one place I want to add more metrics is in HttpClient (bytes, status,
counts, ...) but after the pending HttpClient-related PR is merged.

./lightpanda fetch --log-level fatal --metrics "https://lightpanda.io/"

```
build_info{version="1.0.0-dev.7837+70493ce35"} 1
cdp_connections_total 0
cdp_connection_limit_total 0
cdp_active_connections 0
cdp_commands_total 0
cdp_unknown_commands_total 0
js_heap_limits_total 0
script_errors_total 0
arena_hit_total{size="tiny"} 908
arena_hit_total{size="small"} 35
arena_hit_total{size="medium"} 0
arena_hit_total{size="large"} 27
arena_miss_total{size="tiny"} 193
arena_miss_total{size="small"} 31
arena_miss_total{size="medium"} 5
arena_miss_total{size="large"} 23
navigate_total{type="page"} 1
navigate_total{type="iframe"} 0
navigate_total{type="popup"} 0
js_heap_size_bytes_bucket{le="4194304"} 0
js_heap_size_bytes_bucket{le="8388608"} 0
js_heap_size_bytes_bucket{le="16777216"} 1
js_heap_size_bytes_bucket{le="33554432"} 1
js_heap_size_bytes_bucket{le="67108864"} 1
js_heap_size_bytes_bucket{le="134217728"} 1
js_heap_size_bytes_bucket{le="268435456"} 1
js_heap_size_bytes_bucket{le="536870912"} 1
js_heap_size_bytes_bucket{le="+Inf"} 1
js_heap_size_bytes_sum 11223040
js_heap_size_bytes_count 1
```
This commit is contained in:
Karl Seguin
2026-07-09 15:59:48 +08:00
parent 56c5701b7b
commit 4d81e48e44
14 changed files with 377 additions and 24 deletions

View File

@@ -111,25 +111,27 @@ pub fn deinit(self: *ArenaPool) void {
// - Pass a BucketSize (.tiny, .small, .medium, .large) for explicit bucket selection
// - Pass a usize for automatic bucket selection based on expected size
pub fn acquire(self: *ArenaPool, size_or_bucket: anytype, debug: []const u8) !Allocator {
const bucket = blk: {
const bucket_size: BucketSize = blk: {
const T = @TypeOf(size_or_bucket);
if (T == BucketSize or T == @TypeOf(.enum_literal)) {
break :blk switch (@as(BucketSize, size_or_bucket)) {
.tiny => &self.tiny,
.small => &self.small,
.medium => &self.medium,
.large => &self.large,
};
break :blk @as(BucketSize, size_or_bucket);
}
if (T == usize or T == comptime_int) {
if (size_or_bucket <= self.tiny.retain_bytes) break :blk &self.tiny;
if (size_or_bucket <= self.small.retain_bytes) break :blk &self.small;
if (size_or_bucket <= self.medium.retain_bytes) break :blk &self.medium;
break :blk &self.large;
if (size_or_bucket <= self.tiny.retain_bytes) break :blk .tiny;
if (size_or_bucket <= self.small.retain_bytes) break :blk .small;
if (size_or_bucket <= self.medium.retain_bytes) break :blk .medium;
break :blk .large;
}
@compileError("acquire expects BucketSize or usize, got " ++ @typeName(T));
};
const bucket = switch (bucket_size) {
.tiny => &self.tiny,
.small => &self.small,
.medium => &self.medium,
.large => &self.large,
};
self.mutex.lock();
defer self.mutex.unlock();
@@ -144,9 +146,12 @@ pub fn acquire(self: *ArenaPool, size_or_bucket: anytype, debug: []const u8) !Al
}
gop.value_ptr.* += 1;
}
lp.metrics.arena_hit.incr(bucket_size);
return entry.arena.allocator();
}
lp.metrics.arena_miss.incr(bucket_size);
const entry = try self.entry_pool.create();
entry.* = .{
.next = null,

View File

@@ -203,6 +203,7 @@ const Commands = cli.Builder(.{
.{ .name = "cdp_max_message_size", .type = u32, .default = 1024 * 1024 },
// Don't widen this without growing the reader buffer in the HTTP path.
.{ .name = "cdp_max_http_message_size", .type = u14, .default = 4096 },
.{ .name = "disable_metrics", .type = bool },
},
.shared_options = CommonOptions,
},
@@ -235,6 +236,7 @@ const Commands = cli.Builder(.{
},
.{ .name = "terminate_ms", .type = ?u32 },
.{ .name = "json", .type = bool },
.{ .name = "metrics", .type = bool },
},
.shared_options = CommonOptions,
},
@@ -586,6 +588,20 @@ pub fn cdpMaxMessageSize(self: *const Config) u32 {
};
}
pub fn metricsEndpointEnabled(self: *const Config) bool {
return switch (self.mode) {
.serve => |opts| !opts.disable_metrics,
else => unreachable,
};
}
pub fn dumpMetricsOnExit(self: *const Config) bool {
return switch (self.mode) {
.fetch => |opts| opts.metrics,
else => false,
};
}
pub fn cdpMaxHTTPMessageSize(self: *const Config) u14 {
return switch (self.mode) {
.serve => |opts| opts.cdp_max_http_message_size,

260
src/Metrics.zig Normal file
View File

@@ -0,0 +1,260 @@
// Copyright (C) 2023-2026 Lightpanda (Selecy SAS)
//
// Francis Bouvier <francis@lightpanda.io>
// Pierre Tachoire <pierre@lightpanda.io>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
const std = @import("std");
const lp = @import("lightpanda");
const Metrics = @This();
cdp_connections: Counter = .{},
cdp_connection_limit: Counter = .{},
cdp_active_connections: Gauge = .{},
cdp_commands: Counter = .{},
cdp_unknown_commands: Counter = .{},
js_heap_limits: Counter = .{},
script_errors: Counter = .{},
arena_hit: CounterEnum("size", @import("ArenaPool.zig").BucketSize) = .{},
arena_miss: CounterEnum("size", @import("ArenaPool.zig").BucketSize) = .{},
navigate: CounterEnum("type", @import("telemetry/telemetry.zig").Event.Navigate.Context) = .{},
js_heap_size_bytes: Histogram(&.{
4 * 1024 * 1024,
8 * 1024 * 1024,
16 * 1024 * 1024,
32 * 1024 * 1024,
64 * 1024 * 1024,
128 * 1024 * 1024,
256 * 1024 * 1024,
512 * 1024 * 1024,
}) = .{},
// Emitted as each metric's "# HELP" line. A field without an entry is a
// compile error.
const help = .{
.cdp_connections = "CDP websocket connections accepted",
.cdp_connection_limit = "Connections rejected because --cdp-max-connections was reached",
.cdp_active_connections = "Currently connected CDP clients",
.cdp_commands = "CDP commands dispatched",
.cdp_unknown_commands = "CDP commands rejected for an unknown domain or method",
.js_heap_limits = "Pages terminated for reaching the V8 heap limit",
.script_errors = "Scripts that failed to evaluate, e.g. an uncaught top-level exception",
.arena_hit = "Arena pool acquisitions served from the free list",
.arena_miss = "Arena pool acquisitions that had to allocate a new arena",
.navigate = "Navigations by initiating frame type",
.js_heap_size_bytes = "V8 heap physical size, sampled when a page is closed",
};
pub fn write(self: *const Metrics, writer: *std.Io.Writer) void {
self._write(writer) catch |err| {
lp.log.err(.app, "metrics write", .{ .err = err });
};
}
fn _write(self: *const Metrics, writer: *std.Io.Writer) !void {
try writer.print(
"# HELP build_info Lightpanda build information\n" ++
"# TYPE build_info gauge\nbuild_info{{version=\"{s}\"}} 1\n",
.{lp.build_config.version},
);
inline for (@typeInfo(Metrics).@"struct".fields) |f| {
try @field(self, f.name).write(f.name, @field(help, f.name), writer);
}
}
const Counter = struct {
count: usize = 0,
pub fn incr(self: *Counter) void {
self.incrBy(1);
}
pub fn incrBy(self: *Counter, c: usize) void {
_ = @atomicRmw(usize, &self.count, .Add, c, .monotonic);
}
fn write(self: *const Counter, comptime name: []const u8, comptime help_text: []const u8, writer: *std.Io.Writer) !void {
try writer.writeAll("# HELP " ++ name ++ "_total " ++ help_text ++ "\n" ++ "# TYPE " ++ name ++ "_total counter\n");
try writer.print(name ++ "_total {d}\n", .{self.get()});
}
fn get(self: *const Counter) usize {
return @atomicLoad(usize, &self.count, .monotonic);
}
};
const Gauge = struct {
value: isize = 0,
pub fn incr(self: *Gauge) void {
_ = @atomicRmw(isize, &self.value, .Add, 1, .monotonic);
}
pub fn decr(self: *Gauge) void {
_ = @atomicRmw(isize, &self.value, .Sub, 1, .monotonic);
}
fn write(self: *const Gauge, comptime name: []const u8, comptime help_text: []const u8, writer: *std.Io.Writer) !void {
try writer.writeAll("# HELP " ++ name ++ " " ++ help_text ++ "\n" ++ "# TYPE " ++ name ++ " gauge\n");
try writer.print(name ++ " {d}\n", .{@atomicLoad(isize, &self.value, .monotonic)});
}
};
fn CounterEnum(comptime label: []const u8, comptime T: type) type {
return struct {
counts: std.enums.EnumArray(T, Counter) = .initFill(.{}),
pub const Tag = T;
pub const label_name = label;
const Self = @This();
pub fn incr(self: *Self, tag: T) void {
self.incrBy(tag, 1);
}
pub fn incrBy(self: *Self, tag: T, c: usize) void {
self.counts.getPtr(tag).incrBy(c);
}
fn write(self: *const Self, comptime name: []const u8, comptime help_text: []const u8, writer: *std.Io.Writer) !void {
try writer.writeAll("# HELP " ++ name ++ "_total " ++ help_text ++ "\n" ++ "# TYPE " ++ name ++ "_total counter\n");
inline for (comptime std.enums.values(Tag)) |tag| {
try writer.print(name ++ "_total{{" ++ label ++ "=\"" ++ @tagName(tag) ++ "\"}} {d}\n", .{self.counts.getPtrConst(tag).get()});
}
}
};
}
fn Histogram(comptime upper_bounds: []const usize) type {
comptime {
std.debug.assert(upper_bounds.len > 0);
for (upper_bounds[0 .. upper_bounds.len - 1], upper_bounds[1..]) |a, b| {
std.debug.assert(a < b);
}
}
return struct {
sum: usize = 0,
count: usize = 0,
buckets: [upper_bounds.len]usize = @splat(0),
const Self = @This();
pub fn observe(self: *Self, value: usize) void {
_ = @atomicRmw(usize, &self.count, .Add, 1, .monotonic);
_ = @atomicRmw(usize, &self.sum, .Add, value, .monotonic);
inline for (upper_bounds, 0..) |upper, i| {
if (value <= upper) {
_ = @atomicRmw(usize, &self.buckets[i], .Add, 1, .monotonic);
return;
}
}
// falls in the implicit +Inf bucket, which is derived from count
}
fn write(self: *const Self, comptime name: []const u8, comptime help_text: []const u8, writer: *std.Io.Writer) !void {
try writer.writeAll("# HELP " ++ name ++ " " ++ help_text ++ "\n" ++ "# TYPE " ++ name ++ " histogram\n");
// le buckets are cumulative
var sum: usize = 0;
inline for (upper_bounds, 0..) |upper, i| {
sum += @atomicLoad(usize, &self.buckets[i], .monotonic);
try writer.print(name ++ "_bucket{{le=\"" ++ std.fmt.comptimePrint("{d}", .{upper}) ++ "\"}} {d}\n", .{sum});
}
const count = @atomicLoad(usize, &self.count, .monotonic);
try writer.print(
name ++ "_bucket{{le=\"+Inf\"}} {d}\n" ++ name ++ "_sum {d}\n" ++ name ++ "_count {d}\n",
.{ count, @atomicLoad(usize, &self.sum, .monotonic), count },
);
}
};
}
const testing = @import("testing.zig");
test "Metrics: Counter" {
var w = std.Io.Writer.Allocating.init(testing.allocator);
defer w.deinit();
var c = Counter{};
c.incr();
c.incrBy(3);
try c.write("stat", "counts stats", &w.writer);
try testing.expectEqual(
\\# HELP stat_total counts stats
\\# TYPE stat_total counter
\\stat_total 4
\\
, w.written());
}
test "Metrics: Gauge" {
var w = std.Io.Writer.Allocating.init(testing.allocator);
defer w.deinit();
var g = Gauge{};
g.incr();
g.incr();
g.decr();
try g.write("stat", "counts stats", &w.writer);
try testing.expectEqual(
\\# HELP stat counts stats
\\# TYPE stat gauge
\\stat 1
\\
, w.written());
}
test "Metrics: CounterEnum" {
var w = std.Io.Writer.Allocating.init(testing.allocator);
defer w.deinit();
var c = CounterEnum("color", enum { red, blue }){};
c.incr(.blue);
c.incrBy(.blue, 2);
try c.write("stat", "counts stats", &w.writer);
try testing.expectEqual(
\\# HELP stat_total counts stats
\\# TYPE stat_total counter
\\stat_total{color="red"} 0
\\stat_total{color="blue"} 3
\\
, w.written());
}
test "Metrics: Histogram" {
var w = std.Io.Writer.Allocating.init(testing.allocator);
defer w.deinit();
var h = Histogram(&.{ 10, 100 }){};
h.observe(5);
h.observe(10); // le is inclusive
h.observe(50);
h.observe(200); // +Inf only
try h.write("stat", "counts stats", &w.writer);
try testing.expectEqual(
\\# HELP stat counts stats
\\# TYPE stat histogram
\\stat_bucket{le="10"} 2
\\stat_bucket{le="100"} 3
\\stat_bucket{le="+Inf"} 4
\\stat_sum 265
\\stat_count 4
\\
, w.written());
}

View File

@@ -156,6 +156,7 @@ fn spawnWorker(self: *Server, socket: posix.socket_t) !void {
while (current < self.max_connections) {
current = self.active_threads.cmpxchgWeak(current, current + 1, .monotonic, .monotonic) orelse break;
} else {
lp.metrics.cdp_connection_limit.incr();
return error.MaxThreadsReached;
}
errdefer _ = self.active_threads.fetchSub(1, .monotonic);
@@ -218,6 +219,12 @@ fn handleConnection(self: *Server, socket: posix.socket_t) void {
return;
}
// only count websocket (i.e. CDP) connections, not HTTP requests like
// /json/version probes or /metrics scrapes
lp.metrics.cdp_connections.incr();
lp.metrics.cdp_active_connections.incr();
defer lp.metrics.cdp_active_connections.decr();
{
// Transition from .handshake state to .live
// Lock needed even though the main thread hasn't seen this yet because
@@ -551,6 +558,18 @@ test "server: get /json/version" {
}
}
test "server: get /metrics" {
var c = try createTestClient();
defer c.deinit();
const res = try c.httpRequest("GET /metrics HTTP/1.1\r\n\r\n");
std.debug.print("1\n", .{});
try testing.expect(std.mem.startsWith(u8, res, "HTTP/1.1 200 OK\r\n"));
try testing.expect(std.mem.indexOf(u8, res, "Content-Type: text/plain; version=0.0.4") != null);
try testing.expect(std.mem.indexOf(u8, res, "build_info{version=") != null);
try testing.expect(std.mem.indexOf(u8, res, "# TYPE cdp_connections_total counter") != null);
}
fn assertHTTPError(
comptime expected_status: u16,
comptime expected_body: []const u8,
@@ -648,7 +667,7 @@ fn createTestClient() !TestClient {
const TestClient = struct {
stream: std.net.Stream,
buf: [1024]u8 = undefined,
buf: [4096]u8 = undefined,
reader: WS.Reader(false),
const WS = @import("network/WS.zig");
@@ -664,10 +683,11 @@ const TestClient = struct {
var pos: usize = 0;
var total_length: ?usize = null;
while (true) {
pos += try self.stream.read(self.buf[pos..]);
if (pos == 0) {
return error.NoMoreData;
const n = try self.stream.read(self.buf[pos..]);
if (n == 0) {
return if (pos == self.buf.len) error.MessageTooLarge else error.NoMoreData;
}
pos += n;
const response = self.buf[0..pos];
if (total_length == null) {
const header_end = std.mem.indexOf(u8, response, "\r\n\r\n") orelse continue;

View File

@@ -799,14 +799,17 @@ pub fn navigate(self: *Frame, request_url: [:0]const u8, opts: NavigateOpts) !vo
}
fn recordNavigateTelemetry(self: *Frame, tls: bool) void {
const TE = @import("../telemetry/telemetry.zig").Event;
const context: TE.Navigate.Context = if (self.parent != null)
.iframe
else if (self.window._opener != null)
.popup
else
.page;
lp.metrics.navigate.incr(context);
self._session.browser.app.telemetry.record(.{ .navigate = .{
.tls = tls,
.context = if (self.parent != null)
.iframe
else if (self.window._opener != null)
.popup
else
.page,
.context = context,
} });
}

View File

@@ -17,6 +17,7 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
const std = @import("std");
const lp = @import("lightpanda");
const builtin = @import("builtin");
const js = @import("js/js.zig");
@@ -26,6 +27,7 @@ const Session = @import("Session.zig");
const Factory = @import("Factory.zig");
const Viewport = @import("Viewport.zig");
const v8 = js.v8;
const Allocator = std.mem.Allocator;
const IS_DEBUG = builtin.mode == .Debug;
@@ -165,6 +167,7 @@ pub fn deinit(self: *Page) void {
self.frame.deinit();
const session = self.session;
lp.metrics.js_heap_size_bytes.observe(session.browser.env.isolate.getHeapStatistics().total_physical_size);
defer session.browser.env.memoryPressureNotification(.moderate);
self.identity.deinit();

View File

@@ -935,6 +935,7 @@ pub const Script = struct {
}
const caught = try_catch.caughtOrError(frame.local_arena, error.Unknown);
lp.metrics.script_errors.incr();
log.warn(.js, "eval script", .{
.url = url,
.caught = caught,

View File

@@ -578,6 +578,7 @@ pub fn protectHeapLimit(self: *Env) void {
// V8 expects.
fn nearHeapLimit(data: ?*anyopaque, current_limit: usize, initial_limit: usize) callconv(.c) usize {
const self: *Env = @ptrCast(@alignCast(data.?));
lp.metrics.js_heap_limits.incr();
log.err(.app, "JS heap limit reached", .{
.initial_limit = initial_limit,
.current_limit = current_limit,

View File

@@ -299,6 +299,8 @@ pub fn dispatch(self: *CDP, arena: Allocator, sender: Command.Sender, str: []con
// keeping `str` and the backing storage for `input`'s string slices
// alive for the duration of the call.
fn dispatchParsed(self: *CDP, arena: Allocator, sender: Command.Sender, str: []const u8, input: InputMessage) !void {
lp.metrics.cdp_commands.incr();
var command = Command{
.input = .{
.json = str,
@@ -329,6 +331,10 @@ fn dispatchParsed(self: *CDP, arena: Allocator, sender: Command.Sender, str: []c
};
} else {
dispatchCommand(&command, input.method) catch |err| {
switch (err) {
error.UnknownDomain, error.UnknownMethod => lp.metrics.cdp_unknown_commands.incr(),
else => {},
}
command.sendError(-31998, @errorName(err), .{}) catch return err;
};
}

View File

@@ -43,6 +43,7 @@ socket_flags: usize,
state: State = .handshaking,
reader: WS.Reader(true),
send_arena: ArenaAllocator,
metrics_enabled: bool,
max_http_message_size: usize,
json_version_response: []const u8,
@@ -67,6 +68,7 @@ pub fn init(
.socket = socket,
.arena_pool = &app.arena_pool,
.socket_flags = socket_flags,
.metrics_enabled = config.metricsEndpointEnabled(),
.max_http_message_size = config.cdpMaxHTTPMessageSize(),
.reader = try .init(allocator, config.cdpMaxMessageSize()),
.send_arena = ArenaAllocator.init(allocator),
@@ -85,7 +87,7 @@ pub fn send(self: *Connection, data: []const u8) !void {
defer _ = self.send_arena.reset(.{ .retain_with_limit = 1024 * 32 });
defer if (changed_to_blocking) {
// We had to change our socket to blocking me to get our write out
// We had to change our socket to blocking mode to get our write out
// We need to change it back to non-blocking.
_ = posix.fcntl(self.socket, posix.F.SETFL, self.socket_flags) catch |err| {
log.err(.app, "ws restore nonblocking", .{ .err = err });
@@ -298,9 +300,30 @@ fn handleHttpRequest(self: *Connection, request: []u8) !HttpResult {
return .close;
}
if (self.metrics_enabled and std.mem.eql(u8, url, "/metrics")) {
try self.sendMetrics();
self.shutdown();
return .close;
}
return error.NotFound;
}
fn sendMetrics(self: *Connection) !void {
const allocator = self.send_arena.allocator();
var aw = try std.Io.Writer.Allocating.initCapacity(allocator, 4096);
lp.metrics.write(&aw.writer);
const body = aw.written();
const response = try std.fmt.allocPrint(allocator, "HTTP/1.1 200 OK\r\n" ++
"Content-Length: {d}\r\n" ++
"Connection: Close\r\n" ++
"Content-Type: text/plain; version=0.0.4; charset=utf-8\r\n\r\n" ++
"{s}", .{ body.len, body });
try self.send(response);
}
const empty_json_list_response =
"HTTP/1.1 200 OK\r\n" ++
"Content-Length: 2\r\n" ++

View File

@@ -38,6 +38,9 @@
\\ --cookie <PATH>
\\ Path to a JSON file to load cookies from (read-only).
\\ Defaults to no cookie loading.
\\ --disable-metrics
\\ Disables the /metrics endpoint (Prometheus text format).
\\ Defaults to false (the endpoint is exposed).
\\ --host <HOST>
\\ Host of the CDP server.
\\ Defaults to "127.0.0.1".
@@ -78,6 +81,10 @@
\\ prints one object; multiple URLs print {{"results": [ ... ]}} with
\\ one object per URL. When used with --dump <MODE> the dumped
\\ content is wrapped within each object.
\\ --metrics
\\ Write metrics (Prometheus text format) to stdout on exit, after
\\ any --dump output.
\\ Defaults to false.
\\ --strip-mode <STRIP>
\\ Comma-separated list of tag groups to remove from dump.
\\ Defaults to no-strip.

View File

@@ -58,6 +58,8 @@ pub const core_dump = @import("core_dump.zig");
pub const Updater = @import("Updater.zig");
pub var metrics = @import("Metrics.zig"){};
pub const FetchOpts = struct {
wait_ms: u32 = 5000,
wait_until: ?Config.WaitUntil = null,

View File

@@ -110,6 +110,12 @@ fn run(allocator: Allocator, main_arena: Allocator) !void {
app.telemetry.record(.{ .run = {} });
defer if (app.config.dumpMetricsOnExit()) {
var stdout = std.fs.File.stdout();
var writer = stdout.writer(&.{});
lp.metrics.write(&writer.interface);
};
switch (args.mode) {
.serve => |opts| {
log.debug(.app, "startup", .{ .mode = "serve", .snapshot = app.snapshot.fromEmbedded() });

View File

@@ -109,11 +109,11 @@ pub const Event = union(enum) {
buffer_overflow: BufferOverflow,
llm: LLM,
const Navigate = struct {
pub const Navigate = struct {
tls: bool,
context: Context,
const Context = enum { page, iframe, popup };
pub const Context = enum { page, iframe, popup };
};
const BufferOverflow = struct {