diff --git a/AGENTS.md b/AGENTS.md index 53a2a07d2..f5f688947 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -6,6 +6,8 @@ See [CONTRIBUTING.md](CONTRIBUTING.md) for how to open a pull request (CLA, dev Run `make download-v8` once first: it fetches the prebuilt V8 archive into `.lp-cache/`, which `build.zig` picks up automatically. Without it every build compiles V8 from source (10+ minutes). +The C and Rust dependencies are built in ReleaseFast whatever `-Doptimize` is, so debug and release builds share them. Pass `ZIGFLAGS=-Ddebug_deps` to step into a dependency with a debugger. + ```bash make test # Run all tests make test F="server" # Filter by substring diff --git a/Makefile b/Makefile index 5d3eb4369..a1466144c 100644 --- a/Makefile +++ b/Makefile @@ -3,8 +3,6 @@ ZIG := zig BC := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) -# tikv-jemalloc-sys's nested make can't parse inherited "-- F=..." overrides -MAKEOVERRIDES = # option test filter make test F="server" F= diff --git a/build.zig b/build.zig index 42a58eba2..8b8b41ada 100644 --- a/build.zig +++ b/build.zig @@ -72,6 +72,15 @@ pub fn build(b: *Build) !void { .glibc_version = devFastGlibcVersion(b), }) else requested_target; + // Dependencies never follow -Doptimize, and they build for the requested + // target rather than the dev_fast bundled-CRT query, so debug and release + // builds share one set of dependency objects in the cache. + const debug_deps = b.option(bool, "debug_deps", "Build the C and Rust dependencies in Debug instead of ReleaseFast") orelse false; + const deps: Deps = .{ + .target = requested_target, + .optimize = if (debug_deps) .Debug else .ReleaseFast, + }; + // Without an explicit -Dprebuilt_v8_path, pick up whatever `make // download-v8` cached rather than building V8 from source. const prebuilt_v8_path = prebuilt_v8_path_option orelse if (enable_tsan or enable_asan) null else findPrebuiltV8(b, target, dev_fast); @@ -120,12 +129,12 @@ pub fn build(b: *Build) !void { const v8_archive: ?Build.LazyPath = if (prebuilt_v8_path) |path| .{ .cwd_relative = path } else null; const v8_for_link = if (orderfile != null and v8_archive != null and !shared_v8) markHotSections(b, v8_archive.?) else v8_archive; linkV8(b, lightpanda_module, enable_asan, enable_tsan, v8_for_link, shared_v8); - linkCurl(b, lightpanda_module, enable_tsan, orderfile != null); - linkRust(b, lightpanda_module); + linkCurl(b, lightpanda_module, deps, enable_tsan, orderfile != null); + linkRust(b, lightpanda_module, deps); linkZenai(b, lightpanda_module); linkIsocline(b, lightpanda_module); - linkSqlite(b, lightpanda_module, enable_csan, enable_tsan, orderfile != null); - linkPcre2(b, lightpanda_module, enable_csan, enable_tsan, orderfile != null); + linkSqlite(b, lightpanda_module, deps, enable_csan, enable_tsan, orderfile != null); + linkPcre2(b, lightpanda_module, deps, enable_csan, enable_tsan, orderfile != null); // Check compilation const check = b.step("check", "Check if lightpanda compiles"); @@ -211,6 +220,11 @@ pub fn build(b: *Build) !void { } } +const Deps = struct { + target: Build.ResolvedTarget, + optimize: std.builtin.OptimizeMode, +}; + const ExeConfig = struct { check: *Build.Step, lightpanda_module: *Build.Module, @@ -377,14 +391,14 @@ fn linkV8( mod.addImport("v8", dep.module("v8")); } -fn linkRust(b: *Build, mod: *Build.Module) void { - const is_debug = mod.optimize.? == .Debug; +fn linkRust(b: *Build, mod: *Build.Module, deps: Deps) void { + // Cargo's "dev" profile writes to target/debug. + const profile, const out_subdir = if (deps.optimize == .Debug) .{ "dev", "debug" } else .{ "release", "release" }; // One cargo workspace, one staticlib (src/rust/Cargo.toml explains why). const exec_cargo = b.addSystemCommand(&.{ "cargo", "build", - "--profile", if (is_debug) "dev" else "release", - "--features", if (is_debug) "memstats" else "", + "--profile", profile, "--manifest-path", "src/rust/ffi/Cargo.toml", }); @@ -397,17 +411,13 @@ fn linkRust(b: *Build, mod: *Build.Module) void { // still surfaces the captured output. _ = exec_cargo.captureStdErr(.{}); - // don't let cargo's progress report (sent to stderr) cause Zig's build to - // print a 'failed command: ...' message. (non-zero status still outputs the error) - _ = exec_cargo.captureStdErr(.{}); - // TODO: We can prefer `--artifact-dir` once it become stable. const out_dir = exec_cargo.addPrefixedOutputDirectoryArg("--target-dir=", "rust"); const rust_step = b.step("rust", "Build the Rust staticlib (requires cargo)"); rust_step.dependOn(&exec_cargo.step); - const obj = out_dir.path(b, if (is_debug) "debug" else "release").path(b, "liblightpanda_ffi.a"); + const obj = out_dir.path(b, out_subdir).path(b, "liblightpanda_ffi.a"); mod.addObjectFile(obj); } @@ -429,10 +439,10 @@ fn addDirInputs(b: *Build, run: *Build.Step.Run, root: []const u8, skip_dir: []c } } -fn linkSqlite(b: *Build, mod: *Build.Module, enable_csan: ?std.zig.SanitizeC, is_tsan: bool, section: bool) void { +fn linkSqlite(b: *Build, mod: *Build.Module, deps: Deps, enable_csan: ?std.zig.SanitizeC, is_tsan: bool, section: bool) void { const dep = b.dependency("sqlite3", .{ - .target = mod.resolved_target.?, - .optimize = mod.optimize.?, + .target = deps.target, + .optimize = deps.optimize, }); const lib = sectionize(dep.artifact("sqlite3"), section); @@ -487,10 +497,10 @@ fn linkSqlite(b: *Build, mod: *Build.Module, enable_csan: ?std.zig.SanitizeC, is mod.addImport("sqlite3", translate_c.createModule()); } -fn linkPcre2(b: *Build, mod: *Build.Module, enable_csan: ?std.zig.SanitizeC, is_tsan: bool, section: bool) void { +fn linkPcre2(b: *Build, mod: *Build.Module, deps: Deps, enable_csan: ?std.zig.SanitizeC, is_tsan: bool, section: bool) void { const dep = b.dependency("pcre2", .{ - .target = mod.resolved_target.?, - .optimize = mod.optimize.?, + .target = deps.target, + .optimize = deps.optimize, .linkage = .static, }); @@ -508,34 +518,32 @@ fn linkPcre2(b: *Build, mod: *Build.Module, enable_csan: ?std.zig.SanitizeC, is_ mod.addImport("pcre2", translate_c.createModule()); } -fn linkCurl(b: *Build, mod: *Build.Module, is_tsan: bool, section: bool) void { - const target = mod.resolved_target.?; - - const curl = buildCurl(b, target, mod.optimize.?, is_tsan, section); +fn linkCurl(b: *Build, mod: *Build.Module, deps: Deps, is_tsan: bool, section: bool) void { + const curl = buildCurl(b, deps.target, deps.optimize, is_tsan, section); mod.linkLibrary(curl); const dep = b.dependency("curl", .{}); const translate_c = b.addTranslateC(.{ .root_source_file = dep.path("include/curl/curl.h"), - .target = target, + .target = mod.resolved_target.?, .optimize = mod.optimize.?, }); translate_c.addIncludePath(dep.path("include")); mod.addImport("curl", translate_c.createModule()); - const zlib = buildZlib(b, target, mod.optimize.?, is_tsan, section); + const zlib = buildZlib(b, deps.target, deps.optimize, is_tsan, section); curl.root_module.linkLibrary(zlib); - const brotli = buildBrotli(b, target, mod.optimize.?, is_tsan, section); + const brotli = buildBrotli(b, deps.target, deps.optimize, is_tsan, section); for (brotli) |lib| curl.root_module.linkLibrary(lib); - const nghttp2 = buildNghttp2(b, target, mod.optimize.?, is_tsan, section); + const nghttp2 = buildNghttp2(b, deps.target, deps.optimize, is_tsan, section); curl.root_module.linkLibrary(nghttp2); - const boringssl = buildBoringSsl(b, target, mod.optimize.?, section); + const boringssl = buildBoringSsl(b, deps.target, deps.optimize, section); for (boringssl) |lib| curl.root_module.linkLibrary(lib); - if (target.result.os.tag == .macos) { + if (deps.target.result.os.tag == .macos) { // needed for proxying on mac const framework_path = if (b.sysroot) |sysroot| b.pathJoin(&.{ sysroot, "System/Library/Frameworks" }) diff --git a/src/browser/parser/html5ever.zig b/src/browser/parser/html5ever.zig index 3b7cf3223..4d1a29776 100644 --- a/src/browser/parser/html5ever.zig +++ b/src/browser/parser/html5ever.zig @@ -96,13 +96,6 @@ pub extern "c" fn html5ever_parse_fragment( pub extern "c" fn html5ever_attribute_iterator_next(ctx: *anyopaque) Nullable(Attribute); pub extern "c" fn html5ever_attribute_iterator_count(ctx: *anyopaque) usize; -pub extern "c" fn html5ever_get_memory_usage() MemoryUsage; - -const MemoryUsage = extern struct { - resident: usize, - allocated: usize, -}; - // Streaming parser API pub extern "c" fn html5ever_streaming_parser_create( doc: *anyopaque, diff --git a/src/rust/Cargo.lock b/src/rust/Cargo.lock index 455440f4f..4305378bf 100644 --- a/src/rust/Cargo.lock +++ b/src/rust/Cargo.lock @@ -52,16 +52,6 @@ dependencies = [ "syn 3.0.3", ] -[[package]] -name = "cc" -version = "1.2.39" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1354349954c6fc9cb0deab020f27f783cf0b604e8bb754dc4658ecf0d29c35f" -dependencies = [ - "find-msvc-tools", - "shlex", -] - [[package]] name = "cfg-if" version = "1.0.0" @@ -112,12 +102,6 @@ dependencies = [ "simd-adler32", ] -[[package]] -name = "find-msvc-tools" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ced73b1dacfc750a6db6c0a0c3a3853c8b41997e2e2c563dc90804ae6867959" - [[package]] name = "flate2" version = "1.1.9" @@ -358,8 +342,6 @@ version = "0.1.0" dependencies = [ "lightpanda-html5ever", "lightpanda-render", - "tikv-jemalloc-ctl", - "tikv-jemallocator", ] [[package]] @@ -513,12 +495,6 @@ dependencies = [ "icu_properties", ] -[[package]] -name = "paste" -version = "1.0.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" - [[package]] name = "percent-encoding" version = "2.3.2" @@ -668,12 +644,6 @@ dependencies = [ "syn 3.0.3", ] -[[package]] -name = "shlex" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" - [[package]] name = "simd-adler32" version = "0.3.10" @@ -782,37 +752,6 @@ dependencies = [ "utf-8", ] -[[package]] -name = "tikv-jemalloc-ctl" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "661f1f6a57b3a36dc9174a2c10f19513b4866816e13425d3e418b11cc37bc24c" -dependencies = [ - "libc", - "paste", - "tikv-jemalloc-sys", -] - -[[package]] -name = "tikv-jemalloc-sys" -version = "0.6.1+5.3.0-1-ge13ca993e8ccb9ba9847cc330696e02839f328f7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd8aa5b2ab86a2cefa406d889139c162cbb230092f7d1d7cbc1716405d852a3b" -dependencies = [ - "cc", - "libc", -] - -[[package]] -name = "tikv-jemallocator" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0359b4327f954e0567e69fb191cf1436617748813819c94b8cd4a431422d053a" -dependencies = [ - "libc", - "tikv-jemalloc-sys", -] - [[package]] name = "tiny-skia" version = "0.12.0" diff --git a/src/rust/ffi/Cargo.toml b/src/rust/ffi/Cargo.toml index 9765ad866..3030ac768 100644 --- a/src/rust/ffi/Cargo.toml +++ b/src/rust/ffi/Cargo.toml @@ -11,8 +11,3 @@ crate-type = ["staticlib"] [dependencies] lightpanda-html5ever = { path = "../html5ever" } lightpanda-render = { path = "../render" } -tikv-jemallocator = { version = "0.6.1", features = ["stats"], optional = true } -tikv-jemalloc-ctl = { version = "0.6.1", features = ["stats"], optional = true } - -[features] -memstats = ["dep:tikv-jemallocator", "dep:tikv-jemalloc-ctl"] diff --git a/src/rust/ffi/lib.rs b/src/rust/ffi/lib.rs index f0b051be7..42d88da87 100644 --- a/src/rust/ffi/lib.rs +++ b/src/rust/ffi/lib.rs @@ -18,33 +18,7 @@ //! The one staticlib Zig links. Domain crates define their own `extern "C"` //! entry points; `extern crate` is what pulls an otherwise-unreferenced crate -//! into the archive (edition 2018+ drops unused `--extern` deps). Process-wide -//! concerns (the allocator) live here, not in a domain crate. +//! into the archive (edition 2018+ drops unused `--extern` deps). extern crate lightpanda_html5ever; extern crate lightpanda_render; - -#[cfg(feature = "memstats")] -#[global_allocator] -static GLOBAL: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; - -#[cfg(feature = "memstats")] -#[repr(C)] -pub struct Memory { - pub resident: usize, - pub allocated: usize, -} - -#[cfg(feature = "memstats")] -#[no_mangle] -pub extern "C" fn html5ever_get_memory_usage() -> Memory { - use tikv_jemalloc_ctl::{epoch, stats}; - - // many statistics are cached and only updated when the epoch is advanced. - let _ = epoch::advance(); - - Memory { - resident: stats::resident::read().unwrap_or(0), - allocated: stats::allocated::read().unwrap_or(0), - } -}