From e164bdbf36f60bcc80fddf45fc0f74480f264556 Mon Sep 17 00:00:00 2001 From: Zoltan Kochan Date: Tue, 12 May 2026 14:18:06 +0200 Subject: [PATCH] feat: foundation for the side-effects cache (#397 item #10, part 1) (#422) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Foundation for porting pnpm's side-effects cache (item #10 of #397). This PR lands the three pieces that the actual cache read/write paths depend on: - **New `pacquet-graph-hasher` crate** porting pnpm's `@pnpm/crypto.object-hasher` (`hashObject` / `hashObjectWithoutSorting`) plus `@pnpm/deps.graph-hasher` (`calcDepState` / `calcDepGraphHash`) and `ENGINE_NAME`. Byte-for-byte parity with the JS [`object-hash@3.0.0`](https://github.com/puleos/object-hash/blob/v3.0.0/index.js) bytestream format is load-bearing — the cache key is persisted on disk and shared with pnpm. The headline parity test pins `hashObject({b:1,a:2}) == "48AVoXIXcTKcnHt8qVKp5vNw4gyOB5VfztHwtYBRcAQ="` against the upstream fixture in [`crypto/object-hasher/test/index.ts:6`](https://github.com/pnpm/pnpm/blob/b4f8f47ac2/crypto/object-hasher/test/index.ts#L6). - **`VerifyResult.side_effects_maps`** in `pacquet-store-dir`. The verify path (`check_pkg_files_integrity` and `build_file_maps_from_index`) used to drop `PackageFilesIndex.side_effects` after extraction. It now applies the `added`/`deleted` overlay per cache key and surfaces a `HashMap` — the same shape pnpm uses for its `PackageFilesResponse.sideEffectsMaps`. Mirrors [`applySideEffectsDiffWithMaps`](https://github.com/pnpm/pnpm/blob/b4f8f47ac2/store/create-cafs-store/src/index.ts#L103-L121). - **`Config.side_effects_cache`** config knob (default `true`, matching pnpm). Surfaced on the wire **only** — no consumer yet. Wires up cleanly once the build-phase gate lands in #421, with no config migration needed for downstream callers. ## Scoped narrowly This is **only the foundation**. There is no `is_built` field, no `BuildModules` skip gate, and no end-to-end test of the rebuild-skip behavior. Those land in a separate follow-up tracked in #421, which also covers the WRITE path (pacquet seeding the cache itself). The natural slice for review: graph-hasher unblocks both READ and WRITE, the verify-path surfacing is the only intrusive cross-crate change, and the config knob is forward-looking. ### Hashing scope, explicitly The `pacquet-graph-hasher` port implements only the type arms pacquet actually feeds into `hashObject` for the cache-key path: strings, objects, numbers, booleans, null, and arrays in their ordered form. `Set` / `Map` / `Date` / `Buffer` / `Array` unordered-permutation arms from [upstream `object-hash`](https://github.com/puleos/object-hash/blob/v3.0.0/index.js#L257-L389) are unimplemented — no caller in pacquet's tree feeds them in today, and the test fixtures upstream uses (`hashObject({ b: new Set([…]), a: [...] })`) hash deterministically across orderings only because they're testing the library, not exercising a real call path. Adding them can land alongside a future caller that needs them. --- pacquet/Cargo.lock | 10 + pacquet/Cargo.toml | 1 + pacquet/crates/config/src/lib.rs | 18 + pacquet/crates/config/src/workspace_yaml.rs | 5 +- .../crates/config/src/workspace_yaml/tests.rs | 17 + pacquet/crates/graph-hasher/Cargo.toml | 19 ++ pacquet/crates/graph-hasher/src/dep_state.rs | 321 ++++++++++++++++++ .../crates/graph-hasher/src/engine_name.rs | 92 +++++ pacquet/crates/graph-hasher/src/lib.rs | 32 ++ .../crates/graph-hasher/src/object_hasher.rs | 144 ++++++++ pacquet/crates/graph-hasher/src/tests.rs | 120 +++++++ .../install_package_from_registry/tests.rs | 1 + .../src/check_pkg_files_integrity.rs | 90 ++++- .../src/check_pkg_files_integrity/tests.rs | 175 +++++++++- 14 files changed, 1036 insertions(+), 9 deletions(-) create mode 100644 pacquet/crates/graph-hasher/Cargo.toml create mode 100644 pacquet/crates/graph-hasher/src/dep_state.rs create mode 100644 pacquet/crates/graph-hasher/src/engine_name.rs create mode 100644 pacquet/crates/graph-hasher/src/lib.rs create mode 100644 pacquet/crates/graph-hasher/src/object_hasher.rs create mode 100644 pacquet/crates/graph-hasher/src/tests.rs diff --git a/pacquet/Cargo.lock b/pacquet/Cargo.lock index d3d4d3f439..d0c45b2f6c 100644 --- a/pacquet/Cargo.lock +++ b/pacquet/Cargo.lock @@ -1942,6 +1942,16 @@ dependencies = [ "tempfile", ] +[[package]] +name = "pacquet-graph-hasher" +version = "0.0.1" +dependencies = [ + "base64 0.22.1", + "pretty_assertions", + "serde_json", + "sha2", +] + [[package]] name = "pacquet-integrated-benchmark" version = "0.0.0" diff --git a/pacquet/Cargo.toml b/pacquet/Cargo.toml index f06ff9ca39..ebd4dd74c6 100644 --- a/pacquet/Cargo.toml +++ b/pacquet/Cargo.toml @@ -27,6 +27,7 @@ pacquet-network = { path = "crates/network" } pacquet-config = { path = "crates/config" } pacquet-executor = { path = "crates/executor" } pacquet-diagnostics = { path = "crates/diagnostics" } +pacquet-graph-hasher = { path = "crates/graph-hasher" } pacquet-store-dir = { path = "crates/store-dir" } pacquet-reporter = { path = "crates/reporter" } diff --git a/pacquet/crates/config/src/lib.rs b/pacquet/crates/config/src/lib.rs index cb3663426d..58f36e1aff 100644 --- a/pacquet/crates/config/src/lib.rs +++ b/pacquet/crates/config/src/lib.rs @@ -174,6 +174,24 @@ pub struct Config { #[default = true] pub verify_store_integrity: bool, + /// Whether to consult the side-effects cache + /// (`PackageFilesIndex.sideEffects`) when importing a package. + /// Read from `pnpm-workspace.yaml`'s `sideEffectsCache` field + /// (camelCase, optional, defaults `true`). **Not yet acted on:** + /// the build phase doesn't currently gate the rebuild-skip on + /// this flag, and the WRITE path (populating the cache after a + /// postinstall) is also unimplemented. Both follow up + /// separately — tracked in pnpm/pacquet#421. + /// + /// Default `true`, matching pnpm's `side-effects-cache` at + /// [`config/config/src/index.ts`](https://github.com/pnpm/pnpm/blob/b4f8f47ac2/config/config/src/index.ts). + /// Wiring the config-source plumbing through now means + /// downstream callers can set `sideEffectsCache: false` in + /// `pnpm-workspace.yaml` today and have the value take effect + /// as soon as the read-path gate lands. + #[default = true] + pub side_effects_cache: bool, + /// How many times pacquet retries a failed tarball fetch on transient /// errors before giving up. Mirrors pnpm's `fetchRetries` (default /// `2`, matching `config/config/src/index.ts`). The value is the count diff --git a/pacquet/crates/config/src/workspace_yaml.rs b/pacquet/crates/config/src/workspace_yaml.rs index 5188a63f5f..4bb4f36b3f 100644 --- a/pacquet/crates/config/src/workspace_yaml.rs +++ b/pacquet/crates/config/src/workspace_yaml.rs @@ -50,6 +50,7 @@ pub struct WorkspaceSettings { pub strict_peer_dependencies: Option, pub resolve_peers_from_workspace_root: Option, pub verify_store_integrity: Option, + pub side_effects_cache: Option, pub fetch_retries: Option, pub fetch_retry_factor: Option, pub fetch_retry_mintimeout: Option, @@ -141,8 +142,8 @@ impl WorkspaceSettings { lockfile, prefer_frozen_lockfile, lockfile_include_tarball_url, auto_install_peers, dedupe_peer_dependents, strict_peer_dependencies, resolve_peers_from_workspace_root, verify_store_integrity, - fetch_retries, fetch_retry_factor, fetch_retry_mintimeout, - fetch_retry_maxtimeout, + side_effects_cache, fetch_retries, fetch_retry_factor, + fetch_retry_mintimeout, fetch_retry_maxtimeout, } if let Some(v) = self.modules_dir { diff --git a/pacquet/crates/config/src/workspace_yaml/tests.rs b/pacquet/crates/config/src/workspace_yaml/tests.rs index 3b5d6b46f2..8fac9c716c 100644 --- a/pacquet/crates/config/src/workspace_yaml/tests.rs +++ b/pacquet/crates/config/src/workspace_yaml/tests.rs @@ -125,6 +125,23 @@ fn parses_verify_store_integrity_from_yaml_and_applies() { assert!(!config.verify_store_integrity, "yaml override wins"); } +/// `sideEffectsCache` is the side-effects cache READ-path knob from +/// pnpm-workspace.yaml. Same shape as `verifyStoreIntegrity`: +/// camelCase rename + `apply_to` wiring. Parsing a yaml that flips +/// the default-true setting to false must end up at +/// `config.side_effects_cache == false`. +#[test] +fn parses_side_effects_cache_from_yaml_and_applies() { + let yaml = "sideEffectsCache: false\n"; + let settings: WorkspaceSettings = serde_saphyr::from_str(yaml).unwrap(); + assert_eq!(settings.side_effects_cache, Some(false)); + + let mut config = Config::new(); + assert!(config.side_effects_cache, "the default is `true` to match pnpm"); + settings.apply_to(&mut config, Path::new("/irrelevant")); + assert!(!config.side_effects_cache, "yaml override wins"); +} + #[test] fn apply_leaves_unset_fields_alone() { let yaml = "storeDir: /s\n"; diff --git a/pacquet/crates/graph-hasher/Cargo.toml b/pacquet/crates/graph-hasher/Cargo.toml new file mode 100644 index 0000000000..efd401980c --- /dev/null +++ b/pacquet/crates/graph-hasher/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "pacquet-graph-hasher" +version = "0.0.1" +publish = false +authors.workspace = true +description.workspace = true +edition.workspace = true +homepage.workspace = true +keywords.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +base64 = { workspace = true } +serde_json = { workspace = true } +sha2 = { workspace = true } + +[dev-dependencies] +pretty_assertions = { workspace = true } diff --git a/pacquet/crates/graph-hasher/src/dep_state.rs b/pacquet/crates/graph-hasher/src/dep_state.rs new file mode 100644 index 0000000000..d853b1b6d0 --- /dev/null +++ b/pacquet/crates/graph-hasher/src/dep_state.rs @@ -0,0 +1,321 @@ +use crate::object_hasher::hash_object; +use serde_json::{Value, json}; +use std::collections::{HashMap, HashSet}; + +/// Per-node identifier carrying everything `calc_dep_state` needs to +/// hash a snapshot. Mirrors the relevant subset of pnpm's +/// `DepsGraphNode` at +/// . +/// +/// `full_pkg_id` is the upstream-shaped fingerprint used as the +/// `id` field in the recursive hash — `:` +/// for packages with an integrity (`registry` resolution), +/// or `:` for resolutions +/// without one (e.g. git refs). Pacquet's caller composes this +/// before passing it in; the hasher itself is opaque to how it was +/// computed. +/// +/// `children` maps alias → dep-graph key for the snapshot's +/// children. Pacquet's natural input shape is the lockfile's +/// `snapshots[].dependencies` + `optionalDependencies` flattened, +/// with each value resolved to the snapshot key it points at. +pub struct DepsGraphNode<'a, K> { + pub full_pkg_id: &'a str, + pub children: HashMap<&'a str, K>, +} + +/// Memoized per-depPath state cache. Mirrors pnpm's +/// [`DepsStateCache`](https://github.com/pnpm/pnpm/blob/b4f8f47ac2/deps/graph-hasher/src/index.ts#L21-L23): +/// the result of `hash_object` for each visited node is stashed so +/// the recursive walk over diamond-shaped graphs stays linear. +pub type DepsStateCache = HashMap; + +/// Inputs to [`calc_dep_state`]. Mirrors the option bag at +/// . +pub struct CalcDepStateOptions<'a> { + /// Output of [`crate::engine_name()`] — the platform / arch / + /// node version prefix. Always part of the result. + pub engine_name: &'a str, + /// SHA-256 hex of the patch file for this package (when present). + /// Appended as `;patch=`. + pub patch_file_hash: Option<&'a str>, + /// Whether to include the recursive dep-graph hash as + /// `;deps=`. Upstream sets this to `hasSideEffects` + /// (i.e. `!ignoreScripts && requiresBuild`) at + /// `building/during-install/src/index.ts:202`. + pub include_dep_graph_hash: bool, +} + +/// Mirrors `calcDepState` at +/// . +/// +/// Returns the cache key for the side-effects cache. Format: +/// `[;deps=][;patch=]`. Byte-for-byte +/// parity with pnpm is required — the key is persisted on disk and +/// shared with pnpm. +pub fn calc_dep_state( + graph: &HashMap>, + cache: &mut DepsStateCache, + dep_path: &K, + opts: &CalcDepStateOptions<'_>, +) -> String +where + K: Clone + Eq + std::hash::Hash, +{ + let mut result = opts.engine_name.to_string(); + if opts.include_dep_graph_hash { + let deps_hash = calc_dep_graph_hash(graph, cache, &mut HashSet::new(), dep_path); + result.push_str(";deps="); + result.push_str(&deps_hash); + } + if let Some(patch) = opts.patch_file_hash { + result.push_str(";patch="); + result.push_str(patch); + } + result +} + +/// Recursive helper for the `deps=` portion. Mirrors +/// `calcDepGraphHash` at +/// . +/// +/// Hashes each node as `hashObject({ id, deps })` where `deps` is +/// the alias→child-hash map. `parents` breaks dependency cycles — +/// when a node would re-enter via its own ancestor, the child's +/// contribution becomes `""` (matching upstream's "node not in +/// graph" guard at line 55, which returns the empty string). +fn calc_dep_graph_hash( + graph: &HashMap>, + cache: &mut DepsStateCache, + parents: &mut HashSet, + dep_path: &K, +) -> String +where + K: Clone + Eq + std::hash::Hash, +{ + if let Some(cached) = cache.get(dep_path) { + return cached.clone(); + } + let Some(node) = graph.get(dep_path) else { + return String::new(); + }; + let mut deps_obj = serde_json::Map::new(); + if !node.children.is_empty() && !parents.contains(node.full_pkg_id) { + // Push our `full_pkg_id` for the duration of this subtree + // so cycles short-circuit on the second visit. + let inserted = parents.insert(node.full_pkg_id.to_string()); + for (alias, child_key) in &node.children { + let child_hash = calc_dep_graph_hash(graph, cache, parents, child_key); + deps_obj.insert((*alias).to_string(), Value::String(child_hash)); + } + if inserted { + parents.remove(node.full_pkg_id); + } + } + let hashed = hash_object(&json!({ + "id": node.full_pkg_id, + "deps": Value::Object(deps_obj), + })); + cache.insert(dep_path.clone(), hashed.clone()); + cache.get(dep_path).expect("just inserted").clone() +} + +#[cfg(test)] +mod tests { + use super::{CalcDepStateOptions, DepsGraphNode, calc_dep_state}; + use pretty_assertions::assert_eq; + use std::collections::HashMap; + + /// Engine-only key (no dep graph, no patch). Pure prefix path + /// for the cheapest cache lookup. Mirrors the "include_dep_graph_hash: + /// false" path at + /// . + #[test] + fn engine_only_key() { + let graph: HashMap> = HashMap::new(); + let mut cache = HashMap::new(); + let result = calc_dep_state( + &graph, + &mut cache, + &"foo@1.0.0".to_string(), + &CalcDepStateOptions { + engine_name: "darwin;arm64;node20", + patch_file_hash: None, + include_dep_graph_hash: false, + }, + ); + assert_eq!(result, "darwin;arm64;node20"); + } + + /// Patch hash gets appended as `;patch=`. Combined with + /// the engine prefix when there's no dep graph hash. Mirrors + /// lines 40-42 of `calcDepState`. + #[test] + fn patch_appended_without_dep_graph_hash() { + let graph: HashMap> = HashMap::new(); + let mut cache = HashMap::new(); + let result = calc_dep_state( + &graph, + &mut cache, + &"foo@1.0.0".to_string(), + &CalcDepStateOptions { + engine_name: "linux;x64;node22", + patch_file_hash: Some("sha256-abc"), + include_dep_graph_hash: false, + }, + ); + assert_eq!(result, "linux;x64;node22;patch=sha256-abc"); + } + + /// Dep-graph hash for a leaf (no children) is `hash_object({ + /// id, deps: {} })`. Both sites that consult `deps={}` (the + /// leaf case at `calcLeafGlobalVirtualStorePath` and the + /// children-elided case for cycle/missing-node) must agree. + #[test] + fn dep_graph_hash_for_leaf_uses_id_and_empty_deps() { + let mut graph: HashMap> = HashMap::new(); + graph.insert( + "leaf@1.0.0".to_string(), + DepsGraphNode { full_pkg_id: "leaf@1.0.0:sha512-leaf", children: HashMap::new() }, + ); + let mut cache = HashMap::new(); + let result = calc_dep_state( + &graph, + &mut cache, + &"leaf@1.0.0".to_string(), + &CalcDepStateOptions { + engine_name: "darwin;arm64;node20", + patch_file_hash: None, + include_dep_graph_hash: true, + }, + ); + // Prefix preserved, deps= section appended. + let parts: Vec<&str> = result.split(';').collect(); + assert!(parts.len() == 4, "expected `;;node;deps=`, got {result:?}"); + assert!(parts[3].starts_with("deps="), "fourth segment must be `deps=...`: {result:?}"); + assert!(parts[3][5..].len() >= 40, "hash payload must be non-trivial: {result:?}"); + } + + /// Memoization at the cache layer: `calc_dep_graph_hash` writes + /// each node's hash on first visit and returns the cached + /// value on re-visit. Two leaf nodes with the same + /// `full_pkg_id` must agree. + #[test] + fn cache_makes_repeat_calls_byte_equal() { + let mut graph: HashMap> = HashMap::new(); + graph.insert( + "leaf@1.0.0".to_string(), + DepsGraphNode { full_pkg_id: "leaf@1.0.0:sha512-x", children: HashMap::new() }, + ); + let mut cache = HashMap::new(); + let opts = CalcDepStateOptions { + engine_name: "darwin;arm64;node20", + patch_file_hash: None, + include_dep_graph_hash: true, + }; + let a = calc_dep_state(&graph, &mut cache, &"leaf@1.0.0".to_string(), &opts); + let b = calc_dep_state(&graph, &mut cache, &"leaf@1.0.0".to_string(), &opts); + assert_eq!(a, b); + assert_eq!(cache.len(), 1, "cache must hold exactly the one leaf entry"); + } + + /// Diamond graph: root depends on a and b, both depend on c. + /// Both alias→child entries on the root must agree on the c + /// node's hash, and the recursion must terminate. + #[test] + fn diamond_graph_resolves_consistently() { + let mut graph: HashMap> = HashMap::new(); + let mut root_children = HashMap::new(); + root_children.insert("a", "a@1.0.0".to_string()); + root_children.insert("b", "b@1.0.0".to_string()); + graph.insert( + "root@1.0.0".to_string(), + DepsGraphNode { full_pkg_id: "root@1.0.0:sha512-root", children: root_children }, + ); + let mut a_children = HashMap::new(); + a_children.insert("c", "c@1.0.0".to_string()); + graph.insert( + "a@1.0.0".to_string(), + DepsGraphNode { full_pkg_id: "a@1.0.0:sha512-a", children: a_children }, + ); + let mut b_children = HashMap::new(); + b_children.insert("c", "c@1.0.0".to_string()); + graph.insert( + "b@1.0.0".to_string(), + DepsGraphNode { full_pkg_id: "b@1.0.0:sha512-b", children: b_children }, + ); + graph.insert( + "c@1.0.0".to_string(), + DepsGraphNode { full_pkg_id: "c@1.0.0:sha512-c", children: HashMap::new() }, + ); + let mut cache = HashMap::new(); + let result = calc_dep_state( + &graph, + &mut cache, + &"root@1.0.0".to_string(), + &CalcDepStateOptions { + engine_name: "darwin;arm64;node20", + patch_file_hash: None, + include_dep_graph_hash: true, + }, + ); + // root + a + b + c = 4 cache entries. + assert_eq!(cache.len(), 4, "expected 4 cache entries for diamond, got {cache:#?}"); + assert!(result.contains(";deps="), "result must include deps section: {result:?}"); + } + + /// Cycle: a depends on b, b depends on a. The walk must + /// terminate (parents-set short-circuit) and produce a stable + /// hash. Mirrors upstream's `if (!parents.has(node.fullPkgId))` + /// guard at line 66. + #[test] + fn cyclic_graph_terminates_and_is_stable() { + let mut graph: HashMap> = HashMap::new(); + let mut a_children = HashMap::new(); + a_children.insert("b", "b@1.0.0".to_string()); + graph.insert( + "a@1.0.0".to_string(), + DepsGraphNode { full_pkg_id: "a@1.0.0:sha512-a", children: a_children }, + ); + let mut b_children = HashMap::new(); + b_children.insert("a", "a@1.0.0".to_string()); + graph.insert( + "b@1.0.0".to_string(), + DepsGraphNode { full_pkg_id: "b@1.0.0:sha512-b", children: b_children }, + ); + let mut cache = HashMap::new(); + let opts = CalcDepStateOptions { + engine_name: "darwin;arm64;node20", + patch_file_hash: None, + include_dep_graph_hash: true, + }; + let h1 = calc_dep_state(&graph, &mut cache, &"a@1.0.0".to_string(), &opts); + let h2 = calc_dep_state(&graph, &mut cache, &"a@1.0.0".to_string(), &opts); + assert_eq!(h1, h2); + } + + /// Both patch and dep graph hashes append in upstream's order: + /// `;deps=;patch=`. Mirrors index.js:36-42. + #[test] + fn dep_graph_and_patch_concatenate_in_upstream_order() { + let mut graph: HashMap> = HashMap::new(); + graph.insert( + "x@1.0.0".to_string(), + DepsGraphNode { full_pkg_id: "x@1.0.0:sha512-x", children: HashMap::new() }, + ); + let mut cache = HashMap::new(); + let result = calc_dep_state( + &graph, + &mut cache, + &"x@1.0.0".to_string(), + &CalcDepStateOptions { + engine_name: "darwin;arm64;node20", + patch_file_hash: Some("patchhex"), + include_dep_graph_hash: true, + }, + ); + let deps_pos = result.find(";deps=").expect("deps section present"); + let patch_pos = result.find(";patch=").expect("patch section present"); + assert!(deps_pos < patch_pos, "deps must come before patch in {result:?}"); + } +} diff --git a/pacquet/crates/graph-hasher/src/engine_name.rs b/pacquet/crates/graph-hasher/src/engine_name.rs new file mode 100644 index 0000000000..275e23eaa7 --- /dev/null +++ b/pacquet/crates/graph-hasher/src/engine_name.rs @@ -0,0 +1,92 @@ +/// Compute pnpm's `ENGINE_NAME` string — the same value pnpm uses +/// as the side-effects cache key prefix. +/// +/// Ports +/// : +/// ```js +/// `${process.platform};${process.arch};node${process.version.split('.')[0].substring(1)}` +/// ``` +/// +/// Example outputs: +/// - `"darwin;arm64;node20"` +/// - `"linux;x64;node22"` +/// - `"win32;x64;node24"` +/// +/// `node_major` is the Node major version (e.g. `20`, `22`, `24`). +/// Callers pass it as a number because the discovery side (spawning +/// `node --version` or reading `npm_node_execpath`) is policy and +/// doesn't belong in this hasher crate. +/// +/// `platform` and `arch` default to the running host via the +/// static `std::env::consts` constants mapped through Node's +/// naming scheme. Production callers can pass `None` to get the +/// host values; tests can pin both for cache-key round-trip. +pub fn engine_name(node_major: u32, platform: Option<&str>, arch: Option<&str>) -> String { + let platform = platform.unwrap_or_else(|| host_platform()); + let arch = arch.unwrap_or_else(|| host_arch()); + format!("{platform};{arch};node{node_major}") +} + +/// Map `std::env::consts::OS` to Node's `process.platform` naming. +/// Node uses `darwin` / `linux` / `win32` / `freebsd` / `openbsd` / +/// `sunos` / `aix` / `android`. Rust uses `macos` / `linux` / +/// `windows` / `freebsd` / `openbsd` / `solaris` / `aix` / +/// `android`. Only `macos`, `windows`, and `solaris` differ. +fn host_platform() -> &'static str { + match std::env::consts::OS { + "macos" => "darwin", + "windows" => "win32", + "solaris" => "sunos", + other => other, + } +} + +/// Map `std::env::consts::ARCH` to Node's `process.arch` naming. +/// Node uses `x64` / `arm64` / `ia32` / `arm` / `s390x` / `ppc64` +/// / `ppc64` (LE, same string) / `loong64` / `riscv64`. Rust uses +/// `x86_64` / `aarch64` / `x86` / `arm` / `s390x` / `powerpc64` / +/// `powerpc64le` / `loongarch64` / `riscv64`. Mappings below mirror +/// what Node itself emits on each target — anything left as +/// passthrough (e.g. `arm`, `s390x`, `riscv64`) already matches +/// between the two naming schemes. +fn host_arch() -> &'static str { + match std::env::consts::ARCH { + "x86_64" => "x64", + "aarch64" => "arm64", + "x86" => "ia32", + // Node calls big-endian and little-endian POWER both + // `ppc64`; only big-endian gets `endianness === 'BE'` to + // distinguish them. Rust's two arch values both map here. + "powerpc64" | "powerpc64le" => "ppc64", + "loongarch64" => "loong64", + other => other, + } +} + +#[cfg(test)] +mod tests { + use super::engine_name; + use pretty_assertions::assert_eq; + + /// Format matches pnpm's `${platform};${arch};node${major}` + /// — required for the side-effects cache to interop. + #[test] + fn engine_name_matches_pnpm_format() { + assert_eq!(engine_name(20, Some("darwin"), Some("arm64")), "darwin;arm64;node20"); + assert_eq!(engine_name(22, Some("linux"), Some("x64")), "linux;x64;node22"); + assert_eq!(engine_name(24, Some("win32"), Some("x64")), "win32;x64;node24"); + } + + /// Defaults route through the host mapping. Just assert the + /// shape (three semicolon-separated parts ending in + /// `node`) — the exact OS/arch depends on where the + /// test is run. + #[test] + fn engine_name_host_default_has_expected_shape() { + let name = engine_name(20, None, None); + let parts: Vec<&str> = name.split(';').collect(); + assert_eq!(parts.len(), 3, "expected three parts, got {name:?}"); + assert!(parts[2].starts_with("node"), "third part must start with `node`: {name:?}"); + assert!(parts[2][4..].parse::().is_ok(), "node version must be numeric: {name:?}"); + } +} diff --git a/pacquet/crates/graph-hasher/src/lib.rs b/pacquet/crates/graph-hasher/src/lib.rs new file mode 100644 index 0000000000..2576d03189 --- /dev/null +++ b/pacquet/crates/graph-hasher/src/lib.rs @@ -0,0 +1,32 @@ +//! Pacquet port of pnpm's `@pnpm/crypto.object-hasher` (which itself +//! wraps npm's `object-hash` library) and `@pnpm/deps.graph-hasher`. +//! +//! The cache key used by the side-effects cache is **on disk and +//! shared with pnpm**, so the hash output must be byte-for-byte +//! identical to what pnpm produces. The `object-hash` library uses a +//! specific recursive bytestream format under the hood +//! (`object:::,...` with sorted keys, `string::`, +//! etc.) — pacquet replicates that format here. +//! +//! References (pinned to b4f8f47ac2 / object-hash@3.0.0): +//! - +//! - +//! - + +mod dep_state; +mod engine_name; +mod object_hasher; + +pub use dep_state::{CalcDepStateOptions, DepsStateCache, calc_dep_state}; +pub use engine_name::engine_name; +pub use object_hasher::{hash_object, hash_object_with_encoding, hash_object_without_sorting}; + +/// Hex/base64 encoding option for [`hash_object_with_encoding`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HashEncoding { + Base64, + Hex, +} + +#[cfg(test)] +mod tests; diff --git a/pacquet/crates/graph-hasher/src/object_hasher.rs b/pacquet/crates/graph-hasher/src/object_hasher.rs new file mode 100644 index 0000000000..110a528d52 --- /dev/null +++ b/pacquet/crates/graph-hasher/src/object_hasher.rs @@ -0,0 +1,144 @@ +use crate::HashEncoding; +use base64::Engine as _; +use base64::engine::general_purpose::STANDARD as BASE64; +use serde_json::Value; +use sha2::{Digest, Sha256}; + +/// Mirrors `hashObject` from +/// +/// (sorted keys, sha256, base64). +/// +/// The bytestream the library writes before hashing is described in +/// the (private) `serialize` helper below — it must match pnpm's +/// byte-for-byte because the result is persisted on disk and shared +/// with pnpm. +/// +/// `undefined` in JS maps to no Rust value here; the upstream +/// short-circuit `hashUnknown(undefined)` returns 44 zero characters +/// regardless of options. Callers who need that semantic should +/// branch on the optional before calling. +pub fn hash_object(value: &Value) -> String { + hash_object_with_encoding(value, HashEncoding::Base64, /* sort */ true) +} + +/// Mirrors `hashObjectWithoutSorting` at +/// . +pub fn hash_object_without_sorting(value: &Value, encoding: HashEncoding) -> String { + hash_object_with_encoding(value, encoding, /* sort */ false) +} + +/// General form. `sort = true` sorts object keys before serialization +/// (the `unorderedObjects` option upstream); `sort = false` preserves +/// insertion order. +pub fn hash_object_with_encoding(value: &Value, encoding: HashEncoding, sort: bool) -> String { + let mut bytes = Vec::new(); + serialize(&mut bytes, value, sort); + let digest = Sha256::digest(&bytes); + match encoding { + HashEncoding::Base64 => BASE64.encode(digest), + HashEncoding::Hex => format!("{digest:x}"), + } +} + +/// Recursive bytestream serializer mirroring the `typeHasher` dispatch +/// in object-hash@3.0.0 with `respectType: false`. Only the type +/// arms pacquet actually feeds in are implemented here — `Value` is +/// the union of String / Number / Bool / Null / Array / Object, +/// which covers every input pacquet's graph-hasher and the upstream +/// `hashObject`-using callers ever pass. Anything else would either +/// be unreachable for pacquet or require porting upstream's Date / +/// Set / Map / Buffer arms — explicitly not in scope here. +fn serialize(out: &mut Vec, value: &Value, sort: bool) { + match value { + Value::Null => { + // object-hash writes the literal string `Null` (uppercase + // `N`) for null values at index.js:334. + out.extend_from_slice(b"Null"); + } + Value::Bool(b) => { + // index.js:301-303. + out.extend_from_slice(b"bool:"); + out.extend_from_slice(if *b { b"true" } else { b"false" }); + } + Value::Number(n) => { + // index.js:327-329 — `number:`. JS + // `Number.prototype.toString()` for integers gives the + // plain decimal repr; for floats it produces the + // shortest round-trippable form. Pacquet's cache-key + // inputs only ever hit the integer path in practice + // (`hash_object` is called by `calc_dep_graph_hash` + // with strings + nested objects of strings; the + // upstream `hashObject` test cases use integer + // literals `1`, `2`, `3`). `n.to_string()` from + // `serde_json::Number` matches for integers; for + // non-integer floats `serde_json`'s `f64` formatting + // can diverge from JS's "shortest round-trippable" + // (ECMA-262 §7.1.12.1). If a caller ever passes a + // non-integer in the hot path, the divergence would + // break cross-tool cache-key parity — add the + // JS-compatible float formatter (e.g. the `ryu` crate + // gated on a check) before that lands. + out.extend_from_slice(b"number:"); + out.extend_from_slice(n.to_string().as_bytes()); + } + Value::String(s) => serialize_str(out, s), + Value::Array(arr) => { + // index.js:257-291 — `array::` then each entry. + // pnpm's `hashObject` *does* sort arrays in the + // unordered case, but pacquet does not currently feed + // arrays through this path, so the simpler "ordered" + // variant is what we model. Adding the unordered + // permutation handling can land alongside a real + // caller that needs it. + out.extend_from_slice(b"array:"); + out.extend_from_slice(arr.len().to_string().as_bytes()); + out.push(b':'); + for entry in arr { + serialize(out, entry, sort); + } + } + Value::Object(map) => { + // index.js:225-255 — `object:::,...`. + // Sorted iff `unorderedObjects` (i.e. `sort = true`). + // Each key is dispatched through `_string` and each + // value through the normal dispatcher. + out.extend_from_slice(b"object:"); + out.extend_from_slice(map.len().to_string().as_bytes()); + out.push(b':'); + if sort { + let mut keys: Vec<&String> = map.keys().collect(); + keys.sort(); + for key in keys { + write_pair(out, key, &map[key], sort); + } + } else { + for (key, val) in map { + write_pair(out, key, val, sort); + } + } + } + } +} + +fn write_pair(out: &mut Vec, key: &str, value: &Value, sort: bool) { + // `serialize` for a `Value::String` would force a per-key + // `Value::String(key.to_string())` allocation. Inline the + // string framing instead so hashing a 1000-key object doesn't + // allocate 1000 short-lived Strings on the hot path. + serialize_str(out, key); + out.push(b':'); + serialize(out, value, sort); + out.push(b','); +} + +/// Emit the `string::` framing from object-hash's +/// `_string` arm (index.js:304-307). `length` is UTF-16 code units +/// (JS `.length`), not bytes and not Unicode codepoints. For ASCII +/// strings all three agree. +fn serialize_str(out: &mut Vec, s: &str) { + let utf16_len: usize = s.encode_utf16().count(); + out.extend_from_slice(b"string:"); + out.extend_from_slice(utf16_len.to_string().as_bytes()); + out.push(b':'); + out.extend_from_slice(s.as_bytes()); +} diff --git a/pacquet/crates/graph-hasher/src/tests.rs b/pacquet/crates/graph-hasher/src/tests.rs new file mode 100644 index 0000000000..099ad7d509 --- /dev/null +++ b/pacquet/crates/graph-hasher/src/tests.rs @@ -0,0 +1,120 @@ +use crate::{HashEncoding, hash_object, hash_object_with_encoding, hash_object_without_sorting}; +use pretty_assertions::assert_eq; +use serde_json::json; + +/// Ports `hashObject` `creates a hash` from +/// . +/// Pinning the exact base64 output keeps pacquet's hash byte-for-byte +/// compatible with pnpm's `@pnpm/crypto.object-hasher` — required +/// because the side-effects cache key is shared on disk between the +/// two implementations. +#[test] +fn hash_object_known_base64_value() { + assert_eq!( + hash_object(&json!({ "b": 1, "a": 2 })), + "48AVoXIXcTKcnHt8qVKp5vNw4gyOB5VfztHwtYBRcAQ=", + ); +} + +/// Ports `hashObject` `sorts` from +/// . +/// Two objects with the same keys in a different declared order +/// must produce the same hash. +#[test] +fn hash_object_sorts_object_keys() { + assert_eq!(hash_object(&json!({ "b": 1, "a": 2 })), hash_object(&json!({ "a": 2, "b": 1 }))); +} + +/// Ports `hashObjectWithoutSorting` `creates a hash` from +/// . +/// Different exact value from the sorted variant — different key +/// order produces different bytes. +#[test] +fn hash_object_without_sorting_known_base64_value() { + assert_eq!( + hash_object_without_sorting(&json!({ "b": 1, "a": 2 }), HashEncoding::Base64), + "mh+rYklpd1DBj/dg6dnG+yd8BQhU2UiUoRMSXjPV1JA=", + ); +} + +/// Ports `hashObjectWithoutSorting` `does not sort` from +/// . +/// +/// `serde_json::json!` macro preserves insertion order on its +/// `Map`-backed objects, so `{b:1,a:2}` and `{a:2,b:1}` are distinct +/// inputs here. +#[test] +fn hash_object_without_sorting_distinguishes_key_order() { + let bx = hash_object_without_sorting(&json!({ "b": 1, "a": 2 }), HashEncoding::Base64); + let ax = hash_object_without_sorting(&json!({ "a": 2, "b": 1 }), HashEncoding::Base64); + assert_ne!(bx, ax); +} + +/// Hex encoding is what `calcGraphNodeHash` uses for the GVS path +/// at . +/// Spot-check that the same input under the same options yields +/// the hex repr of the base64 digest. +#[test] +fn hash_object_with_encoding_hex_matches_decoded_base64() { + let value = json!({ "b": 1, "a": 2 }); + let base64 = hash_object(&value); + let hex = hash_object_with_encoding(&value, HashEncoding::Hex, /* sort */ true); + // base64 → bytes; hex → bytes; must match. + let from_b64 = + base64::Engine::decode(&base64::engine::general_purpose::STANDARD, base64.as_bytes()) + .expect("decode base64"); + let from_hex = hex_decode(&hex); + assert_eq!(from_b64, from_hex); +} + +/// The empty object hashes to a stable, non-empty value (the +/// bytestream is literally `object:0:`). +#[test] +fn hash_object_empty_object_is_stable() { + let h1 = hash_object(&json!({})); + let h2 = hash_object(&json!({})); + assert_eq!(h1, h2); + assert!(!h1.is_empty()); +} + +/// Nested-object case that matches the shape pacquet's +/// `calcDepGraphHash` actually feeds into `hash_object`: +/// `{ id: , deps: > }`. Two +/// equivalent inputs (deps in different declared order) must +/// hash identically. +#[test] +fn hash_object_dep_state_shape_sorts_nested_keys() { + let a = hash_object(&json!({ + "id": "foo@1.0.0:sha512-AAA", + "deps": { "b": "h-b", "a": "h-a" }, + })); + let b = hash_object(&json!({ + "deps": { "a": "h-a", "b": "h-b" }, + "id": "foo@1.0.0:sha512-AAA", + })); + assert_eq!(a, b); +} + +/// Non-ASCII string lengths must be counted in UTF-16 code units +/// (JS `string.length`), not in bytes or codepoints. Verifies the +/// `string::` framing. +#[test] +fn hash_object_string_length_counts_utf16_code_units() { + // `"é"` is one codepoint, two UTF-8 bytes, one UTF-16 code + // unit. `"😀"` is one codepoint, four UTF-8 bytes, two UTF-16 + // code units (surrogate pair). + let one_unit = hash_object(&json!({ "k": "é" })); + // Construct an object whose string would be the same length + // as `"é"` under a wrong byte-length implementation, so the + // hashes would collide if pacquet measured bytes. + let two_bytes_ascii = hash_object(&json!({ "k": "ab" })); + assert_ne!(one_unit, two_bytes_ascii, "utf-16 length must differ from byte length"); +} + +fn hex_decode(hex: &str) -> Vec { + assert!(hex.len() % 2 == 0); + (0..hex.len()) + .step_by(2) + .map(|i| u8::from_str_radix(&hex[i..i + 2], 16).expect("hex digit")) + .collect() +} diff --git a/pacquet/crates/package-manager/src/install_package_from_registry/tests.rs b/pacquet/crates/package-manager/src/install_package_from_registry/tests.rs index d0867c8066..d8dbb0e482 100644 --- a/pacquet/crates/package-manager/src/install_package_from_registry/tests.rs +++ b/pacquet/crates/package-manager/src/install_package_from_registry/tests.rs @@ -36,6 +36,7 @@ fn create_config(store_dir: &Path, modules_dir: &Path, virtual_store_dir: &Path) strict_peer_dependencies: false, resolve_peers_from_workspace_root: false, verify_store_integrity: true, + side_effects_cache: true, fetch_retries: 2, fetch_retry_factor: 10, fetch_retry_mintimeout: 10_000, diff --git a/pacquet/crates/store-dir/src/check_pkg_files_integrity.rs b/pacquet/crates/store-dir/src/check_pkg_files_integrity.rs index d9e5602489..b4f97eefe8 100644 --- a/pacquet/crates/store-dir/src/check_pkg_files_integrity.rs +++ b/pacquet/crates/store-dir/src/check_pkg_files_integrity.rs @@ -11,7 +11,7 @@ //! cross-reference (or a pnpm-side change we need to match) stays //! cheap. -use crate::{CafsFileInfo, PackageFilesIndex, StoreDir}; +use crate::{CafsFileInfo, PackageFilesIndex, SideEffectsDiff, StoreDir}; use dashmap::DashSet; use sha2::{Digest, Sha512}; use std::{ @@ -60,10 +60,22 @@ pub type FilesMap = HashMap; /// path` map; it may be partial or empty when a digest in the index /// row couldn't be reconstructed into a CAFS path, so callers should /// gate reuse on `passed` rather than on the map's size. +/// +/// `side_effects_maps` is the optional cache-key → overlaid-FilesMap +/// table from a populated side-effects cache (typically seeded by +/// pnpm). Each value is the post-build files map for one cache key: +/// the base `files_map` with the entry's `added` overlay applied on +/// top of it and `deleted` entries dropped. Mirrors +/// [`PackageFilesResponse.sideEffectsMaps`](https://github.com/pnpm/pnpm/blob/b4f8f47ac2/store/create-cafs-store/src/index.ts#L83-L100) +/// — the importer looks up the entry by the dep-state cache key +/// (`` or `;deps=…;patch=…`, produced by +/// `pacquet-graph-hasher`'s `calc_dep_state`) to decide whether +/// the package is already built. #[derive(Debug)] pub struct VerifyResult { pub passed: bool, pub files_map: FilesMap, + pub side_effects_maps: Option>, } /// Fast path used when `verify-store-integrity` is `false`. @@ -74,13 +86,14 @@ pub struct VerifyResult { /// corrupt CAFS file surfaces lazily at import time (pnpm's `linkOrCopy` /// equivalent). pub fn build_file_maps_from_index(store_dir: &StoreDir, entry: PackageFilesIndex) -> VerifyResult { - let mut files_map = HashMap::with_capacity(entry.files.len()); + let PackageFilesIndex { files, side_effects, .. } = entry; + let mut files_map = HashMap::with_capacity(files.len()); let mut passed = true; // Consume `entry.files` so the owned `String` filenames move into // `files_map` without a per-file clone. On a realistic install the // previous borrow-then-clone cost one allocation per file on every // warm cache hit. - for (filename, info) in entry.files { + for (filename, info) in files { let Some(path) = store_dir.cas_file_path_by_mode(&info.digest, info.mode) else { // A malformed digest (non-hex / too short) makes this entry // unreconstructable. pnpm's `getFilePathByModeInCafs` doesn't @@ -103,7 +116,8 @@ pub fn build_file_maps_from_index(store_dir: &StoreDir, entry: PackageFilesIndex }; files_map.insert(filename, path); } - VerifyResult { passed, files_map } + let side_effects_maps = build_side_effects_maps(store_dir, side_effects, &files_map); + VerifyResult { passed, files_map, side_effects_maps } } /// Careful path used when `verify-store-integrity` is `true` (pnpm's @@ -134,7 +148,7 @@ pub fn check_pkg_files_integrity( // Destructure so the owned `files` HashMap and `algo` String can be // consumed below; moving beats the extra per-file `filename.clone()` // the old borrow-based signature forced on the hot path. - let PackageFilesIndex { files, algo, .. } = entry; + let PackageFilesIndex { files, algo, side_effects, .. } = entry; let mut all_verified = true; let mut files_map = HashMap::with_capacity(files.len()); // `verified_files_cache` is the install-scoped @@ -180,7 +194,71 @@ pub fn check_pkg_files_integrity( } files_map.insert(filename, path); } - VerifyResult { passed: all_verified, files_map } + let side_effects_maps = build_side_effects_maps(store_dir, side_effects, &files_map); + VerifyResult { passed: all_verified, files_map, side_effects_maps } +} + +/// Materialize the per-cache-key overlaid `FilesMap`s from a +/// `PackageFilesIndex.side_effects` entry. Mirrors upstream's +/// [`applySideEffectsDiffWithMaps`](https://github.com/pnpm/pnpm/blob/b4f8f47ac2/store/create-cafs-store/src/index.ts#L103-L121): +/// the overlay is `added` plus base entries that aren't in `deleted`, +/// with `added` winning when both name the same filename. The +/// content of `added` entries is *not* re-verified here — pnpm +/// doesn't do that either; corruption in the side-effects layer +/// would surface at import time via `linkOrCopy` failing on a +/// missing CAS blob. +/// +/// Returns `None` when the entry has no `side_effects` field (the +/// common case for pacquet-written rows today) so callers can +/// trivially distinguish "no cache configured" from "cache configured +/// but empty". +fn build_side_effects_maps( + store_dir: &StoreDir, + side_effects: Option>, + base_files: &FilesMap, +) -> Option> { + let raw = side_effects?; + let mut out: HashMap = HashMap::with_capacity(raw.len()); + 'next_key: for (cache_key, diff) in raw { + let SideEffectsDiff { added, deleted } = diff; + let mut overlay: FilesMap = HashMap::with_capacity(base_files.len()); + if let Some(added) = added { + for (filename, info) in added { + let Some(path) = store_dir.cas_file_path_by_mode(&info.digest, info.mode) else { + // Skip the entire `cache_key` entry rather than + // returning a partial overlay. A future importer + // that flips `is_built = true` on overlay + // presence would otherwise turn a malformed + // digest into a silent corruption: build skipped + // but a required artifact missing from disk. + // Dropping the whole entry sends the package back + // through the rebuild path, which is safe. + tracing::debug!( + target: "pacquet::store_index", + ?filename, + digest = %info.digest, + cache_key, + "malformed CAFS digest in side-effects `added` overlay; dropping this cache_key entry entirely so the importer falls back to rebuild", + ); + continue 'next_key; + }; + overlay.insert(filename, path); + } + } + // Promote `deleted` to a `HashSet` once per cache key so + // the `base_files` walk stays linear in `|base|` instead of + // `O(|base| * |deleted|)`. Pnpm's TS side keeps `deleted` + // as a `Set` for the same reason. + let deleted_set: std::collections::HashSet = + deleted.unwrap_or_default().into_iter().collect(); + for (filename, path) in base_files { + if !deleted_set.contains(filename) && !overlay.contains_key(filename) { + overlay.insert(filename.clone(), path.clone()); + } + } + out.insert(cache_key, overlay); + } + Some(out) } /// Port of pnpm's `verifyFile`. `true` when the on-disk file is either diff --git a/pacquet/crates/store-dir/src/check_pkg_files_integrity/tests.rs b/pacquet/crates/store-dir/src/check_pkg_files_integrity/tests.rs index 3aa0237322..a28e15a111 100644 --- a/pacquet/crates/store-dir/src/check_pkg_files_integrity/tests.rs +++ b/pacquet/crates/store-dir/src/check_pkg_files_integrity/tests.rs @@ -1,7 +1,8 @@ use super::{VerifiedFilesCache, build_file_maps_from_index, check_pkg_files_integrity}; -use crate::{CafsFileInfo, PackageFilesIndex, StoreDir}; +use crate::{CafsFileInfo, PackageFilesIndex, SideEffectsDiff, StoreDir}; use pretty_assertions::assert_eq; use sha2::{Digest, Sha512}; +use std::collections::HashMap; use std::{ fs, io::Write, @@ -321,3 +322,175 @@ impl CafsFileInfo { } } } + +/// No `side_effects` field on the index → `VerifyResult.side_effects_maps` +/// is `None`. Distinguishes "this package never had a cache entry +/// written" from "cache configured but empty for this key" — the +/// importer treats the former as a regular non-built import. +#[test] +fn no_side_effects_yields_none() { + let tmp = tempdir().unwrap(); + let store_dir = StoreDir::new(tmp.path()); + let digest = sha512_hex(b"x"); + let entry = index_with("sha512", vec![("a", info(&digest, 1, 0o644, None))]); + let result = build_file_maps_from_index(&store_dir, entry); + assert!(result.side_effects_maps.is_none()); +} + +/// One cache key, one `added` file, one `deleted` file: the +/// overlay is `added` ∪ (base \ deleted) — entries in `added` win +/// when both layers name the same filename. Mirrors upstream's +/// [`applySideEffectsDiffWithMaps`](https://github.com/pnpm/pnpm/blob/b4f8f47ac2/store/create-cafs-store/src/index.ts#L103-L121). +#[test] +fn side_effects_overlay_adds_and_drops_correctly() { + let tmp = tempdir().unwrap(); + let store_dir = StoreDir::new(tmp.path()); + let base_digest = sha512_hex(b"base"); + let added_digest = sha512_hex(b"added"); + // base files: a.js, b.js. Side-effects for one cache key: + // add c.js, delete b.js. Overlay should land {a.js, c.js}. + let mut side_effects = HashMap::new(); + let mut added = HashMap::new(); + added.insert("c.js".to_string(), info(&added_digest, 5, 0o644, None)); + side_effects.insert( + "darwin;arm64;node20;deps=fake".to_string(), + SideEffectsDiff { added: Some(added), deleted: Some(vec!["b.js".to_string()]) }, + ); + let entry = PackageFilesIndex { + manifest: None, + requires_build: None, + algo: "sha512".into(), + files: HashMap::from([ + ("a.js".to_string(), info(&base_digest, 4, 0o644, None)), + ("b.js".to_string(), info(&base_digest, 4, 0o644, None)), + ]), + side_effects: Some(side_effects), + }; + let result = build_file_maps_from_index(&store_dir, entry); + let maps = result.side_effects_maps.expect("populated"); + let overlay = maps.get("darwin;arm64;node20;deps=fake").expect("entry exists"); + assert!(overlay.contains_key("a.js"), "base survives: {overlay:?}"); + assert!(overlay.contains_key("c.js"), "added overlays: {overlay:?}"); + assert!(!overlay.contains_key("b.js"), "deleted drops: {overlay:?}"); + assert_eq!(overlay.len(), 2); +} + +/// `added` wins over `base` when the filenames collide. The base +/// path is shadowed by the side-effects path. +#[test] +fn side_effects_overlay_added_shadows_base_on_collision() { + let tmp = tempdir().unwrap(); + let store_dir = StoreDir::new(tmp.path()); + let base_digest = sha512_hex(b"base"); + let overlay_digest = sha512_hex(b"overlay-shadow"); + let mut added = HashMap::new(); + added.insert("collide.js".to_string(), info(&overlay_digest, 16, 0o644, None)); + let mut side_effects = HashMap::new(); + side_effects.insert("k1".to_string(), SideEffectsDiff { added: Some(added), deleted: None }); + let entry = PackageFilesIndex { + manifest: None, + requires_build: None, + algo: "sha512".into(), + files: HashMap::from([("collide.js".to_string(), info(&base_digest, 4, 0o644, None))]), + side_effects: Some(side_effects), + }; + let result = build_file_maps_from_index(&store_dir, entry); + let overlay = result.side_effects_maps.unwrap().remove("k1").unwrap(); + let path = overlay.get("collide.js").expect("collide.js present"); + // CAFS layout splits the digest as `<2-char prefix>/`, so the + // path won't contain the digest as a single contiguous substring. + // Verify by checking that the overlay digest's tail (post-prefix + // hex) appears in the path, and that the base digest's tail does + // NOT. + let path_str = path.to_string_lossy(); + assert!( + path_str.contains(&overlay_digest[2..]), + "overlay digest tail should appear in CAFS path: {path:?}", + ); + assert!( + !path_str.contains(&base_digest[2..]), + "base digest tail must NOT appear (shadowed): {path:?}", + ); +} + +/// A malformed digest inside an `added` overlay drops the **whole** +/// cache_key entry — not just the single bad file. Mismatched +/// overlays would otherwise turn a future `is_built = true` decision +/// into a silent corruption (build skipped, required artifact +/// missing). Other cache_key entries on the same package survive. +#[test] +fn side_effects_overlay_malformed_added_digest_drops_cache_key_entry() { + let tmp = tempdir().unwrap(); + let store_dir = StoreDir::new(tmp.path()); + let base_digest = sha512_hex(b"base"); + let good_digest = sha512_hex(b"good-added"); + + let mut k_bad_added = HashMap::new(); + // One good file alongside one bad one — the whole entry should + // still drop, not just the bad file. + k_bad_added.insert("good.js".to_string(), info(&good_digest, 4, 0o644, None)); + k_bad_added.insert("bad.js".to_string(), info("not-hex", 4, 0o644, None)); + + let mut k_good_added = HashMap::new(); + k_good_added.insert("ok.js".to_string(), info(&good_digest, 4, 0o644, None)); + + let mut side_effects = HashMap::new(); + side_effects + .insert("k-bad".to_string(), SideEffectsDiff { added: Some(k_bad_added), deleted: None }); + side_effects + .insert("k-good".to_string(), SideEffectsDiff { added: Some(k_good_added), deleted: None }); + + let entry = PackageFilesIndex { + manifest: None, + requires_build: None, + algo: "sha512".into(), + files: HashMap::from([("base.js".to_string(), info(&base_digest, 4, 0o644, None))]), + side_effects: Some(side_effects), + }; + let result = build_file_maps_from_index(&store_dir, entry); + let maps = result.side_effects_maps.expect("populated"); + assert!(!maps.contains_key("k-bad"), "k-bad must drop entirely on malformed digest"); + assert!(maps.contains_key("k-good"), "k-good must survive: {maps:?}"); +} + +/// Multiple cache keys produce independent overlays. One entry's +/// `added` doesn't bleed into another's. +#[test] +fn side_effects_overlay_keys_are_independent() { + let tmp = tempdir().unwrap(); + let store_dir = StoreDir::new(tmp.path()); + let base_digest = sha512_hex(b"base"); + let added_k1 = sha512_hex(b"k1-added"); + let added_k2 = sha512_hex(b"k2-added"); + let mut side_effects = HashMap::new(); + side_effects.insert( + "k1".to_string(), + SideEffectsDiff { + added: Some(HashMap::from([("a.js".to_string(), info(&added_k1, 1, 0o644, None))])), + deleted: None, + }, + ); + side_effects.insert( + "k2".to_string(), + SideEffectsDiff { + added: Some(HashMap::from([("b.js".to_string(), info(&added_k2, 1, 0o644, None))])), + deleted: None, + }, + ); + let entry = PackageFilesIndex { + manifest: None, + requires_build: None, + algo: "sha512".into(), + files: HashMap::from([("base.js".to_string(), info(&base_digest, 4, 0o644, None))]), + side_effects: Some(side_effects), + }; + let result = build_file_maps_from_index(&store_dir, entry); + let maps = result.side_effects_maps.unwrap(); + let k1 = maps.get("k1").unwrap(); + let k2 = maps.get("k2").unwrap(); + assert!(k1.contains_key("a.js") && !k1.contains_key("b.js"), "k1: {k1:?}"); + assert!(k2.contains_key("b.js") && !k2.contains_key("a.js"), "k2: {k2:?}"); + // Both share base.js. + assert!(k1.contains_key("base.js")); + assert!(k2.contains_key("base.js")); +}